From 8e29f89b39d0afae17210687ba994dd4f908e477 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 10:50:35 +0200 Subject: [PATCH 01/44] feat(plugin-abi): external-plugin ABI schema + abi/sdk crates (M0) Foundation for running plugins out-of-process over 3 transports (proto/shm/wasm), all behind one logical interface. - proto/plugin/v1/*.proto (pkg heph.plugin.v1): common, targetdef, provider, driver, callback, envelope. Source of truth, mirrors the in-process hplugin/hmodel types field-for-field. - crates/plugin-abi: re-exports generated pb, ABI_SEMVER, and a rkyv zero-copy mirror of the hot-path messages for the shm fast path, with a parity test (proto<->rkyv round-trip) so the encodings can't drift. - crates/plugin-sdk: Rust guest SDK skeleton (Ctx cancellation, HostClient callback surface; authors implement the same hplugin::Provider/Driver traits). Transport impls land in M1. - wit/heph-plugin.wit: contract mirror for the wasm tier (validated M4). - proto/Cargo.toml.tpl: add pbjson dep (prost-serde emits pbjson code for numeric fields). Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 179 ++++++++++++++++++++++++--- Cargo.toml | 2 +- crates/plugin-abi/Cargo.toml | 15 +++ crates/plugin-abi/README.md | 54 +++++++++ crates/plugin-abi/src/lib.rs | 23 ++++ crates/plugin-abi/src/shm_types.rs | 188 +++++++++++++++++++++++++++++ crates/plugin-abi/tests/parity.rs | 105 ++++++++++++++++ crates/plugin-sdk/Cargo.toml | 15 +++ crates/plugin-sdk/src/ctx.rs | 46 +++++++ crates/plugin-sdk/src/host.rs | 36 ++++++ crates/plugin-sdk/src/lib.rs | 26 ++++ proto/Cargo.toml.tpl | 1 + proto/plugin/v1/callback.proto | 95 +++++++++++++++ proto/plugin/v1/common.proto | 108 +++++++++++++++++ proto/plugin/v1/driver.proto | 58 +++++++++ proto/plugin/v1/envelope.proto | 117 ++++++++++++++++++ proto/plugin/v1/provider.proto | 60 +++++++++ proto/plugin/v1/targetdef.proto | 125 +++++++++++++++++++ wit/heph-plugin.wit | 166 +++++++++++++++++++++++++ 19 files changed, 1403 insertions(+), 16 deletions(-) create mode 100644 crates/plugin-abi/Cargo.toml create mode 100644 crates/plugin-abi/README.md create mode 100644 crates/plugin-abi/src/lib.rs create mode 100644 crates/plugin-abi/src/shm_types.rs create mode 100644 crates/plugin-abi/tests/parity.rs create mode 100644 crates/plugin-sdk/Cargo.toml create mode 100644 crates/plugin-sdk/src/ctx.rs create mode 100644 crates/plugin-sdk/src/host.rs create mode 100644 crates/plugin-sdk/src/lib.rs create mode 100644 proto/plugin/v1/callback.proto create mode 100644 proto/plugin/v1/common.proto create mode 100644 proto/plugin/v1/driver.proto create mode 100644 proto/plugin/v1/envelope.proto create mode 100644 proto/plugin/v1/provider.proto create mode 100644 proto/plugin/v1/targetdef.proto create mode 100644 wit/heph-plugin.wit diff --git a/Cargo.lock b/Cargo.lock index 2ff20d24..851f2520 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -166,7 +166,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -177,7 +177,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -345,6 +345,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -505,6 +511,29 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64fa3c856b712db6612c019f14756e64e4bcea13337a6b33b696333a9eaa2d06" +[[package]] +name = "bytecheck" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "rancor", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "bytemuck" version = "1.25.0" @@ -1433,7 +1462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2088,7 +2117,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -2788,6 +2817,26 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -2883,7 +2932,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3144,7 +3193,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -3330,6 +3379,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pbjson" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e6349fa080353f4a597daffd05cb81572a9c031a6d4fff7e504947496fcc68" +dependencies = [ + "base64 0.21.7", + "serde", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3475,6 +3534,15 @@ dependencies = [ "walk", ] +[[package]] +name = "plugin-abi" +version = "0.1.0" +dependencies = [ + "anyhow", + "proto-gen", + "rkyv", +] + [[package]] name = "plugin-buildfile" version = "0.1.0" @@ -3588,6 +3656,18 @@ dependencies = [ "plugin", ] +[[package]] +name = "plugin-sdk" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "core", + "model", + "plugin", + "plugin-abi", +] + [[package]] name = "plugingo-e2e" version = "0.1.0" @@ -3829,6 +3909,7 @@ dependencies = [ name = "proto-gen" version = "0.1.0" dependencies = [ + "pbjson", "prost 0.14.4", "prost-types 0.14.4", "serde", @@ -3844,6 +3925,26 @@ dependencies = [ "cc", ] +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "quick-xml" version = "0.39.4" @@ -3986,6 +4087,15 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "rancor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +dependencies = [ + "ptr_meta", +] + [[package]] name = "rand" version = "0.8.6" @@ -4226,13 +4336,22 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rend" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" +dependencies = [ + "bytecheck", +] + [[package]] name = "reqwest" version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -4274,7 +4393,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-core", @@ -4321,6 +4440,36 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +dependencies = [ + "bytecheck", + "bytes", + "hashbrown 0.17.1", + "indexmap", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "rsqlite-vfs" version = "0.1.1" @@ -4377,7 +4526,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4435,7 +4584,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4844,7 +4993,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4903,7 +5052,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5209,7 +5358,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5250,7 +5399,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bitflags 2.13.0", "fancy-regex 0.11.0", "filedescriptor", @@ -6093,7 +6242,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 29474738..c1ea8572 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2024" [workspace] -members = ["gen/proto", "crates/e2e", "crates/testkit", "crates/plugingo-e2e", "crates/htspec-derive", "crates/core", "crates/walk", "crates/proc", "crates/model", "crates/sandboxfuse", "crates/plugin", "crates/builtins", "crates/plugin-buildfile", "crates/driver-support", "crates/plugin-exec", "crates/plugin-nix", "crates/plugin-query", "crates/plugin-go", "crates/telemetry", "crates/tui", "crates/lock", "crates/engine"] +members = ["gen/proto", "crates/e2e", "crates/testkit", "crates/plugingo-e2e", "crates/htspec-derive", "crates/core", "crates/walk", "crates/proc", "crates/model", "crates/sandboxfuse", "crates/plugin", "crates/plugin-abi", "crates/plugin-sdk", "crates/builtins", "crates/plugin-buildfile", "crates/driver-support", "crates/plugin-exec", "crates/plugin-nix", "crates/plugin-query", "crates/plugin-go", "crates/telemetry", "crates/tui", "crates/lock", "crates/engine"] [profile.profiling] inherits = "release" diff --git a/crates/plugin-abi/Cargo.toml b/crates/plugin-abi/Cargo.toml new file mode 100644 index 00000000..a5db6d17 --- /dev/null +++ b/crates/plugin-abi/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "plugin-abi" +version = "0.1.0" +edition = "2024" + +[lints] +workspace = true + +[dependencies] +hproto-gen = { package = "proto-gen", path = "../../gen/proto", features = ["proto_full"] } +anyhow = "1.0.102" +rkyv = "0.8" + +[dev-dependencies] +rkyv = "0.8" diff --git a/crates/plugin-abi/README.md b/crates/plugin-abi/README.md new file mode 100644 index 00000000..25cde45d --- /dev/null +++ b/crates/plugin-abi/README.md @@ -0,0 +1,54 @@ +# External-plugin ABI + +heph plugins can run out-of-process over three transports — **proto** (prost over +a UDS socketpair, portable/polyglot), **shm** (iceoryx2 shared memory, ultra +low-overhead, used by plugin-go), and **wasm** (in-process wasmtime component, +capability-sandboxed) — all behind one logical interface. In-process plugins +(`query`, `hostbin`, `group`, `statictarget`) keep working unchanged; in-proc and +remote coexist permanently. + +Full design + rationale: the approved plan (`.claude/plans/`). This file is the +quick map. + +## Schemas (one contract, three encodings) + +- **Source of truth: proto** — `proto/plugin/v1/*.proto`, package `heph.plugin.v1`, + generated via `buf` (`gen` devenv command) into `gen/proto` (`proto-gen` crate, + alias `hproto-gen`). Files: `common`, `targetdef`, `provider`, `driver`, + `callback`, `envelope`. +- **rkyv mirror** — `crates/plugin-abi/src/shm_types.rs` mirrors only the ≤5 + hot-path messages (`ResultRequest/Response`, `NoteDep{Request,Response}`, …) for + the shm zero-copy fast path. A parity test (`crates/plugin-abi/tests/parity.rs`) + round-trips proto → rkyv → proto so the two cannot drift. +- **WIT mirror** — `wit/heph-plugin.wit` mirrors the interface for the wasm tier + (validated + parity-checked in M4). + +The shm `payload_encoding` is negotiated at handshake: `Rkyv` (Rust, native +zero-copy), `Capnp` (polyglot, e.g. Go), `Prost` elsewhere. + +## Crates + +- `crates/plugin` — the existing in-process contract (`Provider`/`Driver`/ + `ProviderExecutor`/`EResult`/`TargetSpec`/`TargetDef`). Engine-independent. +- `crates/plugin-abi` — raw wire types: re-exports the generated proto (`pb`), + `ABI_SEMVER`, and the rkyv hot-message mirror. Deps: `hproto-gen` only (light). +- `crates/plugin-sdk` — Rust guest SDK. Authors implement the **same** + `hplugin::Provider`/`Driver` traits; the SDK hides framing/encoding/streaming, + manages leases, and exposes `Ctx` (cancellation) + `HostClient` (callbacks). +- `crates/plugin-remote` (M1) — host-side adapter implementing `hplugin` traits + over a transport; registered via the existing engine factory hooks (engine core + unchanged). Transports are cargo-feature-gated modules. + +## Key invariants + +- **Result/input artifacts are abstract readers, not paths.** `Content` (in + `hcore::hartifactcontent`) exposes only `reader`/`walk`/`seekable_reader` — no + path/fd. They cross as opaque `ArtifactHandle` + metadata; bytes are pulled + lazily (`open_artifact`), zero-copy via fd/`memfd` when file-backed. Only driver + `run` outputs (fresh files) cross as paths. +- **The dep-cycle check lives host-side.** Every `result`/`note_dep` registers the + `parent → addr` DepDag edge before any await. Never memoize the callback on the + addr in a plugin — cache the derived output, never the call. `note_dep` is the + cache-hit fast path that registers the edge without fetching. +- **`raw_def` is opaque pass-through** — `RawDefBlob`, produced and consumed only + by the originating driver; the host never inspects it. diff --git a/crates/plugin-abi/src/lib.rs b/crates/plugin-abi/src/lib.rs new file mode 100644 index 00000000..2c0897bf --- /dev/null +++ b/crates/plugin-abi/src/lib.rs @@ -0,0 +1,23 @@ +//! Transport-agnostic wire ABI for heph external plugins. +//! +//! This crate is the raw wire layer shared by all three transports (proto/UDS, +//! shm/iceoryx2, wasm). Plugin authors do NOT use it directly — they use the +//! per-language SDK (`plugin-sdk` for Rust), which sits on top. +//! +//! - [`pb`] re-exports the prost-generated message types (source of truth, +//! generated from `proto/plugin/v1/*.proto` via `buf`). +//! - [`shm_types`] holds rkyv zero-copy mirrors of the ≤5 hot-path messages, +//! used by the shm fast path, with `From`/`Into` conversions to [`pb`]. +//! +//! The handshake negotiates the payload encoding per transport: `Rkyv` (native +//! zero-copy) for Rust plugins on shm, `Capnp` for polyglot shm, `Prost` +//! everywhere else. + +/// The prost-generated wire message types (`heph.plugin.v1`). +pub use hproto_gen::heph::plugin::v1 as pb; + +/// ABI semantic version. Major must match exactly between host and plugin; +/// minor is negotiated to `min(host, plugin)` at handshake. +pub const ABI_SEMVER: &str = "0.1.0"; + +pub mod shm_types; diff --git a/crates/plugin-abi/src/shm_types.rs b/crates/plugin-abi/src/shm_types.rs new file mode 100644 index 00000000..e25ae5c8 --- /dev/null +++ b/crates/plugin-abi/src/shm_types.rs @@ -0,0 +1,188 @@ +//! rkyv zero-copy mirrors of the hot-path messages. +//! +//! Only the messages on the shm fast path are mirrored here (callback `result` +//! / `note_dep` + their payloads). The cold path reuses prost inside the shm +//! sample. Each mirror has `From`/`Into` conversions to the prost type in +//! [`crate::pb`]; the parity test (`tests/parity.rs`) round-trips proto -> rkyv +//! bytes -> proto and asserts equality, so the two representations cannot drift. +//! +//! `args` maps are carried as a sorted `Vec<(String, String)>` for a stable, +//! deterministic archived layout (rkyv map support is avoided on the hot path). + +use crate::pb; + +/// rkyv mirror of [`pb::Addr`]. +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct Addr { + pub package: String, + pub name: String, + /// Sorted (key, value) pairs. + pub args: Vec<(String, String)>, +} + +/// rkyv mirror of [`pb::ArtifactHandle`]. +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct ArtifactHandle { + pub handle_id: String, + pub group: String, + pub name: String, + pub hashout: String, + pub byte_size: u64, + pub support: bool, +} + +/// rkyv mirror of [`pb::ResultRequest`]. +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct ResultRequest { + pub request_id: String, + pub addr: Addr, +} + +/// rkyv mirror of [`pb::ResultResponse`]. +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct ResultResponse { + pub lease_id: String, + pub artifacts: Vec, +} + +/// rkyv mirror of [`pb::NoteDepRequest`]. +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct NoteDepRequest { + pub request_id: String, + pub parent: Addr, + pub addr: Addr, +} + +/// rkyv mirror of [`pb::NoteDepResponse`]. +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct NoteDepResponse { + pub ok: bool, + pub cycle: bool, + pub message: String, +} + +// ---- conversions ---- + +impl From for Addr { + fn from(a: pb::Addr) -> Self { + let mut args: Vec<(String, String)> = a.args.into_iter().collect(); + args.sort(); + Addr { + package: a.package, + name: a.name, + args, + } + } +} + +impl From for pb::Addr { + fn from(a: Addr) -> Self { + pb::Addr { + package: a.package, + name: a.name, + args: a.args.into_iter().collect(), + } + } +} + +impl From for ArtifactHandle { + fn from(h: pb::ArtifactHandle) -> Self { + ArtifactHandle { + handle_id: h.handle_id, + group: h.group, + name: h.name, + hashout: h.hashout, + byte_size: h.byte_size, + support: h.support, + } + } +} + +impl From for pb::ArtifactHandle { + fn from(h: ArtifactHandle) -> Self { + pb::ArtifactHandle { + handle_id: h.handle_id, + group: h.group, + name: h.name, + hashout: h.hashout, + byte_size: h.byte_size, + support: h.support, + } + } +} + +impl From for ResultRequest { + fn from(r: pb::ResultRequest) -> Self { + ResultRequest { + request_id: r.request_id, + addr: r.addr.unwrap_or_default().into(), + } + } +} + +impl From for pb::ResultRequest { + fn from(r: ResultRequest) -> Self { + pb::ResultRequest { + request_id: r.request_id, + addr: Some(r.addr.into()), + } + } +} + +impl From for ResultResponse { + fn from(r: pb::ResultResponse) -> Self { + ResultResponse { + lease_id: r.lease_id, + artifacts: r.artifacts.into_iter().map(Into::into).collect(), + } + } +} + +impl From for pb::ResultResponse { + fn from(r: ResultResponse) -> Self { + pb::ResultResponse { + lease_id: r.lease_id, + artifacts: r.artifacts.into_iter().map(Into::into).collect(), + } + } +} + +impl From for NoteDepRequest { + fn from(r: pb::NoteDepRequest) -> Self { + NoteDepRequest { + request_id: r.request_id, + parent: r.parent.unwrap_or_default().into(), + addr: r.addr.unwrap_or_default().into(), + } + } +} + +impl From for pb::NoteDepRequest { + fn from(r: NoteDepRequest) -> Self { + pb::NoteDepRequest { + request_id: r.request_id, + parent: Some(r.parent.into()), + addr: Some(r.addr.into()), + } + } +} + +impl From for NoteDepResponse { + fn from(r: pb::NoteDepResponse) -> Self { + NoteDepResponse { + ok: r.ok, + cycle: r.cycle, + message: r.message, + } + } +} + +impl From for pb::NoteDepResponse { + fn from(r: NoteDepResponse) -> Self { + pb::NoteDepResponse { + ok: r.ok, + cycle: r.cycle, + message: r.message, + } + } +} diff --git a/crates/plugin-abi/tests/parity.rs b/crates/plugin-abi/tests/parity.rs new file mode 100644 index 00000000..326e4255 --- /dev/null +++ b/crates/plugin-abi/tests/parity.rs @@ -0,0 +1,105 @@ +//! Parity test: proto -> rkyv mirror -> rkyv bytes -> (zero-copy access + +//! deserialize) -> proto must round-trip exactly. This catches drift between +//! the prost schema and the hand-written rkyv hot-message mirrors: if a field +//! is added/removed on one side, the conversion stops compiling or the +//! round-trip stops matching. + +use plugin_abi::{pb, shm_types}; +use std::collections::HashMap; + +fn sample_addr(pkg: &str, name: &str) -> pb::Addr { + let mut args = HashMap::new(); + args.insert("goos".to_string(), "linux".to_string()); + args.insert("goarch".to_string(), "amd64".to_string()); + pb::Addr { + package: pkg.to_string(), + name: name.to_string(), + args, + } +} + +/// proto -> mirror -> rkyv bytes -> checked zero-copy access + deserialize -> +/// mirror -> proto == original. Run on concrete types so the archived type's +/// `CheckBytes` bound (from the `Archive` derive) resolves. +macro_rules! roundtrip { + ($proto:ty, $mirror:ty, $value:expr) => {{ + let original: $proto = $value; + let mirror: $mirror = original.clone().into(); + let bytes = rkyv::to_bytes::(&mirror).expect("rkyv serialize"); + // zero-copy access must validate + let archived = + rkyv::access::<<$mirror as rkyv::Archive>::Archived, rkyv::rancor::Error>(&bytes) + .expect("rkyv access"); + let back: $mirror = + rkyv::deserialize::<$mirror, rkyv::rancor::Error>(archived).expect("rkyv deserialize"); + let proto_back: $proto = back.into(); + assert_eq!(original, proto_back); + }}; +} + +#[test] +fn result_request_parity() { + roundtrip!( + pb::ResultRequest, + shm_types::ResultRequest, + pb::ResultRequest { + request_id: "req-1".to_string(), + addr: Some(sample_addr("//foo/bar", "lib")), + } + ); +} + +#[test] +fn result_response_parity() { + roundtrip!( + pb::ResultResponse, + shm_types::ResultResponse, + pb::ResultResponse { + lease_id: "lease-7".to_string(), + artifacts: vec![ + pb::ArtifactHandle { + handle_id: "h1".to_string(), + group: "out".to_string(), + name: "package.bin".to_string(), + hashout: "abc123".to_string(), + byte_size: 4096, + support: false, + }, + pb::ArtifactHandle { + handle_id: "h2".to_string(), + group: "log".to_string(), + name: "stderr".to_string(), + hashout: String::new(), + byte_size: 0, + support: true, + }, + ], + } + ); +} + +#[test] +fn note_dep_request_parity() { + roundtrip!( + pb::NoteDepRequest, + shm_types::NoteDepRequest, + pb::NoteDepRequest { + request_id: "req-2".to_string(), + parent: Some(sample_addr("//a", "x")), + addr: Some(sample_addr("//b", "y")), + } + ); +} + +#[test] +fn note_dep_response_parity() { + roundtrip!( + pb::NoteDepResponse, + shm_types::NoteDepResponse, + pb::NoteDepResponse { + ok: false, + cycle: true, + message: "cycle: //a:x -> //b:y -> //a:x".to_string(), + } + ); +} diff --git a/crates/plugin-sdk/Cargo.toml b/crates/plugin-sdk/Cargo.toml new file mode 100644 index 00000000..d754b7a8 --- /dev/null +++ b/crates/plugin-sdk/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "plugin-sdk" +version = "0.1.0" +edition = "2024" + +[lints] +workspace = true + +[dependencies] +hplugin = { package = "plugin", path = "../plugin" } +hmodel = { package = "model", path = "../model" } +hcore = { package = "core", path = "../core" } +plugin-abi = { path = "../plugin-abi" } +anyhow = "1.0.102" +async-trait = "0.1.89" diff --git a/crates/plugin-sdk/src/ctx.rs b/crates/plugin-sdk/src/ctx.rs new file mode 100644 index 00000000..cd58f603 --- /dev/null +++ b/crates/plugin-sdk/src/ctx.rs @@ -0,0 +1,46 @@ +//! Per-call context handed to plugin methods by the SDK. + +use hcore::hasync::Cancellable; +use std::sync::Arc; + +/// Context for one in-flight plugin call. Wraps the request id and the +/// cancellation signal so authors can do `ctx.is_cancelled()` / `ctx.cancelled().await` +/// without touching the transport. The SDK fires the signal when a `Cancel` +/// frame arrives for this request (or, for the wasm tier, when the host's +/// `cancelled()` import returns true). +#[derive(Clone)] +pub struct Ctx { + request_id: String, + cancel: Arc, +} + +impl Ctx { + /// Build a context from a request id and a cancellation signal. + pub fn new(request_id: impl Into, cancel: Arc) -> Self { + Self { + request_id: request_id.into(), + cancel, + } + } + + /// The request id this call belongs to (used to correlate callbacks). + pub fn request_id(&self) -> &str { + &self.request_id + } + + /// Non-blocking cancellation check. + pub fn is_cancelled(&self) -> bool { + self.cancel.is_cancelled() + } + + /// Resolves when this call is cancelled. + pub async fn cancelled(&self) { + self.cancel.cancelled().await; + } + + /// Borrow the underlying signal to pass into engine/SDK helpers that take + /// `&dyn Cancellable`. + pub fn cancellable(&self) -> &(dyn Cancellable + Send + Sync) { + &*self.cancel + } +} diff --git a/crates/plugin-sdk/src/host.rs b/crates/plugin-sdk/src/host.rs new file mode 100644 index 00000000..e65206c3 --- /dev/null +++ b/crates/plugin-sdk/src/host.rs @@ -0,0 +1,36 @@ +//! The host callback surface, from the guest plugin's perspective. + +use anyhow::Result; +use async_trait::async_trait; +use hmodel::htaddr::Addr; +use hmodel::htmatcher::Matcher; +use hplugin::eresult::EResult; +use std::sync::Arc; + +/// The callbacks a plugin makes back into the host while serving `get`/`parse`. +/// +/// Mirrors [`hplugin::provider::ProviderExecutor`] plus the [`HostClient::note_dep`] +/// cache-hit fast path. The SDK implements this over the negotiated transport +/// (M1, in `plugin-remote`); plugin code uses it exactly like the in-process +/// `executor`, so the dep-resolution call sites don't change when a plugin goes +/// out-of-process. +/// +/// Like `ProviderExecutor`, **do not memoize these calls on the addr inside a +/// provider** — the host registers the `parent -> addr` dep edge on each call +/// (the synchronous cycle check). Cache the derived output, never the call. +#[async_trait] +pub trait HostClient: Send + Sync { + /// Resolve a target's result (artifacts + lease). Registers the dep edge + /// host-side; bytes are pulled lazily through the returned `EResult`'s + /// SDK-backed `Content` handles. + async fn result(&self, addr: &Addr) -> Result>; + + /// Resolve all targets matching `m` (`extra_skip` unioned with the request's + /// skip set for this call only). + async fn query(&self, m: &Matcher, extra_skip: &[String]) -> Result>; + + /// Register a `parent -> addr` dependency edge only, without fetching the + /// result — the cache-hit fast path. Returns an error whose context marks a + /// cycle when the edge closes one. Batched by the SDK on the shm transport. + async fn note_dep(&self, parent: &Addr, addr: &Addr) -> Result<()>; +} diff --git a/crates/plugin-sdk/src/lib.rs b/crates/plugin-sdk/src/lib.rs new file mode 100644 index 00000000..3607af85 --- /dev/null +++ b/crates/plugin-sdk/src/lib.rs @@ -0,0 +1,26 @@ +//! Rust SDK for writing heph plugins that run out-of-process. +//! +//! This is the ergonomic surface plugin authors use; it sits on top of the raw +//! wire layer ([`plugin_abi`]) and hides framing, encoding, streaming, leases +//! and cancellation. A plugin author implements the **same** [`hplugin::provider::Provider`] +//! / [`hplugin::driver::Driver`] traits as an in-process plugin, so moving an +//! existing plugin out-of-process is near-zero-change. +//! +//! One SDK exists per guest language; this is the Rust one. The proto/WIT +//! contract is designed so a future Go/Python SDK is mechanical. +//! +//! Status: M0 skeleton — public surface (author traits, [`Ctx`], [`HostClient`]) +//! is defined; transport-backed implementations and the `serve` entry point +//! land in M1 (`plugin-remote`). + +pub mod ctx; +pub mod host; + +pub use ctx::Ctx; +pub use host::HostClient; + +/// Re-export of the author-facing contract so a plugin depends only on the SDK. +pub use hplugin::{driver, eresult, provider}; + +/// The ABI version this SDK builds against (negotiated at handshake). +pub use plugin_abi::ABI_SEMVER; diff --git a/proto/Cargo.toml.tpl b/proto/Cargo.toml.tpl index 3483de64..0d5ed488 100644 --- a/proto/Cargo.toml.tpl +++ b/proto/Cargo.toml.tpl @@ -7,6 +7,7 @@ edition = "2021" prost = "0.14" prost-types = "0.14" serde = { version = "1.0", features = ["derive"] } +pbjson = "0.7" [lints.clippy] all = "allow" diff --git a/proto/plugin/v1/callback.proto b/proto/plugin/v1/callback.proto new file mode 100644 index 00000000..843b7cb2 --- /dev/null +++ b/proto/plugin/v1/callback.proto @@ -0,0 +1,95 @@ +syntax = "proto3"; + +// Host-exported callback surface (the AbiHost). The plugin calls back into the +// host while serving get()/parse(). Carries the bidirectional hot path. +package heph.plugin.v1; + +import "plugin/v1/common.proto"; + +// A handle to one result/input artifact. Result artifacts are ABSTRACT readers +// (file / sqlite blob / in-mem / remote), NOT paths — so they cross as an +// opaque handle + metadata; bytes are pulled lazily via OpenArtifact. The host +// holds the underlying read-guard alive under the enclosing lease_id. +message ArtifactHandle { + string handle_id = 1; + string group = 2; + string name = 3; + string hashout = 4; + uint64 byte_size = 5; // 0 if the backing can't answer cheaply + bool support = 6; // support_artifact vs primary artifact +} + +// executor.result(addr). Registers the parent->addr DepDag edge host-side +// (cycle check) and returns artifact handles + a lease. +message ResultRequest { + string request_id = 1; + Addr addr = 2; +} +message ResultResponse { + string lease_id = 1; + repeated ArtifactHandle artifacts = 2; +} + +// Lazy byte access for an artifact handle. Reply streams bytes via the envelope +// stream channel (StreamItem), keyed by the issuing frame id. +message OpenArtifactRequest { + string lease_id = 1; + string handle_id = 2; + uint64 offset = 3; +} + +// Cache-hit fast path: register the dep edge only (no lease, batchable). +// Returns CycleError when the inserted edge closes a cycle. +message NoteDepRequest { + string request_id = 1; + Addr parent = 2; + Addr addr = 3; +} +message NoteDepResponse { + bool ok = 1; + bool cycle = 2; + string message = 3; +} + +// executor.query(matcher, extra_skip). +message QueryRequest { + string request_id = 1; + Matcher matcher = 2; + repeated string extra_skip = 3; +} +message QueryResponse { + repeated Addr addrs = 1; +} + +// The host-side CachedWalker (used by list_packages). +message WalkRequest { + string dir = 1; +} +enum EntryKind { + ENTRY_KIND_UNSPECIFIED = 0; + ENTRY_KIND_FILE = 1; + ENTRY_KIND_DIR = 2; + ENTRY_KIND_SYMLINK = 3; + ENTRY_KIND_OTHER = 4; +} +message DirEntry { + string name = 1; + EntryKind kind = 2; +} +message WalkResponse { + repeated DirEntry entries = 1; +} + +// Provider/driver config access. +message ConfigGetRequest { + string key = 1; +} +message ConfigGetResponse { + Value value = 1; +} + +// Drop the read-guards the host holds under lease_id. +message ReleaseLeaseRequest { + string lease_id = 1; +} +message ReleaseLeaseResponse {} diff --git a/proto/plugin/v1/common.proto b/proto/plugin/v1/common.proto new file mode 100644 index 00000000..3cb8006d --- /dev/null +++ b/proto/plugin/v1/common.proto @@ -0,0 +1,108 @@ +syntax = "proto3"; + +// Shared value/address/sandbox types for the heph external-plugin ABI. +// Mirrors the in-process contract types in crates/plugin and crates/model so +// the same logical interface can be carried over proto (UDS), shm (iceoryx2) +// and wasm transports. See crates/plugin-abi/README.md. +package heph.plugin.v1; + +// Dynamic config value. Mirrors hcore::htvalue::Value. Kept distinct from +// google.protobuf.Value because heph distinguishes Int/Uint/Float and Null. +message Value { + message Null {} + message Map { + map entries = 1; + } + message List { + repeated Value items = 1; + } + oneof kind { + string string_val = 1; + bool bool_val = 2; + double float_val = 3; + int64 int_val = 4; + uint64 uint_val = 5; + Null null_val = 6; + Map map_val = 7; + List list_val = 8; + } +} + +// Target address. Mirrors hmodel::Addr (package //pkg : name @args). +message Addr { + string package = 1; + string name = 2; + map args = 3; +} + +// A reference to (an output of) a target. Mirrors driver::TargetAddr. +message TargetAddr { + Addr ref = 1; + optional string output = 2; + repeated string filters = 3; +} + +// Provider-emitted state attached to a package. Mirrors provider::State. +message State { + string package = 1; + string provider = 2; + map state = 3; +} + +// Query predicate. Mirrors hmodel::Matcher (recursive). +message Matcher { + message List { + repeated Matcher matchers = 1; + } + oneof kind { + Addr addr = 1; + string label = 2; + string package = 3; + string package_prefix = 4; + string tree_output_to = 5; + List or = 6; + List and = 7; + Matcher not = 8; + } +} + +// ---- Sandbox (driver::sandbox) ---- + +enum DepMode { + DEP_MODE_UNSPECIFIED = 0; + DEP_MODE_NONE = 1; + DEP_MODE_LINK = 2; +} + +message Tool { + TargetAddr ref = 1; + string group = 2; + bool hash = 3; + string id = 4; +} + +message Dep { + TargetAddr ref = 1; + DepMode mode = 2; + string group = 3; + bool runtime = 4; + bool hash = 5; + string id = 6; +} + +message Env { + // EnvValue: literal string, or "pass" (inherit from host env). + oneof value { + string literal = 1; + bool pass = 2; // set true for Pass + } + bool hash = 3; + bool append = 4; + string append_prefix = 5; +} + +message Sandbox { + repeated Tool tools = 1; + repeated Dep deps = 2; + map env = 3; +} diff --git a/proto/plugin/v1/driver.proto b/proto/plugin/v1/driver.proto new file mode 100644 index 00000000..61a01045 --- /dev/null +++ b/proto/plugin/v1/driver.proto @@ -0,0 +1,58 @@ +syntax = "proto3"; + +// Driver-exported methods. Mirrors the hplugin::Driver trait. +// NOTE: schema() (DriverSchema) is not yet modeled; added when needed. +package heph.plugin.v1; + +import "plugin/v1/common.proto"; +import "plugin/v1/targetdef.proto"; +import "plugin/v1/callback.proto"; + +message ParseRequest { + string request_id = 1; + TargetSpec target_spec = 2; +} +message ParseResponse { + TargetDef target_def = 1; +} + +message ApplyTransitiveRequest { + string request_id = 1; + TargetDef target_def = 2; + Sandbox sandbox = 3; +} +message ApplyTransitiveResponse { + TargetDef target_def = 1; +} + +// Run inputs are materialized dep artifacts — also abstract, so they cross as +// ArtifactHandles; the host materializes them into the sandbox dir. +message RunInput { + ArtifactHandle artifact = 1; + string origin_id = 2; + Addr source_addr = 3; + repeated string filters = 4; + map annotations = 5; +} + +message RunRequest { + string request_id = 1; + TargetDef target = 2; + string tree_root_path = 3; + repeated RunInput inputs = 4; + string hashin = 5; + string sandbox_dir = 6; + // stdio crosses as inherited fds at spawn, not through the RPC channel; + // these flags only say whether the host wired each stream. + bool has_stdin = 7; + bool has_stdout = 8; + bool has_stderr = 9; + // true => run_shell, false => run + bool shell = 10; +} + +// fuse_slot_guards and sandbox_cleanup are non-transferable host RAII, +// constructed host-side after the remote run returns. +message RunResponse { + repeated OutputArtifactRef artifacts = 1; +} diff --git a/proto/plugin/v1/envelope.proto b/proto/plugin/v1/envelope.proto new file mode 100644 index 00000000..64fce724 --- /dev/null +++ b/proto/plugin/v1/envelope.proto @@ -0,0 +1,117 @@ +syntax = "proto3"; + +// Transport framing + handshake. One mux loop per connection: every Frame +// carries a correlation id; bodies are either host->plugin calls, plugin->host +// callbacks, or control. Used directly by the proto transport; the shm/wasm +// transports reuse the same logical bodies. +package heph.plugin.v1; + +import "plugin/v1/provider.proto"; +import "plugin/v1/driver.proto"; +import "plugin/v1/callback.proto"; + +enum Transport { + TRANSPORT_UNSPECIFIED = 0; + TRANSPORT_PROTO = 1; + TRANSPORT_SHM = 2; + TRANSPORT_WASM = 3; +} + +enum PluginKind { + PLUGIN_KIND_UNSPECIFIED = 0; + PLUGIN_KIND_PROVIDER = 1; + PLUGIN_KIND_DRIVER = 2; + PLUGIN_KIND_BOTH = 3; +} + +enum PayloadEncoding { + PAYLOAD_ENCODING_UNSPECIFIED = 0; + PAYLOAD_ENCODING_PROST = 1; + PAYLOAD_ENCODING_RKYV = 2; // Rust zero-copy fast path (shm) + PAYLOAD_ENCODING_CAPNP = 3; // polyglot zero-copy (shm, e.g. Go) +} + +// Plugin -> host on connect. +message Handshake { + string abi_semver = 1; + Transport transport = 2; + string plugin_name = 3; + PluginKind kind = 4; + uint64 capabilities = 5; + PayloadEncoding payload_encoding = 6; +} + +// Host -> plugin reply. +message Hello { + string abi_semver = 1; + uint64 host_capabilities = 2; +} + +message Cancel { + uint64 request_id = 1; +} + +message Error { + enum Kind { + KIND_UNSPECIFIED = 0; + KIND_OTHER = 1; + KIND_CANCELLED = 2; + KIND_NOT_FOUND = 3; + } + Kind kind = 1; + string message = 2; +} + +// One element of a streamed response (e.g. an encoded ListResponse, or a chunk +// of artifact bytes from OpenArtifact), correlated by the frame id. +message StreamItem { + bytes item = 1; +} +message StreamEnd { + optional Error error = 1; +} + +message Frame { + uint64 id = 1; + oneof body { + // control + Handshake handshake = 2; + Hello hello = 3; + Cancel cancel = 4; + Error error = 5; + StreamItem stream_item = 6; + StreamEnd stream_end = 7; + + // plugin-exported calls (host->plugin request / plugin->host response) + ConfigRequest config_req = 16; + ConfigResponse config_resp = 17; + ListRequest list_req = 18; // results stream via StreamItem(ListResponse) + ListPackagesRequest list_packages_req = 19; // results stream via StreamItem + GetRequest get_req = 20; + GetResponse get_resp = 21; + GetError get_err = 22; + ProbeRequest probe_req = 23; + ProbeResponse probe_resp = 24; + ParseRequest parse_req = 25; + ParseResponse parse_resp = 26; + ApplyTransitiveRequest apply_transitive_req = 27; + ApplyTransitiveResponse apply_transitive_resp = 28; + RunRequest run_req = 29; + RunResponse run_resp = 30; + + // host-exported callbacks (plugin->host request / host->plugin response) + ResultRequest result_req = 40; + ResultResponse result_resp = 41; + OpenArtifactRequest open_artifact_req = 42; + NoteDepRequest note_dep_req = 43; + NoteDepResponse note_dep_resp = 44; + QueryRequest query_req = 45; + QueryResponse query_resp = 46; + WalkRequest walk_req = 47; + WalkResponse walk_resp = 48; + ConfigGetRequest config_get_req = 49; + ConfigGetResponse config_get_resp = 50; + ReleaseLeaseRequest release_lease_req = 51; + ReleaseLeaseResponse release_lease_resp = 52; + } +} diff --git a/proto/plugin/v1/provider.proto b/proto/plugin/v1/provider.proto new file mode 100644 index 00000000..e0ce0f61 --- /dev/null +++ b/proto/plugin/v1/provider.proto @@ -0,0 +1,60 @@ +syntax = "proto3"; + +// Provider-exported methods. Mirrors the hplugin::Provider trait. +// NOTE: functions()/state_schema() (BUILD-file Starlark integration) are not +// yet modeled here; they are not on the remote data path and will be added +// when the remote provider needs to surface provider functions. +package heph.plugin.v1; + +import "plugin/v1/common.proto"; +import "plugin/v1/targetdef.proto"; + +message ConfigRequest {} +message ConfigResponse { + string name = 1; +} + +message ListRequest { + string request_id = 1; + string package = 2; + repeated State states = 3; +} +message ListResponse { + Addr addr = 1; +} + +message ListPackagesRequest { + string prefix = 1; +} +message ListPackageResponse { + string pkg = 1; +} + +// executor is NOT serialized: callbacks travel on the host-callback channel +// (callback.proto), correlated to this request by request_id. +message GetRequest { + string request_id = 1; + Addr addr = 2; + repeated State states = 3; +} +message GetResponse { + TargetSpec target_spec = 1; +} +// Mirrors provider::GetError { NotFound, Other(_) }. +message GetError { + enum Kind { + KIND_UNSPECIFIED = 0; + KIND_NOT_FOUND = 1; + KIND_OTHER = 2; + } + Kind kind = 1; + string message = 2; +} + +message ProbeRequest { + string request_id = 1; + string package = 2; +} +message ProbeResponse { + repeated State states = 1; +} diff --git a/proto/plugin/v1/targetdef.proto b/proto/plugin/v1/targetdef.proto new file mode 100644 index 00000000..65c34029 --- /dev/null +++ b/proto/plugin/v1/targetdef.proto @@ -0,0 +1,125 @@ +syntax = "proto3"; + +// Target spec/def wire types. Mirrors crates/plugin/src/driver.rs. +package heph.plugin.v1; + +import "plugin/v1/common.proto"; + +// Provider output. Mirrors driver::TargetSpec. +message TargetSpec { + Addr addr = 1; + string driver = 2; + map config = 3; + repeated string labels = 4; + Sandbox transitive = 5; +} + +// Opaque driver-specific config blob. The host never inspects this; it is +// produced and consumed only by the originating driver (which both parse()s +// and run()s the target). Mirrors the erased TargetDef::raw_def. +message RawDefBlob { + enum Format { + FORMAT_UNSPECIFIED = 0; + FORMAT_JSON = 1; + FORMAT_MSGPACK = 2; + } + string driver = 1; + Format format = 2; + bytes data = 3; +} + +enum InputMode { + INPUT_MODE_UNSPECIFIED = 0; + INPUT_MODE_STANDARD = 1; + INPUT_MODE_LINK = 2; + INPUT_MODE_TOOL = 3; +} + +message Input { + TargetAddr ref = 1; + InputMode mode = 2; + string origin_id = 3; + map annotations = 4; + bool hashed = 5; + bool runtime = 6; +} + +enum CodegenMode { + CODEGEN_MODE_UNSPECIFIED = 0; + CODEGEN_MODE_NONE = 1; + CODEGEN_MODE_COPY = 2; + CODEGEN_MODE_IN_PLACE = 3; +} + +// Mirrors driver::path::Path + path::Content. +message Path { + oneof content { + string file_path = 1; + string dir_path = 2; + string glob = 3; + } + CodegenMode codegen_tree = 4; + bool collect = 5; +} + +message Output { + string group = 1; + repeated Path paths = 2; +} + +message CacheConfig { + bool enabled = 1; + bool remote_enabled = 2; + uint32 history = 3; +} + +// Driver parsed form. Mirrors driver::targetdef::TargetDef. +message TargetDef { + Addr addr = 1; + repeated string labels = 2; + RawDefBlob raw_def = 3; + repeated Input inputs = 4; + repeated Output outputs = 5; + repeated Path support_files = 6; + CacheConfig cache = 7; + bool pty = 8; + bytes hash = 9; + bool transparent = 10; +} + +// ---- Run outputs (driver -> host). Mirrors outputartifact::OutputArtifact. +// These are fresh files the plugin just wrote into its sandbox dir, so they +// cross as paths (or small inline Raw bytes). Distinct from result()/input +// artifacts, which are abstract readers passed as ArtifactHandle (callback.proto). + +enum ArtifactType { + ARTIFACT_TYPE_UNSPECIFIED = 0; + ARTIFACT_TYPE_OUTPUT = 1; + ARTIFACT_TYPE_LOG = 2; + ARTIFACT_TYPE_SUPPORT_FILE = 3; +} + +message ContentFile { + string source_path = 1; + string out_path = 2; + bool x = 3; +} + +message ContentRaw { + bytes data = 1; + string path = 2; + bool x = 3; +} + +message OutputArtifactRef { + string group = 1; + string name = 2; + ArtifactType type = 3; + oneof content { + ContentFile file = 4; + ContentRaw raw = 5; + string tar_path = 6; + string cpio_path = 7; + } + string hashout = 8; +} diff --git a/wit/heph-plugin.wit b/wit/heph-plugin.wit new file mode 100644 index 00000000..6bc72149 --- /dev/null +++ b/wit/heph-plugin.wit @@ -0,0 +1,166 @@ +// WIT mirror of the heph external-plugin ABI, for the wasm (component-model) +// transport. The proto schema in proto/plugin/v1/*.proto is the source of +// truth; this file mirrors the same logical interface for wasm guests. +// +// NOTE: this is consumed by the wasm tier (milestone M4). It is authored now so +// all three transports share one contract; wit-bindgen validation + a +// proto<->wit parity check land in M4. Keep field sets in sync with the proto. + +package heph:plugin@0.1.0; + +interface types { + // hmodel::Addr + record addr { + package: string, + name: string, + args: list>, + } + + // hcore::htvalue::Value (recursive) + variant value { + string-val(string), + bool-val(bool), + float-val(f64), + int-val(s64), + uint-val(u64), + null-val, + map-val(list>), + list-val(list), + } + + record target-addr { + %ref: addr, + output: option, + filters: list, + } + + record state { + package: string, + provider: string, + state: list>, + } + + enum dep-mode { none, link } + + record tool { %ref: target-addr, group: string, hash: bool, id: string } + record dep { + %ref: target-addr, mode: dep-mode, group: string, + runtime: bool, hash: bool, id: string, + } + variant env-value { literal(string), %pass } + record env { value: env-value, hash: bool, append: bool, append-prefix: string } + record sandbox { + tools: list, + deps: list, + env: list>, + } + + record target-spec { + addr: addr, + driver: string, + config: list>, + labels: list, + transitive: sandbox, + } + + enum raw-def-format { json, msgpack } + // opaque driver-specific blob; host never inspects it + record raw-def-blob { driver: string, format: raw-def-format, data: list } + + enum input-mode { standard, link, tool } + record input { + %ref: target-addr, mode: input-mode, origin-id: string, + annotations: list>, hashed: bool, runtime: bool, + } + + enum codegen-mode { none, copy, in-place } + variant path-content { file-path(string), dir-path(string), glob(string) } + record path { content: path-content, codegen-tree: codegen-mode, collect: bool } + record output { group: string, paths: list } + record cache-config { enabled: bool, remote-enabled: bool, history: u32 } + + record target-def { + addr: addr, + labels: list, + raw-def: raw-def-blob, + inputs: list, + outputs: list, + support-files: list, + cache: cache-config, + pty: bool, + hash: list, + transparent: bool, + } + + // result/input artifacts cross as opaque handles (abstract readers) + record artifact-handle { + handle-id: string, group: string, name: string, + hashout: string, byte-size: u64, support: bool, + } + + // driver run outputs (fresh files the plugin wrote) + enum artifact-type { output, log, support-file } + record content-file { source-path: string, out-path: string, x: bool } + record content-raw { data: list, path: string, x: bool } + variant artifact-content { + file(content-file), raw(content-raw), tar-path(string), cpio-path(string), + } + record output-artifact-ref { + group: string, name: string, %type: artifact-type, + content: artifact-content, hashout: string, + } +} + +// Host-exported callback surface (the plugin imports this). +interface host { + use types.{addr, value, artifact-handle}; + + record result-ref { lease-id: string, artifacts: list } + + // executor.result: registers the dep edge host-side, returns handles+lease. + result: func(request-id: string, addr: addr) -> result; + // lazy artifact bytes (abstract Content has no path). + open-artifact: func(lease-id: string, handle-id: string, offset: u64) -> result, string>; + // cache-hit fast path: register edge only; err carries cycle message. + note-dep: func(request-id: string, parent: addr, %addr: addr) -> result<_, string>; + query: func(request-id: string, matcher-label: string, extra-skip: list) -> result, string>; + walk: func(dir: string) -> result, string>; + config-get: func(key: string) -> option; + release-lease: func(lease-id: string); + // cancellation: wasm can't be preempted, so the guest polls this. + cancelled: func(request-id: string) -> bool; +} + +// Plugin-exported provider methods (the guest exports this). +interface provider { + use types.{addr, target-spec, state}; + + config: func() -> string; // provider name + list: func(request-id: string, package: string, states: list) -> result, string>; + list-packages: func(prefix: string) -> result, string>; + get: func(request-id: string, addr: addr, states: list) -> result, string>; + probe: func(request-id: string, package: string) -> result, string>; +} + +// Plugin-exported driver methods (the guest exports this). +interface driver { + use types.{target-spec, target-def, sandbox, output-artifact-ref, artifact-handle, addr}; + + record run-input { + artifact: artifact-handle, origin-id: string, source-addr: addr, + filters: list, annotations: list>, + } + + config: func() -> string; + parse: func(request-id: string, spec: target-spec) -> result; + apply-transitive: func(request-id: string, def: target-def, sandbox: sandbox) -> result; + run: func(request-id: string, def: target-def, tree-root: string, inputs: list, + hashin: string, sandbox-dir: string, shell: bool) -> result, string>; +} + +// A plugin imports the host callbacks and exports provider and/or driver. +world plugin { + import host; + export provider; + export driver; +} From 05b604801e3076ea43082e4d30967733d7c0e3df Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 10:55:51 +0200 Subject: [PATCH 02/44] feat(plugin-abi): pb<->engine conversions (convert feature) [M1] Free-function conversions (orphan rule forbids From impls) between the prost wire types and hplugin/hmodel/hcore: Addr, Value, State, TargetAddr, Sandbox (Tool/Dep/Env), TargetSpec, Matcher. Provider-path scope; driver-path (TargetDef/raw_def/run) follows in M2. Round-trip unit tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-abi/Cargo.toml | 8 + crates/plugin-abi/src/convert.rs | 357 +++++++++++++++++++++++++++++++ crates/plugin-abi/src/lib.rs | 3 + 3 files changed, 368 insertions(+) create mode 100644 crates/plugin-abi/src/convert.rs diff --git a/crates/plugin-abi/Cargo.toml b/crates/plugin-abi/Cargo.toml index a5db6d17..7132c7bb 100644 --- a/crates/plugin-abi/Cargo.toml +++ b/crates/plugin-abi/Cargo.toml @@ -10,6 +10,14 @@ workspace = true hproto-gen = { package = "proto-gen", path = "../../gen/proto", features = ["proto_full"] } anyhow = "1.0.102" rkyv = "0.8" +# convert feature: conversions between pb wire types and the in-process +# hplugin/hmodel/hcore types. Optional so the bare wire crate stays light. +hplugin = { package = "plugin", path = "../plugin", optional = true } +hmodel = { package = "model", path = "../model", optional = true } +hcore = { package = "core", path = "../core", optional = true } + +[features] +convert = ["dep:hplugin", "dep:hmodel", "dep:hcore"] [dev-dependencies] rkyv = "0.8" diff --git a/crates/plugin-abi/src/convert.rs b/crates/plugin-abi/src/convert.rs new file mode 100644 index 00000000..64aefef7 --- /dev/null +++ b/crates/plugin-abi/src/convert.rs @@ -0,0 +1,357 @@ +//! Conversions between the prost wire types ([`crate::pb`]) and the in-process +//! `hplugin`/`hmodel`/`hcore` types. Free functions (not `From` impls) because +//! both sides are foreign to this crate (orphan rule). +//! +//! Provider-path scope for now (Addr/Value/State/Sandbox/TargetSpec/Matcher); +//! driver-path conversions (TargetDef/raw_def/Input/Output/run) are added when +//! the remote driver path needs them (M2). + +use crate::pb; +use hcore::htvalue::Value; +use hmodel::htaddr::Addr; +use hmodel::htmatcher::Matcher; +use hmodel::htpkg::PkgBuf; +use hplugin::driver::sandbox::{Dep, Env, EnvValue, Mode, Sandbox, Tool}; +use hplugin::driver::TargetAddr; +use hplugin::provider::{State, TargetSpec}; +use std::collections::BTreeMap; + +// ---- Addr ---- + +pub fn addr_to_pb(a: &Addr) -> pb::Addr { + pb::Addr { + package: a.package.as_str().to_string(), + name: a.name.clone(), + args: a.args.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + } +} + +pub fn addr_from_pb(a: pb::Addr) -> Addr { + let args: BTreeMap = a.args.into_iter().collect(); + Addr::new(PkgBuf::from(a.package), a.name, args) +} + +// ---- Value ---- + +pub fn value_to_pb(v: &Value) -> pb::Value { + use pb::value::{Kind, List, Map, Null}; + let kind = match v { + Value::String(s) => Kind::StringVal(s.clone()), + Value::Bool(b) => Kind::BoolVal(*b), + Value::Float(f) => Kind::FloatVal(*f), + Value::Int(i) => Kind::IntVal(*i), + Value::Uint(u) => Kind::UintVal(*u), + Value::Null() => Kind::NullVal(Null {}), + Value::Map(m) => Kind::MapVal(Map { + entries: m.iter().map(|(k, v)| (k.clone(), value_to_pb(v))).collect(), + }), + Value::List(l) => Kind::ListVal(List { + items: l.iter().map(value_to_pb).collect(), + }), + }; + pb::Value { kind: Some(kind) } +} + +pub fn value_from_pb(v: pb::Value) -> Value { + use pb::value::Kind; + match v.kind { + Some(Kind::StringVal(s)) => Value::String(s), + Some(Kind::BoolVal(b)) => Value::Bool(b), + Some(Kind::FloatVal(f)) => Value::Float(f), + Some(Kind::IntVal(i)) => Value::Int(i), + Some(Kind::UintVal(u)) => Value::Uint(u), + Some(Kind::NullVal(_)) | None => Value::Null(), + Some(Kind::MapVal(m)) => Value::Map( + m.entries + .into_iter() + .map(|(k, v)| (k, value_from_pb(v))) + .collect(), + ), + Some(Kind::ListVal(l)) => Value::List(l.items.into_iter().map(value_from_pb).collect()), + } +} + +// ---- State ---- + +pub fn state_to_pb(s: &State) -> pb::State { + pb::State { + package: s.package.as_str().to_string(), + provider: s.provider.clone(), + state: s + .state + .iter() + .map(|(k, v)| (k.clone(), value_to_pb(v))) + .collect(), + } +} + +pub fn state_from_pb(s: pb::State) -> State { + State { + package: PkgBuf::from(s.package), + provider: s.provider, + state: s + .state + .into_iter() + .map(|(k, v)| (k, value_from_pb(v))) + .collect(), + } +} + +// ---- TargetAddr ---- + +pub fn target_addr_to_pb(t: &TargetAddr) -> pb::TargetAddr { + pb::TargetAddr { + r#ref: Some(addr_to_pb(&t.r#ref)), + output: t.output.clone(), + filters: t.filters.clone(), + } +} + +pub fn target_addr_from_pb(t: pb::TargetAddr) -> TargetAddr { + TargetAddr { + r#ref: addr_from_pb(t.r#ref.unwrap_or_default()), + output: t.output, + filters: t.filters, + } +} + +// ---- Sandbox ---- + +fn tool_to_pb(t: &Tool) -> pb::Tool { + pb::Tool { + r#ref: Some(target_addr_to_pb(&t.r#ref)), + group: t.group.clone(), + hash: t.hash, + id: t.id.clone(), + } +} + +fn tool_from_pb(t: pb::Tool) -> Tool { + Tool { + r#ref: target_addr_from_pb(t.r#ref.unwrap_or_default()), + group: t.group, + hash: t.hash, + id: t.id, + } +} + +fn dep_to_pb(d: &Dep) -> pb::Dep { + let mode = match d.mode { + Mode::None => pb::DepMode::None, + Mode::Link => pb::DepMode::Link, + }; + pb::Dep { + r#ref: Some(target_addr_to_pb(&d.r#ref)), + mode: mode as i32, + group: d.group.clone(), + runtime: d.runtime, + hash: d.hash, + id: d.id.clone(), + } +} + +fn dep_from_pb(d: pb::Dep) -> Dep { + let mode = match pb::DepMode::try_from(d.mode).unwrap_or(pb::DepMode::None) { + pb::DepMode::Link => Mode::Link, + _ => Mode::None, + }; + Dep { + r#ref: target_addr_from_pb(d.r#ref.unwrap_or_default()), + mode, + group: d.group, + runtime: d.runtime, + hash: d.hash, + id: d.id, + } +} + +fn env_to_pb(e: &Env) -> pb::Env { + let value = match &e.value { + EnvValue::Literal(s) => pb::env::Value::Literal(s.clone()), + EnvValue::Pass => pb::env::Value::Pass(true), + }; + pb::Env { + value: Some(value), + hash: e.hash, + append: e.append, + append_prefix: e.append_prefix.clone(), + } +} + +fn env_from_pb(e: pb::Env) -> Env { + let value = match e.value { + Some(pb::env::Value::Literal(s)) => EnvValue::Literal(s), + Some(pb::env::Value::Pass(_)) => EnvValue::Pass, + None => EnvValue::Literal(String::new()), + }; + Env { + value, + hash: e.hash, + append: e.append, + append_prefix: e.append_prefix, + } +} + +pub fn sandbox_to_pb(s: &Sandbox) -> pb::Sandbox { + pb::Sandbox { + tools: s.tools.iter().map(tool_to_pb).collect(), + deps: s.deps.iter().map(dep_to_pb).collect(), + env: s.env.iter().map(|(k, v)| (k.clone(), env_to_pb(v))).collect(), + } +} + +pub fn sandbox_from_pb(s: pb::Sandbox) -> Sandbox { + // tool_keys/dep_keys are rebuilt by push_tool/push_dep (private dedup sets). + let mut sb = Sandbox::default(); + for t in s.tools { + sb.push_tool(tool_from_pb(t)); + } + for d in s.deps { + sb.push_dep(dep_from_pb(d)); + } + sb.env = s + .env + .into_iter() + .map(|(k, v)| (k, env_from_pb(v))) + .collect(); + sb +} + +// ---- TargetSpec ---- + +pub fn target_spec_to_pb(t: &TargetSpec) -> pb::TargetSpec { + pb::TargetSpec { + addr: Some(addr_to_pb(&t.addr)), + driver: t.driver.clone(), + config: t + .config + .iter() + .map(|(k, v)| (k.clone(), value_to_pb(v))) + .collect(), + labels: t.labels.clone(), + transitive: Some(sandbox_to_pb(&t.transitive)), + } +} + +pub fn target_spec_from_pb(t: pb::TargetSpec) -> TargetSpec { + TargetSpec { + addr: addr_from_pb(t.addr.unwrap_or_default()), + driver: t.driver, + config: t + .config + .into_iter() + .map(|(k, v)| (k, value_from_pb(v))) + .collect(), + labels: t.labels, + transitive: sandbox_from_pb(t.transitive.unwrap_or_default()), + } +} + +// ---- Matcher ---- + +pub fn matcher_to_pb(m: &Matcher) -> pb::Matcher { + use pb::matcher::{Kind, List}; + let kind = match m { + Matcher::Addr(a) => Kind::Addr(addr_to_pb(a)), + Matcher::Label(l) => Kind::Label(l.clone()), + Matcher::Package(p) => Kind::Package(p.as_str().to_string()), + Matcher::PackagePrefix(p) => Kind::PackagePrefix(p.as_str().to_string()), + Matcher::TreeOutputTo(p) => Kind::TreeOutputTo(p.as_str().to_string()), + Matcher::Or(ms) => Kind::Or(List { + matchers: ms.iter().map(matcher_to_pb).collect(), + }), + Matcher::And(ms) => Kind::And(List { + matchers: ms.iter().map(matcher_to_pb).collect(), + }), + Matcher::Not(inner) => Kind::Not(Box::new(matcher_to_pb(inner))), + }; + pb::Matcher { kind: Some(kind) } +} + +pub fn matcher_from_pb(m: pb::Matcher) -> Matcher { + use pb::matcher::Kind; + match m.kind { + Some(Kind::Addr(a)) => Matcher::Addr(addr_from_pb(a)), + Some(Kind::Label(l)) => Matcher::Label(l), + Some(Kind::Package(p)) => Matcher::Package(PkgBuf::from(p)), + Some(Kind::PackagePrefix(p)) => Matcher::PackagePrefix(PkgBuf::from(p)), + Some(Kind::TreeOutputTo(p)) => Matcher::TreeOutputTo(PkgBuf::from(p)), + Some(Kind::Or(l)) => Matcher::Or(l.matchers.into_iter().map(matcher_from_pb).collect()), + Some(Kind::And(l)) => Matcher::And(l.matchers.into_iter().map(matcher_from_pb).collect()), + Some(Kind::Not(inner)) => Matcher::Not(Box::new(matcher_from_pb(*inner))), + // An empty matcher matches nothing sensible; default to an empty Or. + None => Matcher::Or(vec![]), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + fn addr(pkg: &str, name: &str) -> Addr { + let mut args = BTreeMap::new(); + args.insert("goos".to_string(), "linux".to_string()); + Addr::new(PkgBuf::from(pkg), name.to_string(), args) + } + + #[test] + fn addr_roundtrip() { + let a = addr("//foo/bar", "lib"); + assert_eq!(addr_from_pb(addr_to_pb(&a)), a); + } + + #[test] + fn value_roundtrip() { + let v = Value::Map(HashMap::from([ + ("s".to_string(), Value::String("x".to_string())), + ("i".to_string(), Value::Int(-3)), + ("u".to_string(), Value::Uint(7)), + ("f".to_string(), Value::Float(1.5)), + ("b".to_string(), Value::Bool(true)), + ("n".to_string(), Value::Null()), + ("l".to_string(), Value::List(vec![Value::Int(1), Value::Int(2)])), + ])); + assert_eq!(value_from_pb(value_to_pb(&v)), v); + } + + #[test] + fn target_spec_roundtrip() { + let mut spec = TargetSpec { + addr: addr("//a", "x"), + driver: "exec".to_string(), + config: HashMap::from([("cmd".to_string(), Value::String("echo".to_string()))]), + labels: vec!["lbl".to_string()], + transitive: Sandbox::default(), + }; + spec.transitive.push_dep(Dep { + r#ref: TargetAddr { + r#ref: addr("//b", "y"), + output: Some("out".to_string()), + filters: vec![], + }, + mode: Mode::Link, + group: "g".to_string(), + runtime: true, + hash: true, + id: "id1".to_string(), + }); + let back = target_spec_from_pb(target_spec_to_pb(&spec)); + assert_eq!(back.addr, spec.addr); + assert_eq!(back.driver, spec.driver); + assert_eq!(back.config, spec.config); + assert_eq!(back.labels, spec.labels); + assert_eq!(back.transitive.deps.len(), 1); + assert_eq!(back.transitive.deps[0].id, "id1"); + assert!(matches!(back.transitive.deps[0].mode, Mode::Link)); + } + + #[test] + fn matcher_roundtrip() { + let m = Matcher::And(vec![ + Matcher::Package(PkgBuf::from("//a")), + Matcher::Not(Box::new(Matcher::Label("x".to_string()))), + ]); + assert_eq!(matcher_from_pb(matcher_to_pb(&m)), m); + } +} diff --git a/crates/plugin-abi/src/lib.rs b/crates/plugin-abi/src/lib.rs index 2c0897bf..240f0368 100644 --- a/crates/plugin-abi/src/lib.rs +++ b/crates/plugin-abi/src/lib.rs @@ -21,3 +21,6 @@ pub use hproto_gen::heph::plugin::v1 as pb; pub const ABI_SEMVER: &str = "0.1.0"; pub mod shm_types; + +#[cfg(feature = "convert")] +pub mod convert; From db8fd3b0db64f19402311600ab45c91ce4c7640c Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 11:10:22 +0200 Subject: [PATCH 03/44] feat(plugin-remote): proto transport + host service + guest serve [M1] Working bidirectional, request-id-multiplexed plugin transport, proven end-to-end in-process over a UDS socketpair. - plugin-abi (transport feature): length-prefixed Frame framing + a symmetric Mux (outbound calls incl. streaming + cancellable; inbound requests dispatched to an InboundHandler). Two id spaces never collide since each side only matches response-typed frames to its own ids. - plugin-remote: RemoteProvider impls hplugin::Provider over the mux; host callback service serves result/note_dep/query/open_artifact/ release_lease against the per-request engine executor; lease table holds artifact read-guards. - plugin-sdk: serve() drives an author Provider from inbound frames and exposes host callbacks as a ProviderExecutor (MuxExecutor), so plugin code calls executor.result() exactly as in-process. - e2e test: config, streaming list, get + result() callback round-trip, cancellation. All green. Artifact byte streaming (RemoteContent reader/walk) + lease-on-drop are stubbed for M2; note_dep falls back to result() until the engine edge-only API lands. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 29 +++ Cargo.toml | 2 +- crates/plugin-abi/Cargo.toml | 7 + crates/plugin-abi/src/frame.rs | 41 ++++ crates/plugin-abi/src/lib.rs | 5 + crates/plugin-abi/src/mux.rs | 211 ++++++++++++++++ crates/plugin-remote/Cargo.toml | 29 +++ crates/plugin-remote/src/host.rs | 189 ++++++++++++++ crates/plugin-remote/src/lease.rs | 40 +++ crates/plugin-remote/src/lib.rs | 19 ++ crates/plugin-remote/src/provider.rs | 182 ++++++++++++++ crates/plugin-remote/tests/proto_e2e.rs | 243 ++++++++++++++++++ crates/plugin-sdk/Cargo.toml | 6 +- crates/plugin-sdk/src/lib.rs | 2 + crates/plugin-sdk/src/serve.rs | 314 ++++++++++++++++++++++++ 15 files changed, 1317 insertions(+), 2 deletions(-) create mode 100644 crates/plugin-abi/src/frame.rs create mode 100644 crates/plugin-abi/src/mux.rs create mode 100644 crates/plugin-remote/Cargo.toml create mode 100644 crates/plugin-remote/src/host.rs create mode 100644 crates/plugin-remote/src/lease.rs create mode 100644 crates/plugin-remote/src/lib.rs create mode 100644 crates/plugin-remote/src/provider.rs create mode 100644 crates/plugin-remote/tests/proto_e2e.rs create mode 100644 crates/plugin-sdk/src/serve.rs diff --git a/Cargo.lock b/Cargo.lock index 851f2520..58806111 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3539,8 +3539,15 @@ name = "plugin-abi" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", + "core", + "model", + "plugin", + "prost 0.14.4", "proto-gen", "rkyv", + "tokio", + "tracing", ] [[package]] @@ -3656,6 +3663,24 @@ dependencies = [ "plugin", ] +[[package]] +name = "plugin-remote" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "core", + "futures", + "model", + "plugin", + "plugin-abi", + "plugin-sdk", + "prost 0.14.4", + "tokio", + "tracing", + "walk", +] + [[package]] name = "plugin-sdk" version = "0.1.0" @@ -3663,9 +3688,13 @@ dependencies = [ "anyhow", "async-trait", "core", + "futures", "model", "plugin", "plugin-abi", + "prost 0.14.4", + "tokio", + "tracing", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c1ea8572..ef3d7aff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2024" [workspace] -members = ["gen/proto", "crates/e2e", "crates/testkit", "crates/plugingo-e2e", "crates/htspec-derive", "crates/core", "crates/walk", "crates/proc", "crates/model", "crates/sandboxfuse", "crates/plugin", "crates/plugin-abi", "crates/plugin-sdk", "crates/builtins", "crates/plugin-buildfile", "crates/driver-support", "crates/plugin-exec", "crates/plugin-nix", "crates/plugin-query", "crates/plugin-go", "crates/telemetry", "crates/tui", "crates/lock", "crates/engine"] +members = ["gen/proto", "crates/e2e", "crates/testkit", "crates/plugingo-e2e", "crates/htspec-derive", "crates/core", "crates/walk", "crates/proc", "crates/model", "crates/sandboxfuse", "crates/plugin", "crates/plugin-abi", "crates/plugin-sdk", "crates/plugin-remote", "crates/builtins", "crates/plugin-buildfile", "crates/driver-support", "crates/plugin-exec", "crates/plugin-nix", "crates/plugin-query", "crates/plugin-go", "crates/telemetry", "crates/tui", "crates/lock", "crates/engine"] [profile.profiling] inherits = "release" diff --git a/crates/plugin-abi/Cargo.toml b/crates/plugin-abi/Cargo.toml index 7132c7bb..54fbba7f 100644 --- a/crates/plugin-abi/Cargo.toml +++ b/crates/plugin-abi/Cargo.toml @@ -15,9 +15,16 @@ rkyv = "0.8" hplugin = { package = "plugin", path = "../plugin", optional = true } hmodel = { package = "model", path = "../model", optional = true } hcore = { package = "core", path = "../core", optional = true } +# transport feature: generic length-prefixed Frame framing + bidirectional mux +# over any AsyncRead/AsyncWrite. Shared by the host adapter and the guest SDK. +prost = { version = "0.14", optional = true } +tokio = { version = "1.52", features = ["io-util", "rt", "sync", "macros"], optional = true } +async-trait = { version = "0.1.89", optional = true } +tracing = { version = "0.1", optional = true } [features] convert = ["dep:hplugin", "dep:hmodel", "dep:hcore"] +transport = ["dep:prost", "dep:tokio", "dep:async-trait", "dep:tracing"] [dev-dependencies] rkyv = "0.8" diff --git a/crates/plugin-abi/src/frame.rs b/crates/plugin-abi/src/frame.rs new file mode 100644 index 00000000..cc0ebd99 --- /dev/null +++ b/crates/plugin-abi/src/frame.rs @@ -0,0 +1,41 @@ +//! Length-prefixed `Frame` framing over any byte stream. +//! +//! Wire format: a 4-byte little-endian length prefix followed by the +//! prost-encoded [`crate::pb::Frame`]. Used by the proto transport directly; +//! the shm/wasm transports carry the same `Frame` bodies by other means. + +use crate::pb; +use prost::Message; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +/// Max accepted frame size (16 MiB). Bulk artifact bytes stream as chunks, so +/// a single frame should never approach this; it guards against a corrupt +/// length prefix allocating unbounded memory. +const MAX_FRAME_LEN: u32 = 16 * 1024 * 1024; + +/// Encode and write one frame (length prefix + payload), then flush. +pub async fn write_frame(w: &mut W, f: &pb::Frame) -> anyhow::Result<()> { + let buf = f.encode_to_vec(); + let len = u32::try_from(buf.len()).map_err(|_e| anyhow::anyhow!("frame too large"))?; + w.write_u32_le(len).await?; + w.write_all(&buf).await?; + w.flush().await?; + Ok(()) +} + +/// Read one frame. Returns `Ok(None)` on a clean EOF at a frame boundary +/// (peer closed the connection), `Err` on a partial/corrupt frame. +pub async fn read_frame(r: &mut R) -> anyhow::Result> { + let len = match r.read_u32_le().await { + Ok(len) => len, + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None), + Err(e) => return Err(e.into()), + }; + if len > MAX_FRAME_LEN { + anyhow::bail!("frame length {len} exceeds max {MAX_FRAME_LEN}"); + } + let mut buf = vec![0u8; len as usize]; + r.read_exact(&mut buf).await?; + let frame = pb::Frame::decode(&buf[..])?; + Ok(Some(frame)) +} diff --git a/crates/plugin-abi/src/lib.rs b/crates/plugin-abi/src/lib.rs index 240f0368..c492f1cd 100644 --- a/crates/plugin-abi/src/lib.rs +++ b/crates/plugin-abi/src/lib.rs @@ -24,3 +24,8 @@ pub mod shm_types; #[cfg(feature = "convert")] pub mod convert; + +#[cfg(feature = "transport")] +pub mod frame; +#[cfg(feature = "transport")] +pub mod mux; diff --git a/crates/plugin-abi/src/mux.rs b/crates/plugin-abi/src/mux.rs new file mode 100644 index 00000000..035c660c --- /dev/null +++ b/crates/plugin-abi/src/mux.rs @@ -0,0 +1,211 @@ +//! Bidirectional, request-id-multiplexed frame mux over one connection. +//! +//! Symmetric: both ends use the same `Mux`. Each side issues ids for the +//! requests *it* initiates and routes inbound *response* frames to its own +//! pending table; inbound *request* frames are dispatched to the side's +//! [`InboundHandler`]. Because a side only ever matches response-typed frames +//! against ids it issued, the two id spaces never collide. +//! +//! On the host this carries plugin calls outbound + callbacks inbound; on the +//! guest, the reverse. + +use crate::frame::{read_frame, write_frame}; +use crate::pb; +use async_trait::async_trait; +use std::collections::HashMap; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::sync::{mpsc, oneshot}; + +pub use pb::frame::Body; + +/// Handles inbound *request* frames (the calls this side serves). The handler +/// sends its reply (and, for streamed methods, stream items + a stream-end) +/// back through `mux` tagged with the same `id`. +#[async_trait] +pub trait InboundHandler: Send + Sync + 'static { + async fn handle(&self, id: u64, body: Body, mux: Arc); +} + +enum Pending { + Unary(oneshot::Sender), + Stream(mpsc::UnboundedSender), +} + +/// True for frames that complete (or feed) a request this side issued. +fn is_response(body: &Body) -> bool { + matches!( + body, + Body::Hello(_) + | Body::Error(_) + | Body::StreamItem(_) + | Body::StreamEnd(_) + | Body::ConfigResp(_) + | Body::GetResp(_) + | Body::GetErr(_) + | Body::ProbeResp(_) + | Body::ParseResp(_) + | Body::ApplyTransitiveResp(_) + | Body::RunResp(_) + | Body::ResultResp(_) + | Body::NoteDepResp(_) + | Body::QueryResp(_) + | Body::WalkResp(_) + | Body::ConfigGetResp(_) + | Body::ReleaseLeaseResp(_) + ) +} + +fn is_stream_terminal(body: &Body) -> bool { + matches!(body, Body::StreamEnd(_) | Body::Error(_)) +} + +pub struct Mux { + out: mpsc::UnboundedSender, + next_id: AtomicU64, + pending: Mutex>, +} + +impl Mux { + /// Start the read and write tasks over the given half-streams, dispatching + /// inbound requests to `handler`. Returns a handle for issuing outbound + /// calls and sending responses. + pub fn start(read: R, write: W, handler: Arc) -> Arc + where + R: AsyncRead + Unpin + Send + 'static, + W: AsyncWrite + Unpin + Send + 'static, + { + let (out_tx, out_rx) = mpsc::unbounded_channel::(); + let mux = Arc::new(Mux { + out: out_tx, + next_id: AtomicU64::new(1), + pending: Mutex::new(HashMap::new()), + }); + + // writer task + tokio::spawn(writer_loop(write, out_rx)); + // reader task + tokio::spawn(reader_loop(read, Arc::clone(&mux), handler)); + + mux + } + + fn alloc_id(&self) -> u64 { + self.next_id.fetch_add(1, Ordering::Relaxed) + } + + /// Send a frame with an explicit id (used by handlers to reply). + pub fn send_body(&self, id: u64, body: Body) { + drop(self.out.send(pb::Frame { + id, + body: Some(body), + })); + } + + /// Issue a unary request and await its single response body. + pub async fn call(&self, body: Body) -> anyhow::Result { + self.call_cancellable(body, std::future::pending()).await + } + + /// Like [`Mux::call`], but races the request against `cancel`. On cancel, + /// drops the pending waiter and sends a `Cancel` frame for the request id so + /// the serving side can abort, then returns an error. `cancel` is a bare + /// future (e.g. `ctoken.cancelled()`) so the mux needs no cancellation dep. + pub async fn call_cancellable(&self, body: Body, cancel: F) -> anyhow::Result + where + F: std::future::Future, + { + let id = self.alloc_id(); + let (tx, rx) = oneshot::channel(); + self.pending.lock().expect("mux pending").insert(id, Pending::Unary(tx)); + self.send_body(id, body); + tokio::pin!(cancel); + tokio::select! { + resp = rx => { + let resp = resp + .map_err(|_e| anyhow::anyhow!("plugin connection closed before response"))?; + if let Body::Error(e) = resp { + anyhow::bail!("plugin error: {}", e.message); + } + Ok(resp) + } + () = &mut cancel => { + self.pending.lock().expect("mux pending").remove(&id); + self.send_body(id, Body::Cancel(pb::Cancel { request_id: id })); + anyhow::bail!("cancelled") + } + } + } + + /// Issue a streaming request. The returned receiver yields response bodies + /// (`StreamItem`s) until a `StreamEnd`/`Error` arrives (which is forwarded + /// as the final body, then the channel closes). + pub fn call_stream(&self, body: Body) -> mpsc::UnboundedReceiver { + let id = self.alloc_id(); + let (tx, rx) = mpsc::unbounded_channel(); + self.pending.lock().expect("mux pending").insert(id, Pending::Stream(tx)); + self.send_body(id, body); + rx + } + + fn route_response(&self, id: u64, body: Body) { + let mut pending = self.pending.lock().expect("mux pending"); + match pending.get(&id) { + Some(Pending::Unary(_)) => { + if let Some(Pending::Unary(tx)) = pending.remove(&id) { + drop(tx.send(body)); + } + } + Some(Pending::Stream(tx)) => { + let terminal = is_stream_terminal(&body); + drop(tx.send(body)); + if terminal { + pending.remove(&id); + } + } + None => { + tracing::warn!(id, "response for unknown request id"); + } + } + } +} + +async fn writer_loop(mut write: W, mut rx: mpsc::UnboundedReceiver) { + while let Some(frame) = rx.recv().await { + if let Err(e) = write_frame(&mut write, &frame).await { + tracing::warn!(error = %e, "frame write failed; closing writer"); + break; + } + } +} + +async fn reader_loop( + mut read: R, + mux: Arc, + handler: Arc, +) { + loop { + match read_frame(&mut read).await { + Ok(Some(frame)) => { + let id = frame.id; + let Some(body) = frame.body else { continue }; + if is_response(&body) { + mux.route_response(id, body); + } else { + let mux = Arc::clone(&mux); + let handler = Arc::clone(&handler); + tokio::spawn(async move { + handler.handle(id, body, mux).await; + }); + } + } + Ok(None) => break, // peer closed + Err(e) => { + tracing::warn!(error = %e, "frame read failed; closing reader"); + break; + } + } + } +} diff --git a/crates/plugin-remote/Cargo.toml b/crates/plugin-remote/Cargo.toml new file mode 100644 index 00000000..065f836f --- /dev/null +++ b/crates/plugin-remote/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "plugin-remote" +version = "0.1.0" +edition = "2024" + +[lints] +workspace = true + +[dependencies] +hplugin = { package = "plugin", path = "../plugin" } +hmodel = { package = "model", path = "../model" } +hcore = { package = "core", path = "../core" } +hwalk = { package = "walk", path = "../walk" } +plugin-abi = { path = "../plugin-abi", features = ["convert", "transport"] } +anyhow = "1.0.102" +async-trait = "0.1.89" +futures = "0.3.32" +prost = "0.14" +tokio = { version = "1.52", features = ["rt", "sync", "io-util", "macros", "net"] } +tracing = "0.1" + +[features] +# heavy transports, off by default; proto (UDS) is always available +shm = [] +wasm = [] + +[dev-dependencies] +plugin-sdk = { path = "../plugin-sdk" } +tokio = { version = "1.52", features = ["rt-multi-thread", "macros", "net", "time"] } diff --git a/crates/plugin-remote/src/host.rs b/crates/plugin-remote/src/host.rs new file mode 100644 index 00000000..ecc78d6c --- /dev/null +++ b/crates/plugin-remote/src/host.rs @@ -0,0 +1,189 @@ +//! Host-side callback service: serves the `AbiHost` surface (result / note_dep +//! / query / open_artifact / release_lease) that a remote plugin calls back +//! into while serving `get`/`parse`. Each call is dispatched against the +//! per-request scope registered by the host-side Provider/Driver for the +//! duration of that request. + +use crate::lease::LeaseTable; +use async_trait::async_trait; +use hcore::hartifactcontent::Content; +use hplugin::provider::ProviderExecutor; +use plugin_abi::convert; +use plugin_abi::mux::{Body, InboundHandler, Mux}; +use plugin_abi::pb; +use std::collections::HashMap; +use std::io::Read; +use std::sync::{Arc, Mutex}; + +/// The per-request callback target: the engine executor for that request. +pub(crate) struct Scope { + pub executor: Arc, +} + +#[derive(Default)] +pub(crate) struct HostInner { + scopes: Mutex>>, + pub leases: LeaseTable, +} + +impl HostInner { + pub fn register(&self, request_id: String, executor: Arc) { + self.scopes + .lock() + .expect("scopes") + .insert(request_id, Arc::new(Scope { executor })); + } + + pub fn unregister(&self, request_id: &str) { + self.scopes.lock().expect("scopes").remove(request_id); + } + + fn scope(&self, request_id: &str) -> Option> { + self.scopes.lock().expect("scopes").get(request_id).cloned() + } +} + +pub(crate) struct HostCallbackHandler { + pub inner: Arc, +} + +fn err_frame(message: String) -> Body { + Body::Error(pb::Error { + kind: pb::error::Kind::Other as i32, + message, + }) +} + +#[async_trait] +impl InboundHandler for HostCallbackHandler { + async fn handle(&self, id: u64, body: Body, mux: Arc) { + match body { + Body::ResultReq(req) => self.handle_result(id, req, &mux).await, + Body::NoteDepReq(req) => self.handle_note_dep(id, req, &mux).await, + Body::QueryReq(req) => self.handle_query(id, req, &mux).await, + Body::OpenArtifactReq(req) => self.handle_open_artifact(id, req, &mux).await, + Body::ReleaseLeaseReq(req) => { + self.inner.leases.release(&req.lease_id); + mux.send_body(id, Body::ReleaseLeaseResp(pb::ReleaseLeaseResponse {})); + } + Body::Cancel(_) => { /* host-as-server: cooperative cancel is M2 */ } + other => { + mux.send_body( + id, + err_frame(format!("unhandled inbound callback: {other:?}")), + ); + } + } + } +} + +impl HostCallbackHandler { + async fn handle_result(&self, id: u64, req: pb::ResultRequest, mux: &Arc) { + let Some(scope) = self.inner.scope(&req.request_id) else { + mux.send_body(id, err_frame(format!("unknown request scope {}", req.request_id))); + return; + }; + let addr = convert::addr_from_pb(req.addr.unwrap_or_default()); + match scope.executor.result(&addr).await { + Ok(eres) => { + let artifacts: Vec> = eres.artifacts.clone(); + let mut handles = Vec::with_capacity(artifacts.len()); + for (idx, art) in artifacts.iter().enumerate() { + let hashout = eres + .artifacts_meta + .get(idx) + .map(|m| m.hashout.clone()) + .or_else(|| art.hashout().ok()) + .unwrap_or_default(); + handles.push(pb::ArtifactHandle { + handle_id: idx.to_string(), + group: String::new(), + name: String::new(), + hashout, + byte_size: art.byte_size().unwrap_or(0), + support: false, + }); + } + let lease_id = self.inner.leases.insert(artifacts); + mux.send_body( + id, + Body::ResultResp(pb::ResultResponse { + lease_id, + artifacts: handles, + }), + ); + } + Err(e) => mux.send_body(id, err_frame(e.to_string())), + } + } + + async fn handle_note_dep(&self, id: u64, req: pb::NoteDepRequest, mux: &Arc) { + // M1: fall back to a full result() to register the dep edge (the engine + // registers parent->addr before any await). The true edge-only fast + // path is an engine API added in M2/M3. + let Some(scope) = self.inner.scope(&req.request_id) else { + mux.send_body(id, err_frame(format!("unknown request scope {}", req.request_id))); + return; + }; + let addr = convert::addr_from_pb(req.addr.unwrap_or_default()); + let resp = match scope.executor.result(&addr).await { + Ok(_) => pb::NoteDepResponse { + ok: true, + cycle: false, + message: String::new(), + }, + Err(e) => { + let msg = e.to_string(); + let cycle = msg.to_lowercase().contains("cycle"); + pb::NoteDepResponse { + ok: false, + cycle, + message: msg, + } + } + }; + mux.send_body(id, Body::NoteDepResp(resp)); + } + + async fn handle_query(&self, id: u64, req: pb::QueryRequest, mux: &Arc) { + let Some(scope) = self.inner.scope(&req.request_id) else { + mux.send_body(id, err_frame(format!("unknown request scope {}", req.request_id))); + return; + }; + let matcher = convert::matcher_from_pb(req.matcher.unwrap_or_default()); + match scope.executor.query(&matcher, &req.extra_skip).await { + Ok(addrs) => mux.send_body( + id, + Body::QueryResp(pb::QueryResponse { + addrs: addrs.iter().map(convert::addr_to_pb).collect(), + }), + ), + Err(e) => mux.send_body(id, err_frame(e.to_string())), + } + } + + async fn handle_open_artifact(&self, id: u64, req: pb::OpenArtifactRequest, mux: &Arc) { + let idx: usize = req.handle_id.parse().unwrap_or(usize::MAX); + let Some(content) = self.inner.leases.get(&req.lease_id, idx) else { + mux.send_body(id, err_frame(format!("unknown artifact {}#{}", req.lease_id, req.handle_id))); + return; + }; + // Content::reader is sync; read on a blocking thread. M1 reads the whole + // artifact into one chunk; offset/chunked streaming is M3. + let bytes = tokio::task::spawn_blocking(move || -> anyhow::Result> { + let mut r = content.reader()?; + let mut buf = Vec::new(); + r.read_to_end(&mut buf)?; + Ok(buf) + }) + .await; + match bytes { + Ok(Ok(buf)) => { + mux.send_body(id, Body::StreamItem(pb::StreamItem { item: buf.into() })); + mux.send_body(id, Body::StreamEnd(pb::StreamEnd { error: None })); + } + Ok(Err(e)) => mux.send_body(id, err_frame(e.to_string())), + Err(e) => mux.send_body(id, err_frame(format!("artifact read task failed: {e}"))), + } + } +} diff --git a/crates/plugin-remote/src/lease.rs b/crates/plugin-remote/src/lease.rs new file mode 100644 index 00000000..5ba531d3 --- /dev/null +++ b/crates/plugin-remote/src/lease.rs @@ -0,0 +1,40 @@ +//! Lease table: holds the artifact read-guards alive across a `result()` +//! callback so the plugin can pull bytes lazily. Entries are dropped on +//! `release_lease` (SDK auto-fires on `Content` drop) or on disconnect. + +use hcore::hartifactcontent::Content; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +#[derive(Default)] +pub struct LeaseTable { + next: AtomicU64, + leases: Mutex>>>, +} + +impl LeaseTable { + /// Store the artifacts for one result() callback, returning the lease id. + pub fn insert(&self, artifacts: Vec>) -> String { + let id = format!("L{}", self.next.fetch_add(1, Ordering::Relaxed)); + self.leases + .lock() + .expect("lease table") + .insert(id.clone(), artifacts); + id + } + + /// Fetch the `idx`th artifact of a lease (for `open_artifact`). + pub fn get(&self, lease_id: &str, idx: usize) -> Option> { + self.leases + .lock() + .expect("lease table") + .get(lease_id) + .and_then(|v| v.get(idx).cloned()) + } + + /// Drop all guards held under `lease_id`. + pub fn release(&self, lease_id: &str) { + self.leases.lock().expect("lease table").remove(lease_id); + } +} diff --git a/crates/plugin-remote/src/lib.rs b/crates/plugin-remote/src/lib.rs new file mode 100644 index 00000000..37110e76 --- /dev/null +++ b/crates/plugin-remote/src/lib.rs @@ -0,0 +1,19 @@ +//! Host-side adapter for running external plugins behind the in-process +//! `hplugin::Provider`/`Driver` traits. The engine registers a `RemoteProvider` +//! (or, later, `RemoteDriver`) through its normal factory hooks and stays +//! unaware the plugin is out-of-process. +//! +//! Transports: proto (UDS, always available) is the default; `shm` (iceoryx2, +//! M3) and `wasm` (wasmtime, M4) are feature-gated. + +pub mod lease; + +mod host; +mod provider; + +pub use provider::RemoteProvider; + +#[cfg(feature = "shm")] +pub mod shm; +#[cfg(feature = "wasm")] +pub mod wasm; diff --git a/crates/plugin-remote/src/provider.rs b/crates/plugin-remote/src/provider.rs new file mode 100644 index 00000000..7ce0b8fe --- /dev/null +++ b/crates/plugin-remote/src/provider.rs @@ -0,0 +1,182 @@ +//! `RemoteProvider`: implements the in-process [`hplugin::provider::Provider`] +//! trait by forwarding each method to an out-of-process plugin over a [`Mux`], +//! while serving the plugin's callbacks (result/query/…) against the per-request +//! executor. The engine registers it through the normal factory hooks and is +//! otherwise unaware the plugin is remote. + +use crate::host::{HostCallbackHandler, HostInner}; +use anyhow::Result; +use futures::future::BoxFuture; +use hcore::hasync::Cancellable; +use hmodel::htpkg::PkgBuf; +use hplugin::provider::{ + ConfigRequest, ConfigResponse, GetError, GetRequest, GetResponse, ListPackageResponse, + ListPackagesRequest, ListRequest, ListResponse, ProbeRequest, ProbeResponse, Provider, +}; +use plugin_abi::convert; +use plugin_abi::mux::{Body, Mux}; +use plugin_abi::pb; +use prost::Message; +use std::sync::Arc; +use tokio::io::{AsyncRead, AsyncWrite}; + +pub struct RemoteProvider { + name: String, + mux: Arc, + inner: Arc, +} + +impl RemoteProvider { + /// Connect over an established duplex byte stream (e.g. a UDS socketpair). + /// `name` is the provider name (from the handshake) returned by `config`. + pub fn connect(read: R, write: W, name: impl Into) -> Self + where + R: AsyncRead + Unpin + Send + 'static, + W: AsyncWrite + Unpin + Send + 'static, + { + let inner = Arc::new(HostInner::default()); + let handler = Arc::new(HostCallbackHandler { + inner: Arc::clone(&inner), + }); + let mux = Mux::start(read, write, handler); + Self { + name: name.into(), + mux, + inner, + } + } +} + +impl Provider for RemoteProvider { + fn config(&self, _req: ConfigRequest) -> Result { + // Name is learned at handshake; no round trip needed. + Ok(ConfigResponse { + name: self.name.clone(), + }) + } + + fn list<'a>( + &'a self, + req: ListRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, Result> + Send>>> { + Box::pin(async move { + let body = Body::ListReq(pb::ListRequest { + request_id: req.request_id, + package: req.package.as_str().to_string(), + states: req.states.iter().map(convert::state_to_pb).collect(), + }); + let mut rx = self.mux.call_stream(body); + let mut out: Vec> = Vec::new(); + while let Some(b) = rx.recv().await { + match b { + Body::StreamItem(si) => { + let lr = pb::ListResponse::decode(&si.item[..])?; + out.push(Ok(ListResponse { + addr: convert::addr_from_pb(lr.addr.unwrap_or_default()), + })); + } + Body::StreamEnd(se) => { + if let Some(e) = se.error { + anyhow::bail!("{}", e.message); + } + break; + } + Body::Error(e) => anyhow::bail!("{}", e.message), + _ => {} + } + } + Ok(Box::new(out.into_iter()) as Box> + Send>) + }) + } + + fn list_packages<'a>( + &'a self, + req: ListPackagesRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, Result> + Send>>> { + Box::pin(async move { + let body = Body::ListPackagesReq(pb::ListPackagesRequest { + prefix: req.prefix.as_str().to_string(), + }); + let mut rx = self.mux.call_stream(body); + let mut out: Vec> = Vec::new(); + while let Some(b) = rx.recv().await { + match b { + Body::StreamItem(si) => { + let lpr = pb::ListPackageResponse::decode(&si.item[..])?; + out.push(Ok(ListPackageResponse { + pkg: PkgBuf::from(lpr.pkg), + })); + } + Body::StreamEnd(se) => { + if let Some(e) = se.error { + anyhow::bail!("{}", e.message); + } + break; + } + Body::Error(e) => anyhow::bail!("{}", e.message), + _ => {} + } + } + Ok(Box::new(out.into_iter()) + as Box> + Send>) + }) + } + + fn get<'a>( + &'a self, + req: GetRequest, + ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, std::result::Result> { + Box::pin(async move { + let request_id = req.request_id.clone(); + // Register the executor so the plugin's result()/query() callbacks + // for this request route back to it. + self.inner + .register(request_id.clone(), Arc::clone(&req.executor)); + let body = Body::GetReq(pb::GetRequest { + request_id: req.request_id, + addr: Some(convert::addr_to_pb(&req.addr)), + states: req.states.iter().map(convert::state_to_pb).collect(), + }); + let res = self.mux.call_cancellable(body, ctoken.cancelled()).await; + self.inner.unregister(&request_id); + match res { + Ok(Body::GetResp(gr)) => Ok(GetResponse { + target_spec: convert::target_spec_from_pb(gr.target_spec.unwrap_or_default()), + }), + Ok(Body::GetErr(ge)) => { + if ge.kind == pb::get_error::Kind::NotFound as i32 { + Err(GetError::NotFound) + } else { + Err(GetError::Other(anyhow::anyhow!("{}", ge.message))) + } + } + Ok(other) => Err(GetError::Other(anyhow::anyhow!( + "unexpected get response: {other:?}" + ))), + Err(e) => Err(GetError::Other(e)), + } + }) + } + + fn probe<'a>( + &'a self, + req: ProbeRequest, + ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let body = Body::ProbeReq(pb::ProbeRequest { + request_id: req.request_id, + package: req.package.as_str().to_string(), + }); + match self.mux.call_cancellable(body, ctoken.cancelled()).await? { + Body::ProbeResp(pr) => Ok(ProbeResponse { + states: pr.states.into_iter().map(convert::state_from_pb).collect(), + }), + other => anyhow::bail!("unexpected probe response: {other:?}"), + } + }) + } +} diff --git a/crates/plugin-remote/tests/proto_e2e.rs b/crates/plugin-remote/tests/proto_e2e.rs new file mode 100644 index 00000000..a2025d6d --- /dev/null +++ b/crates/plugin-remote/tests/proto_e2e.rs @@ -0,0 +1,243 @@ +//! End-to-end test of the proto transport over an in-process UDS socketpair: +//! a guest `Provider` served by the SDK, driven by a host `RemoteProvider`, +//! exercising config, streaming list, get + the bidirectional `result()` +//! callback, and cancellation. (Real subprocess spawn is a thin wrapper added +//! next; this proves the protocol.) + +use futures::future::BoxFuture; +use hcore::hartifactcontent::{Content, WalkEntry}; +use hcore::hasync::{Cancellable, StdCancellationToken}; +use hplugin::eresult::{ArtifactMeta, EResult}; +use hplugin::provider::{ + ConfigRequest, ConfigResponse, GetError, GetRequest, GetResponse, ListPackageResponse, + ListPackagesRequest, ListRequest, ListResponse, ProbeRequest, ProbeResponse, Provider, + ProviderExecutor, TargetSpec, +}; +use hmodel::htaddr::Addr; +use hmodel::htmatcher::Matcher; +use hmodel::htpkg::PkgBuf; +use plugin_remote::RemoteProvider; +use std::collections::BTreeMap; +use std::io::{Cursor, Read}; +use std::sync::Arc; +use tokio::net::UnixStream; + +const DEP_HASH: &str = "deadbeef"; + +fn addr(pkg: &str, name: &str) -> Addr { + Addr::new(PkgBuf::from(pkg), name.to_string(), BTreeMap::new()) +} + +// ---- guest plugin ---- + +struct TestProvider; + +impl Provider for TestProvider { + fn config(&self, _req: ConfigRequest) -> anyhow::Result { + Ok(ConfigResponse { + name: "test".to_string(), + }) + } + + fn list<'a>( + &'a self, + _req: ListRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> + Send>>> + { + Box::pin(async move { + let items = vec![ + Ok(ListResponse { + addr: addr("//pkg", "a"), + }), + Ok(ListResponse { + addr: addr("//pkg", "b"), + }), + ]; + Ok(Box::new(items.into_iter()) + as Box> + Send>) + }) + } + + fn list_packages<'a>( + &'a self, + _req: ListPackagesRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture< + 'a, + anyhow::Result> + Send>>, + > { + Box::pin(async move { + let items = vec![Ok(ListPackageResponse { + pkg: PkgBuf::from("//pkg"), + })]; + Ok(Box::new(items.into_iter()) + as Box> + Send>) + }) + } + + fn get<'a>( + &'a self, + req: GetRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // Call back into the host to resolve a dependency. + let dep = addr("//dep", "lib"); + let eres = req + .executor + .result(&dep) + .await + .map_err(GetError::Other)?; + // Verify the callback round-tripped the artifact metadata. + let got = eres + .artifacts_meta + .first() + .map(|m| m.hashout.clone()) + .unwrap_or_default(); + if got != DEP_HASH { + return Err(GetError::Other(anyhow::anyhow!( + "callback hashout mismatch: {got:?}" + ))); + } + let mut spec = TargetSpec::default(); + spec.addr = req.addr; + spec.driver = "exec".to_string(); + Ok(GetResponse { target_spec: spec }) + }) + } + + fn probe<'a>( + &'a self, + _req: ProbeRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> { + Box::pin(async move { Ok(ProbeResponse { states: vec![] }) }) + } +} + +// ---- host stub engine executor ---- + +struct MemContent; + +impl Content for MemContent { + fn reader(&self) -> anyhow::Result> { + Ok(Box::new(Cursor::new(Vec::new()))) + } + fn walk(&self) -> anyhow::Result> + '_>> { + Ok(Box::new(std::iter::empty())) + } + fn hashout(&self) -> anyhow::Result { + Ok(DEP_HASH.to_string()) + } + fn byte_size(&self) -> Option { + Some(0) + } +} + +struct StubExec; + +impl ProviderExecutor for StubExec { + fn result<'a>(&'a self, _addr: &'a Addr) -> BoxFuture<'a, anyhow::Result>> { + Box::pin(async move { + Ok(Arc::new(EResult { + artifacts: vec![Arc::new(MemContent) as Arc], + support_artifacts: vec![], + artifacts_meta: vec![ArtifactMeta { + hashout: DEP_HASH.to_string(), + }], + })) + }) + } + + fn query<'a>( + &'a self, + _m: &'a Matcher, + _extra_skip: &'a [String], + ) -> BoxFuture<'a, anyhow::Result>> { + Box::pin(async move { Ok(vec![]) }) + } +} + +fn wire() -> (RemoteProvider, Arc) { + let (a, b) = UnixStream::pair().expect("socketpair"); + let (ar, aw) = a.into_split(); + let (br, bw) = b.into_split(); + let host = RemoteProvider::connect(ar, aw, "test"); + let guest = plugin_sdk::serve(Arc::new(TestProvider), br, bw); + (host, guest) +} + +#[tokio::test] +async fn config_returns_name() { + let (host, _guest) = wire(); + let resp = host.config(ConfigRequest {}).expect("config"); + assert_eq!(resp.name, "test"); +} + +#[tokio::test] +async fn list_streams_addrs() { + let (host, _guest) = wire(); + let ctoken = StdCancellationToken::new(); + let iter = host + .list( + ListRequest { + request_id: "r1".to_string(), + package: PkgBuf::from("//pkg"), + states: vec![], + }, + &ctoken, + ) + .await + .expect("list"); + let addrs: Vec<_> = iter.map(|r| r.expect("item").addr).collect(); + assert_eq!(addrs.len(), 2); + assert_eq!(addrs[0], addr("//pkg", "a")); + assert_eq!(addrs[1], addr("//pkg", "b")); +} + +#[tokio::test] +async fn get_invokes_result_callback() { + let (host, _guest) = wire(); + let ctoken = StdCancellationToken::new(); + let executor: Arc = Arc::new(StubExec); + let resp = host + .get( + GetRequest { + request_id: "r2".to_string(), + addr: addr("//pkg", "a"), + states: vec![], + executor, + }, + &ctoken, + ) + .await + .expect("get ok (callback round-tripped)"); + assert_eq!(resp.target_spec.driver, "exec"); + assert_eq!(resp.target_spec.addr, addr("//pkg", "a")); +} + +#[tokio::test] +async fn get_cancellation_returns_error() { + let (host, _guest) = wire(); + let ctoken = StdCancellationToken::new(); + ctoken.cancel(); // pre-cancelled + let executor: Arc = Arc::new(StubExec); + let res = host + .get( + GetRequest { + request_id: "r3".to_string(), + addr: addr("//pkg", "a"), + states: vec![], + executor, + }, + &ctoken, + ) + .await; + let err = res.err().expect("should be cancelled"); + let msg = match err { + GetError::Other(e) => e.to_string(), + GetError::NotFound => "notfound".to_string(), + }; + assert!(msg.contains("cancelled"), "unexpected error: {msg}"); +} diff --git a/crates/plugin-sdk/Cargo.toml b/crates/plugin-sdk/Cargo.toml index d754b7a8..060f76ac 100644 --- a/crates/plugin-sdk/Cargo.toml +++ b/crates/plugin-sdk/Cargo.toml @@ -10,6 +10,10 @@ workspace = true hplugin = { package = "plugin", path = "../plugin" } hmodel = { package = "model", path = "../model" } hcore = { package = "core", path = "../core" } -plugin-abi = { path = "../plugin-abi" } +plugin-abi = { path = "../plugin-abi", features = ["convert", "transport"] } anyhow = "1.0.102" async-trait = "0.1.89" +futures = "0.3.32" +prost = "0.14" +tokio = { version = "1.52", features = ["rt", "sync", "io-util", "macros"] } +tracing = "0.1" diff --git a/crates/plugin-sdk/src/lib.rs b/crates/plugin-sdk/src/lib.rs index 3607af85..2a9128d0 100644 --- a/crates/plugin-sdk/src/lib.rs +++ b/crates/plugin-sdk/src/lib.rs @@ -15,9 +15,11 @@ pub mod ctx; pub mod host; +pub mod serve; pub use ctx::Ctx; pub use host::HostClient; +pub use serve::serve; /// Re-export of the author-facing contract so a plugin depends only on the SDK. pub use hplugin::{driver, eresult, provider}; diff --git a/crates/plugin-sdk/src/serve.rs b/crates/plugin-sdk/src/serve.rs new file mode 100644 index 00000000..6582d434 --- /dev/null +++ b/crates/plugin-sdk/src/serve.rs @@ -0,0 +1,314 @@ +//! Guest-side serving: drive an author's [`Provider`] from inbound frames, and +//! expose the host callbacks (`result`/`query`) as a [`ProviderExecutor`] so the +//! author's code calls back into the host exactly as in-process. + +use anyhow::Result; +use async_trait::async_trait; +use futures::future::BoxFuture; +use hcore::hartifactcontent::{Content, WalkEntry}; +use hcore::hasync::StdCancellationToken; +use hplugin::eresult::{ArtifactMeta, EResult}; +use hplugin::provider::{ + ConfigRequest, GetError, GetRequest, ListPackagesRequest, ListRequest, ProbeRequest, Provider, + ProviderExecutor, +}; +use hmodel::htaddr::Addr; +use hmodel::htmatcher::Matcher; +use hmodel::htpkg::PkgBuf; +use plugin_abi::convert; +use plugin_abi::mux::{Body, InboundHandler, Mux}; +use plugin_abi::pb; +use prost::Message; +use std::collections::HashMap; +use std::io::Read; +use std::sync::{Arc, Mutex}; + +/// Serve `provider` over an established duplex byte stream. Returns the mux +/// handle; keep it alive for the lifetime of the connection (the read/write +/// tasks run in the background and end on EOF). +pub fn serve(provider: Arc, read: R, write: W) -> Arc +where + R: tokio::io::AsyncRead + Unpin + Send + 'static, + W: tokio::io::AsyncWrite + Unpin + Send + 'static, +{ + let handler = Arc::new(GuestHandler { + provider, + tokens: Mutex::new(HashMap::new()), + }); + Mux::start(read, write, handler) +} + +struct GuestHandler { + provider: Arc, + // frame-id -> cancellation token for the in-flight method call + tokens: Mutex>>, +} + +impl GuestHandler { + fn new_token(&self, id: u64) -> Arc { + let tok = Arc::new(StdCancellationToken::new()); + self.tokens.lock().expect("tokens").insert(id, Arc::clone(&tok)); + tok + } + fn drop_token(&self, id: u64) { + self.tokens.lock().expect("tokens").remove(&id); + } +} + +fn err_frame(message: String) -> Body { + Body::Error(pb::Error { + kind: pb::error::Kind::Other as i32, + message, + }) +} + +#[async_trait] +impl InboundHandler for GuestHandler { + async fn handle(&self, id: u64, body: Body, mux: Arc) { + match body { + Body::ConfigReq(_) => match self.provider.config(ConfigRequest {}) { + Ok(resp) => mux.send_body(id, Body::ConfigResp(pb::ConfigResponse { name: resp.name })), + Err(e) => mux.send_body(id, err_frame(e.to_string())), + }, + Body::ListReq(req) => { + let tok = self.new_token(id); + let lreq = ListRequest { + request_id: req.request_id, + package: PkgBuf::from(req.package), + states: req.states.into_iter().map(convert::state_from_pb).collect(), + }; + let res = self.provider.list(lreq, &*tok).await; + self.stream_addrs(id, res, &mux); + self.drop_token(id); + } + Body::ListPackagesReq(req) => { + let tok = self.new_token(id); + let lreq = ListPackagesRequest { + prefix: PkgBuf::from(req.prefix), + }; + let res = self.provider.list_packages(lreq, &*tok).await; + match res { + Ok(iter) => { + for item in iter { + match item { + Ok(lpr) => mux.send_body( + id, + Body::StreamItem(pb::StreamItem { + item: pb::ListPackageResponse { + pkg: lpr.pkg.as_str().to_string(), + } + .encode_to_vec().into(), + }), + ), + Err(e) => { + mux.send_body(id, stream_err(e.to_string())); + self.drop_token(id); + return; + } + } + } + mux.send_body(id, Body::StreamEnd(pb::StreamEnd { error: None })); + } + Err(e) => mux.send_body(id, stream_err(e.to_string())), + } + self.drop_token(id); + } + Body::GetReq(req) => { + let tok = self.new_token(id); + let executor: Arc = Arc::new(MuxExecutor { + mux: Arc::clone(&mux), + request_id: req.request_id.clone(), + }); + let greq = GetRequest { + request_id: req.request_id, + addr: convert::addr_from_pb(req.addr.unwrap_or_default()), + states: req.states.into_iter().map(convert::state_from_pb).collect(), + executor, + }; + match self.provider.get(greq, &*tok).await { + Ok(gr) => mux.send_body( + id, + Body::GetResp(pb::GetResponse { + target_spec: Some(convert::target_spec_to_pb(&gr.target_spec)), + }), + ), + Err(GetError::NotFound) => mux.send_body( + id, + Body::GetErr(pb::GetError { + kind: pb::get_error::Kind::NotFound as i32, + message: String::new(), + }), + ), + Err(GetError::Other(e)) => mux.send_body( + id, + Body::GetErr(pb::GetError { + kind: pb::get_error::Kind::Other as i32, + message: e.to_string(), + }), + ), + } + self.drop_token(id); + } + Body::ProbeReq(req) => { + let tok = self.new_token(id); + let preq = ProbeRequest { + request_id: req.request_id, + package: PkgBuf::from(req.package), + }; + match self.provider.probe(preq, &*tok).await { + Ok(pr) => mux.send_body( + id, + Body::ProbeResp(pb::ProbeResponse { + states: pr.states.iter().map(convert::state_to_pb).collect(), + }), + ), + Err(e) => mux.send_body(id, err_frame(e.to_string())), + } + self.drop_token(id); + } + Body::Cancel(c) => { + if let Some(tok) = self.tokens.lock().expect("tokens").get(&c.request_id) { + tok.cancel(); + } + } + other => mux.send_body(id, err_frame(format!("unhandled inbound request: {other:?}"))), + } + } +} + +impl GuestHandler { + fn stream_addrs( + &self, + id: u64, + res: Result> + Send>>, + mux: &Arc, + ) { + match res { + Ok(iter) => { + for item in iter { + match item { + Ok(lr) => mux.send_body( + id, + Body::StreamItem(pb::StreamItem { + item: pb::ListResponse { + addr: Some(convert::addr_to_pb(&lr.addr)), + } + .encode_to_vec().into(), + }), + ), + Err(e) => { + mux.send_body(id, stream_err(e.to_string())); + return; + } + } + } + mux.send_body(id, Body::StreamEnd(pb::StreamEnd { error: None })); + } + Err(e) => mux.send_body(id, stream_err(e.to_string())), + } + } +} + +fn stream_err(message: String) -> Body { + Body::StreamEnd(pb::StreamEnd { + error: Some(pb::Error { + kind: pb::error::Kind::Other as i32, + message, + }), + }) +} + +/// A `ProviderExecutor` that forwards `result`/`query` to the host over the mux. +struct MuxExecutor { + mux: Arc, + request_id: String, +} + +impl ProviderExecutor for MuxExecutor { + fn result<'a>(&'a self, addr: &'a Addr) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let body = Body::ResultReq(pb::ResultRequest { + request_id: self.request_id.clone(), + addr: Some(convert::addr_to_pb(addr)), + }); + match self.mux.call(body).await? { + Body::ResultResp(rr) => { + let artifacts: Vec> = rr + .artifacts + .iter() + .map(|h| { + Arc::new(RemoteContent { + hashout: h.hashout.clone(), + byte_size: h.byte_size, + }) as Arc + }) + .collect(); + let artifacts_meta = rr + .artifacts + .iter() + .map(|h| ArtifactMeta { + hashout: h.hashout.clone(), + }) + .collect(); + // M1 does not read artifact bytes, so release the lease now. + // Lazy byte streaming + lease-on-drop lands in M2. + drop( + self.mux + .call(Body::ReleaseLeaseReq(pb::ReleaseLeaseRequest { + lease_id: rr.lease_id, + })) + .await, + ); + Ok(Arc::new(EResult { + artifacts, + support_artifacts: vec![], + artifacts_meta, + })) + } + other => anyhow::bail!("unexpected result response: {other:?}"), + } + }) + } + + fn query<'a>( + &'a self, + m: &'a Matcher, + extra_skip: &'a [String], + ) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let body = Body::QueryReq(pb::QueryRequest { + request_id: self.request_id.clone(), + matcher: Some(convert::matcher_to_pb(m)), + extra_skip: extra_skip.to_vec(), + }); + match self.mux.call(body).await? { + Body::QueryResp(qr) => { + Ok(qr.addrs.into_iter().map(convert::addr_from_pb).collect()) + } + other => anyhow::bail!("unexpected query response: {other:?}"), + } + }) + } +} + +/// Host artifact handle on the guest side. M1 exposes only metadata; byte access +/// (`reader`/`walk`) streams over the mux in M2. +struct RemoteContent { + hashout: String, + byte_size: u64, +} + +impl Content for RemoteContent { + fn reader(&self) -> Result> { + anyhow::bail!("remote artifact byte access lands in M2") + } + fn walk(&self) -> Result> + '_>> { + anyhow::bail!("remote artifact walk lands in M2") + } + fn hashout(&self) -> Result { + Ok(self.hashout.clone()) + } + fn byte_size(&self) -> Option { + Some(self.byte_size) + } +} From 82b462be5d693631e97156eed3b9c9abad34cd68 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 11:18:27 +0200 Subject: [PATCH 04/44] feat(plugin-remote): subprocess spawn + reference plugin + transport stubs [M1] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - plugin-remote::spawn_plugin: spawn a plugin child over a UDS socketpair passed on inherited fd 3 (dup2 in pre_exec); proto protocol over it. - plugin-sdk::serve_inherited: guest entry — serve over fd 3, block until the host disconnects (Mux::wait_closed, new connection-closed signal). - crates/plugin-echo: reference plugin binary (heph-plugin-echo) + a real subprocess e2e test (config, streaming list, get + result() callback through the child). Green. - plugin-remote shm/wasm: feature-gated transport stubs (M3/M4). M1 complete: one logical interface, proto transport proven both in-process and across a real subprocess. shm/wasm scaffolded. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 15 ++++ Cargo.toml | 2 +- crates/plugin-abi/src/mux.rs | 34 +++++++- crates/plugin-echo/Cargo.toml | 24 ++++++ crates/plugin-echo/src/main.rs | 101 +++++++++++++++++++++++ crates/plugin-echo/tests/subprocess.rs | 103 ++++++++++++++++++++++++ crates/plugin-remote/Cargo.toml | 1 + crates/plugin-remote/src/lib.rs | 4 + crates/plugin-remote/src/shm.rs | 7 ++ crates/plugin-remote/src/spawn.rs | 56 +++++++++++++ crates/plugin-remote/src/wasm.rs | 6 ++ crates/plugin-remote/tests/proto_e2e.rs | 8 +- crates/plugin-sdk/Cargo.toml | 2 +- crates/plugin-sdk/src/lib.rs | 2 + crates/plugin-sdk/src/serve.rs | 16 ++++ 15 files changed, 374 insertions(+), 7 deletions(-) create mode 100644 crates/plugin-echo/Cargo.toml create mode 100644 crates/plugin-echo/src/main.rs create mode 100644 crates/plugin-echo/tests/subprocess.rs create mode 100644 crates/plugin-remote/src/shm.rs create mode 100644 crates/plugin-remote/src/spawn.rs create mode 100644 crates/plugin-remote/src/wasm.rs diff --git a/Cargo.lock b/Cargo.lock index 58806111..5bc3bc42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3578,6 +3578,20 @@ dependencies = [ "walk", ] +[[package]] +name = "plugin-echo" +version = "0.1.0" +dependencies = [ + "anyhow", + "core", + "futures", + "model", + "plugin", + "plugin-remote", + "plugin-sdk", + "tokio", +] + [[package]] name = "plugin-exec" version = "0.1.0" @@ -3671,6 +3685,7 @@ dependencies = [ "async-trait", "core", "futures", + "libc", "model", "plugin", "plugin-abi", diff --git a/Cargo.toml b/Cargo.toml index ef3d7aff..5f56d832 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2024" [workspace] -members = ["gen/proto", "crates/e2e", "crates/testkit", "crates/plugingo-e2e", "crates/htspec-derive", "crates/core", "crates/walk", "crates/proc", "crates/model", "crates/sandboxfuse", "crates/plugin", "crates/plugin-abi", "crates/plugin-sdk", "crates/plugin-remote", "crates/builtins", "crates/plugin-buildfile", "crates/driver-support", "crates/plugin-exec", "crates/plugin-nix", "crates/plugin-query", "crates/plugin-go", "crates/telemetry", "crates/tui", "crates/lock", "crates/engine"] +members = ["gen/proto", "crates/e2e", "crates/testkit", "crates/plugingo-e2e", "crates/htspec-derive", "crates/core", "crates/walk", "crates/proc", "crates/model", "crates/sandboxfuse", "crates/plugin", "crates/plugin-abi", "crates/plugin-sdk", "crates/plugin-remote", "crates/plugin-echo", "crates/builtins", "crates/plugin-buildfile", "crates/driver-support", "crates/plugin-exec", "crates/plugin-nix", "crates/plugin-query", "crates/plugin-go", "crates/telemetry", "crates/tui", "crates/lock", "crates/engine"] [profile.profiling] inherits = "release" diff --git a/crates/plugin-abi/src/mux.rs b/crates/plugin-abi/src/mux.rs index 035c660c..6f264584 100644 --- a/crates/plugin-abi/src/mux.rs +++ b/crates/plugin-abi/src/mux.rs @@ -14,10 +14,10 @@ use crate::pb; use async_trait::async_trait; use std::collections::HashMap; use std::sync::Mutex; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::{mpsc, oneshot, Notify}; pub use pb::frame::Body; @@ -66,6 +66,8 @@ pub struct Mux { out: mpsc::UnboundedSender, next_id: AtomicU64, pending: Mutex>, + closed: AtomicBool, + closed_notify: Notify, } impl Mux { @@ -82,6 +84,8 @@ impl Mux { out: out_tx, next_id: AtomicU64::new(1), pending: Mutex::new(HashMap::new()), + closed: AtomicBool::new(false), + closed_notify: Notify::new(), }); // writer task @@ -96,6 +100,31 @@ impl Mux { self.next_id.fetch_add(1, Ordering::Relaxed) } + /// True once the connection has closed (peer EOF or I/O error). + pub fn is_closed(&self) -> bool { + self.closed.load(Ordering::Acquire) + } + + /// Resolves when the connection closes. A guest's `main` awaits this to + /// stay alive for the lifetime of the connection and exit on disconnect. + pub async fn wait_closed(&self) { + loop { + if self.is_closed() { + return; + } + let notified = self.closed_notify.notified(); + if self.is_closed() { + return; + } + notified.await; + } + } + + fn mark_closed(&self) { + self.closed.store(true, Ordering::Release); + self.closed_notify.notify_waiters(); + } + /// Send a frame with an explicit id (used by handlers to reply). pub fn send_body(&self, id: u64, body: Body) { drop(self.out.send(pb::Frame { @@ -208,4 +237,5 @@ async fn reader_loop( } } } + mux.mark_closed(); } diff --git a/crates/plugin-echo/Cargo.toml b/crates/plugin-echo/Cargo.toml new file mode 100644 index 00000000..d0dbda5b --- /dev/null +++ b/crates/plugin-echo/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "plugin-echo" +version = "0.1.0" +edition = "2024" + +[lints] +workspace = true + +[[bin]] +name = "heph-plugin-echo" +path = "src/main.rs" + +[dependencies] +plugin-sdk = { path = "../plugin-sdk" } +hplugin = { package = "plugin", path = "../plugin" } +hmodel = { package = "model", path = "../model" } +hcore = { package = "core", path = "../core" } +anyhow = "1.0.102" +futures = "0.3.32" +tokio = { version = "1.52", features = ["rt-multi-thread", "macros"] } + +[dev-dependencies] +plugin-remote = { path = "../plugin-remote" } +tokio = { version = "1.52", features = ["rt-multi-thread", "macros", "time"] } diff --git a/crates/plugin-echo/src/main.rs b/crates/plugin-echo/src/main.rs new file mode 100644 index 00000000..c0af4a6e --- /dev/null +++ b/crates/plugin-echo/src/main.rs @@ -0,0 +1,101 @@ +//! Reference out-of-process plugin: a minimal provider served over the proto +//! transport. Demonstrates the SDK surface (implement `hplugin::Provider`, call +//! back into the host via `executor`) and serves as the subprocess fixture for +//! the transport e2e test. + +use futures::future::BoxFuture; +use hcore::hasync::Cancellable; +use hmodel::htaddr::Addr; +use hmodel::htpkg::PkgBuf; +use hplugin::provider::{ + ConfigRequest, ConfigResponse, GetError, GetRequest, GetResponse, ListPackageResponse, + ListPackagesRequest, ListRequest, ListResponse, ProbeRequest, ProbeResponse, Provider, + TargetSpec, +}; +use std::collections::BTreeMap; +use std::sync::Arc; + +struct EchoProvider; + +fn addr(pkg: &str, name: &str) -> Addr { + Addr::new(PkgBuf::from(pkg), name.to_string(), BTreeMap::new()) +} + +impl Provider for EchoProvider { + fn config(&self, _req: ConfigRequest) -> anyhow::Result { + Ok(ConfigResponse { + name: "echo".to_string(), + }) + } + + fn list<'a>( + &'a self, + _req: ListRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> + Send>>> + { + Box::pin(async move { + let items = vec![ + Ok(ListResponse { + addr: addr("//pkg", "a"), + }), + Ok(ListResponse { + addr: addr("//pkg", "b"), + }), + ]; + Ok(Box::new(items.into_iter()) + as Box> + Send>) + }) + } + + fn list_packages<'a>( + &'a self, + _req: ListPackagesRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture< + 'a, + anyhow::Result> + Send>>, + > { + Box::pin(async move { + let items = vec![Ok(ListPackageResponse { + pkg: PkgBuf::from("//pkg"), + })]; + Ok(Box::new(items.into_iter()) + as Box> + Send>) + }) + } + + fn get<'a>( + &'a self, + req: GetRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // Exercise the bidirectional callback: resolve a dependency. + let _eres = req + .executor + .result(&addr("//dep", "lib")) + .await + .map_err(GetError::Other)?; + let spec = TargetSpec { + addr: req.addr, + driver: "exec".to_string(), + ..Default::default() + }; + Ok(GetResponse { target_spec: spec }) + }) + } + + fn probe<'a>( + &'a self, + _req: ProbeRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> { + Box::pin(async move { Ok(ProbeResponse { states: vec![] }) }) + } +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { + plugin_sdk::serve_inherited(Arc::new(EchoProvider)).await +} diff --git a/crates/plugin-echo/tests/subprocess.rs b/crates/plugin-echo/tests/subprocess.rs new file mode 100644 index 00000000..a04ab468 --- /dev/null +++ b/crates/plugin-echo/tests/subprocess.rs @@ -0,0 +1,103 @@ +//! Real subprocess e2e: spawn the `heph-plugin-echo` binary, connect over the +//! proto transport (inherited fd 3), and drive it — config, streaming list, and +//! a `get` that triggers the bidirectional `result()` callback into a host stub. + +use futures::future::BoxFuture; +use hcore::hartifactcontent::{Content, WalkEntry}; +use hcore::hasync::StdCancellationToken; +use hplugin::eresult::{ArtifactMeta, EResult}; +use hplugin::provider::{ + ConfigRequest, GetRequest, ListRequest, Provider, ProviderExecutor, +}; +use hmodel::htaddr::Addr; +use hmodel::htmatcher::Matcher; +use hmodel::htpkg::PkgBuf; +use plugin_remote::spawn_plugin; +use std::collections::BTreeMap; +use std::io::{Cursor, Read}; +use std::path::Path; +use std::sync::Arc; + +fn addr(pkg: &str, name: &str) -> Addr { + Addr::new(PkgBuf::from(pkg), name.to_string(), BTreeMap::new()) +} + +struct MemContent; +impl Content for MemContent { + fn reader(&self) -> anyhow::Result> { + Ok(Box::new(Cursor::new(Vec::new()))) + } + fn walk(&self) -> anyhow::Result> + '_>> { + Ok(Box::new(std::iter::empty())) + } + fn hashout(&self) -> anyhow::Result { + Ok("deadbeef".to_string()) + } +} + +struct StubExec; +impl ProviderExecutor for StubExec { + fn result<'a>(&'a self, _addr: &'a Addr) -> BoxFuture<'a, anyhow::Result>> { + Box::pin(async move { + Ok(Arc::new(EResult { + artifacts: vec![Arc::new(MemContent) as Arc], + support_artifacts: vec![], + artifacts_meta: vec![ArtifactMeta { + hashout: "deadbeef".to_string(), + }], + })) + }) + } + fn query<'a>( + &'a self, + _m: &'a Matcher, + _extra_skip: &'a [String], + ) -> BoxFuture<'a, anyhow::Result>> { + Box::pin(async move { Ok(vec![]) }) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn echo_plugin_subprocess() { + let exe = env!("CARGO_BIN_EXE_heph-plugin-echo"); + let (host, mut child) = spawn_plugin(Path::new(exe), &[], "echo").expect("spawn"); + + // config (no round trip — name from handshake) + assert_eq!(host.config(ConfigRequest {}).expect("config").name, "echo"); + + let ctoken = StdCancellationToken::new(); + + // streaming list + let iter = host + .list( + ListRequest { + request_id: "r1".to_string(), + package: PkgBuf::from("//pkg"), + states: vec![], + }, + &ctoken, + ) + .await + .expect("list"); + let addrs: Vec<_> = iter.map(|r| r.expect("item").addr).collect(); + assert_eq!(addrs, vec![addr("//pkg", "a"), addr("//pkg", "b")]); + + // get -> triggers result() callback back into the host StubExec + let executor: Arc = Arc::new(StubExec); + let resp = host + .get( + GetRequest { + request_id: "r2".to_string(), + addr: addr("//pkg", "a"), + states: vec![], + executor, + }, + &ctoken, + ) + .await + .expect("get (callback round-tripped through the subprocess)"); + assert_eq!(resp.target_spec.driver, "exec"); + + drop(child.kill()); + drop(child.wait()); +} diff --git a/crates/plugin-remote/Cargo.toml b/crates/plugin-remote/Cargo.toml index 065f836f..a2a3ae4f 100644 --- a/crates/plugin-remote/Cargo.toml +++ b/crates/plugin-remote/Cargo.toml @@ -18,6 +18,7 @@ futures = "0.3.32" prost = "0.14" tokio = { version = "1.52", features = ["rt", "sync", "io-util", "macros", "net"] } tracing = "0.1" +libc = "0.2" [features] # heavy transports, off by default; proto (UDS) is always available diff --git a/crates/plugin-remote/src/lib.rs b/crates/plugin-remote/src/lib.rs index 37110e76..a59d1fcc 100644 --- a/crates/plugin-remote/src/lib.rs +++ b/crates/plugin-remote/src/lib.rs @@ -10,8 +10,12 @@ pub mod lease; mod host; mod provider; +#[cfg(unix)] +mod spawn; pub use provider::RemoteProvider; +#[cfg(unix)] +pub use spawn::{spawn_plugin, PLUGIN_FD}; #[cfg(feature = "shm")] pub mod shm; diff --git a/crates/plugin-remote/src/shm.rs b/crates/plugin-remote/src/shm.rs new file mode 100644 index 00000000..472a4eea --- /dev/null +++ b/crates/plugin-remote/src/shm.rs @@ -0,0 +1,7 @@ +//! shm transport (iceoryx2) — milestone M3. +//! +//! Will carry the same `Frame` bodies as the proto transport over two iceoryx2 +//! request-response services (one per direction), with the rkyv/capnp payload +//! fast path and batched `note_dep`. Scaffolded here so the feature wiring and +//! the transport-agnostic boundary exist from M1; the implementation lands in +//! M3 (the perf-hardening milestone). diff --git a/crates/plugin-remote/src/spawn.rs b/crates/plugin-remote/src/spawn.rs new file mode 100644 index 00000000..5e2c5f53 --- /dev/null +++ b/crates/plugin-remote/src/spawn.rs @@ -0,0 +1,56 @@ +//! Spawn a plugin subprocess for the proto transport. +//! +//! The host creates a UDS socketpair, passes one end to the child on a fixed +//! inherited fd (3), and speaks the `Frame` protocol over it. The child reads +//! fd 3 (see `plugin_sdk::serve_inherited`). Stdio is left to the child for its +//! own logging. +//! +//! M1 returns the child handle so the caller manages its lifetime; integrating +//! `hproc::process_supervisor` reaping + the `launch` sandbox modes is M4. + +use crate::RemoteProvider; +use std::os::unix::io::{AsRawFd, RawFd}; +use std::os::unix::process::CommandExt; +use std::path::Path; +use std::process::{Child, Command}; + +/// The fd the plugin child inherits the protocol socket on. +pub const PLUGIN_FD: RawFd = 3; + +/// Spawn `program` as a plugin and connect to it over proto. Returns the host +/// adapter plus the child handle (kill/wait it to control its lifetime). +pub fn spawn_plugin( + program: &Path, + args: &[String], + name: impl Into, +) -> anyhow::Result<(RemoteProvider, Child)> { + let (parent, child_end) = std::os::unix::net::UnixStream::pair()?; + parent.set_nonblocking(true)?; + let child_fd = child_end.as_raw_fd(); + + let mut cmd = Command::new(program); + cmd.args(args); + // Runs post-fork, pre-exec. dup2 clears CLOEXEC on the new fd so fd 3 + // survives exec, while the original (CLOEXEC) socketpair fd closes. + let pre = move || -> std::io::Result<()> { + // SAFETY: dup2 is async-signal-safe; child_fd is a valid inherited fd. + let rc = unsafe { libc::dup2(child_fd, PLUGIN_FD) }; + if rc < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }; + // SAFETY: `pre` only calls the async-signal-safe dup2. + unsafe { + cmd.pre_exec(pre); + } + let child = cmd + .spawn() + .map_err(|e| anyhow::anyhow!("spawn plugin {}: {e}", program.display()))?; + // Parent no longer needs its copy of the child end. + drop(child_end); + + let tokio_parent = tokio::net::UnixStream::from_std(parent)?; + let (r, w) = tokio_parent.into_split(); + Ok((RemoteProvider::connect(r, w, name), child)) +} diff --git a/crates/plugin-remote/src/wasm.rs b/crates/plugin-remote/src/wasm.rs new file mode 100644 index 00000000..ae9fc4b4 --- /dev/null +++ b/crates/plugin-remote/src/wasm.rs @@ -0,0 +1,6 @@ +//! wasm transport (in-process wasmtime component) — milestone M4. +//! +//! Will instantiate a plugin `.wasm` component via wasmtime, bind the WIT +//! interface (`wit/heph-plugin.wit`), wire the `AbiHost` callbacks as host +//! imports, and grant capabilities (WASI preopens) per the `launch` policy. +//! Scaffolded here so the feature wiring exists from M1; implemented in M4. diff --git a/crates/plugin-remote/tests/proto_e2e.rs b/crates/plugin-remote/tests/proto_e2e.rs index a2025d6d..758c4f88 100644 --- a/crates/plugin-remote/tests/proto_e2e.rs +++ b/crates/plugin-remote/tests/proto_e2e.rs @@ -100,9 +100,11 @@ impl Provider for TestProvider { "callback hashout mismatch: {got:?}" ))); } - let mut spec = TargetSpec::default(); - spec.addr = req.addr; - spec.driver = "exec".to_string(); + let spec = TargetSpec { + addr: req.addr, + driver: "exec".to_string(), + ..Default::default() + }; Ok(GetResponse { target_spec: spec }) }) } diff --git a/crates/plugin-sdk/Cargo.toml b/crates/plugin-sdk/Cargo.toml index 060f76ac..5f4acf5c 100644 --- a/crates/plugin-sdk/Cargo.toml +++ b/crates/plugin-sdk/Cargo.toml @@ -15,5 +15,5 @@ anyhow = "1.0.102" async-trait = "0.1.89" futures = "0.3.32" prost = "0.14" -tokio = { version = "1.52", features = ["rt", "sync", "io-util", "macros"] } +tokio = { version = "1.52", features = ["rt", "sync", "io-util", "macros", "net"] } tracing = "0.1" diff --git a/crates/plugin-sdk/src/lib.rs b/crates/plugin-sdk/src/lib.rs index 2a9128d0..a9b3e2a1 100644 --- a/crates/plugin-sdk/src/lib.rs +++ b/crates/plugin-sdk/src/lib.rs @@ -20,6 +20,8 @@ pub mod serve; pub use ctx::Ctx; pub use host::HostClient; pub use serve::serve; +#[cfg(unix)] +pub use serve::serve_inherited; /// Re-export of the author-facing contract so a plugin depends only on the SDK. pub use hplugin::{driver, eresult, provider}; diff --git a/crates/plugin-sdk/src/serve.rs b/crates/plugin-sdk/src/serve.rs index 6582d434..30b8318b 100644 --- a/crates/plugin-sdk/src/serve.rs +++ b/crates/plugin-sdk/src/serve.rs @@ -38,6 +38,22 @@ where Mux::start(read, write, handler) } +/// Guest entry point: serve `provider` over the inherited fd 3 (set up by the +/// host's `spawn_plugin`) and block until the host disconnects. A plugin's +/// `main` typically does just `serve_inherited(Arc::new(MyProvider)).await`. +#[cfg(unix)] +pub async fn serve_inherited(provider: Arc) -> anyhow::Result<()> { + use std::os::unix::io::FromRawFd; + // SAFETY: fd 3 is the protocol socket the host passed us at spawn; we own it. + let std_stream = unsafe { std::os::unix::net::UnixStream::from_raw_fd(3) }; + std_stream.set_nonblocking(true)?; + let stream = tokio::net::UnixStream::from_std(std_stream)?; + let (r, w) = stream.into_split(); + let mux = serve(provider, r, w); + mux.wait_closed().await; + Ok(()) +} + struct GuestHandler { provider: Arc, // frame-id -> cancellation token for the in-flight method call From c78d2afe62699593b8ff3cdd8dac0248c94c74ef Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 11:23:38 +0200 Subject: [PATCH 05/44] feat(plugin-sdk): artifact byte streaming over the result() callback [M2] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete the provider result() path: the guest now eagerly pulls each artifact's bytes via open_artifact (streamed chunks), materializes them as RemoteContent, and reads/walks them locally (tar walker — the only content type the cache produces). This is the plugin-go hot read (package.bin). e2e extended: host hands back a tar artifact, guest get() fetches it through the transport, walks it, and reads package.bin = "hello". Green. Lazy/offset chunking + lease-on-Content-drop remain M3 perf work. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-remote/tests/proto_e2e.rs | 45 +++++++++++++-- crates/plugin-sdk/src/serve.rs | 75 +++++++++++++++++-------- 2 files changed, 91 insertions(+), 29 deletions(-) diff --git a/crates/plugin-remote/tests/proto_e2e.rs b/crates/plugin-remote/tests/proto_e2e.rs index 758c4f88..cbb2213b 100644 --- a/crates/plugin-remote/tests/proto_e2e.rs +++ b/crates/plugin-remote/tests/proto_e2e.rs @@ -5,7 +5,7 @@ //! next; this proves the protocol.) use futures::future::BoxFuture; -use hcore::hartifactcontent::{Content, WalkEntry}; +use hcore::hartifactcontent::{Content, WalkEntry, WalkEntryKind}; use hcore::hasync::{Cancellable, StdCancellationToken}; use hplugin::eresult::{ArtifactMeta, EResult}; use hplugin::provider::{ @@ -100,6 +100,29 @@ impl Provider for TestProvider { "callback hashout mismatch: {got:?}" ))); } + // Read package.bin out of the artifact — exercises byte streaming + + // tar walk through the transport (the plugin-go hot read). + let art = eres + .artifacts + .first() + .ok_or_else(|| GetError::Other(anyhow::anyhow!("no artifacts")))?; + let mut found: Option = None; + for entry in art.walk().map_err(GetError::Other)? { + let entry = entry.map_err(GetError::Other)?; + if entry.path.file_name().and_then(|n| n.to_str()) == Some("package.bin") + && let WalkEntryKind::File { mut data, .. } = entry.kind + { + let mut s = String::new(); + data.read_to_string(&mut s) + .map_err(|e| GetError::Other(e.into()))?; + found = Some(s); + } + } + if found.as_deref() != Some("hello") { + return Err(GetError::Other(anyhow::anyhow!( + "package.bin mismatch: {found:?}" + ))); + } let spec = TargetSpec { addr: req.addr, driver: "exec".to_string(), @@ -120,11 +143,23 @@ impl Provider for TestProvider { // ---- host stub engine executor ---- -struct MemContent; +/// A tar artifact containing `package.bin` = "hello" — what the host would +/// hand back from a cache result; the guest must fetch + walk it. +fn pkg_tar() -> Vec { + let mut p = hcore::hartifactcontent::tar::TarPacker::new(); + p.create_raw(b"hello".to_vec(), "package.bin", false); + let mut buf = Vec::new(); + p.pack(&mut buf).expect("pack tar"); + buf +} + +struct MemContent { + bytes: Vec, +} impl Content for MemContent { fn reader(&self) -> anyhow::Result> { - Ok(Box::new(Cursor::new(Vec::new()))) + Ok(Box::new(Cursor::new(self.bytes.clone()))) } fn walk(&self) -> anyhow::Result> + '_>> { Ok(Box::new(std::iter::empty())) @@ -133,7 +168,7 @@ impl Content for MemContent { Ok(DEP_HASH.to_string()) } fn byte_size(&self) -> Option { - Some(0) + Some(self.bytes.len() as u64) } } @@ -143,7 +178,7 @@ impl ProviderExecutor for StubExec { fn result<'a>(&'a self, _addr: &'a Addr) -> BoxFuture<'a, anyhow::Result>> { Box::pin(async move { Ok(Arc::new(EResult { - artifacts: vec![Arc::new(MemContent) as Arc], + artifacts: vec![Arc::new(MemContent { bytes: pkg_tar() }) as Arc], support_artifacts: vec![], artifacts_meta: vec![ArtifactMeta { hashout: DEP_HASH.to_string(), diff --git a/crates/plugin-sdk/src/serve.rs b/crates/plugin-sdk/src/serve.rs index 30b8318b..003b342b 100644 --- a/crates/plugin-sdk/src/serve.rs +++ b/crates/plugin-sdk/src/serve.rs @@ -240,6 +240,32 @@ struct MuxExecutor { request_id: String, } +impl MuxExecutor { + /// Pull all bytes of one artifact via `open_artifact` (streamed as chunks). + async fn fetch_artifact(&self, lease_id: &str, handle_id: &str) -> Result> { + let mut rx = self.mux.call_stream(Body::OpenArtifactReq(pb::OpenArtifactRequest { + lease_id: lease_id.to_string(), + handle_id: handle_id.to_string(), + offset: 0, + })); + let mut bytes = Vec::new(); + while let Some(b) = rx.recv().await { + match b { + Body::StreamItem(si) => bytes.extend_from_slice(&si.item), + Body::StreamEnd(se) => { + if let Some(e) = se.error { + anyhow::bail!("{}", e.message); + } + break; + } + Body::Error(e) => anyhow::bail!("{}", e.message), + _ => {} + } + } + Ok(bytes) + } +} + impl ProviderExecutor for MuxExecutor { fn result<'a>(&'a self, addr: &'a Addr) -> BoxFuture<'a, Result>> { Box::pin(async move { @@ -249,25 +275,23 @@ impl ProviderExecutor for MuxExecutor { }); match self.mux.call(body).await? { Body::ResultResp(rr) => { - let artifacts: Vec> = rr - .artifacts - .iter() - .map(|h| { - Arc::new(RemoteContent { - hashout: h.hashout.clone(), - byte_size: h.byte_size, - }) as Arc - }) - .collect(); - let artifacts_meta = rr - .artifacts - .iter() - .map(|h| ArtifactMeta { + // Eagerly pull each artifact's bytes over the mux. M2: whole + // artifact in one fetch; lazy/offset chunking is M3. Most + // plugins read a tiny file (e.g. package.bin) so this is + // cheap in practice. + let mut artifacts: Vec> = Vec::with_capacity(rr.artifacts.len()); + let mut artifacts_meta = Vec::with_capacity(rr.artifacts.len()); + for h in &rr.artifacts { + let bytes = self.fetch_artifact(&rr.lease_id, &h.handle_id).await?; + artifacts.push(Arc::new(RemoteContent { + bytes, hashout: h.hashout.clone(), - }) - .collect(); - // M1 does not read artifact bytes, so release the lease now. - // Lazy byte streaming + lease-on-drop lands in M2. + }) as Arc); + artifacts_meta.push(ArtifactMeta { + hashout: h.hashout.clone(), + }); + } + // Bytes are now owned locally; release the host-side guards. drop( self.mux .call(Body::ReleaseLeaseReq(pb::ReleaseLeaseRequest { @@ -307,24 +331,27 @@ impl ProviderExecutor for MuxExecutor { } } -/// Host artifact handle on the guest side. M1 exposes only metadata; byte access -/// (`reader`/`walk`) streams over the mux in M2. +/// A host artifact materialized on the guest side: the bytes are fetched eagerly +/// over the mux, then read/walked locally. Artifacts are tar (the only content +/// type the cache produces today), so `walk` uses the tar walker. struct RemoteContent { + bytes: Vec, hashout: String, - byte_size: u64, } impl Content for RemoteContent { fn reader(&self) -> Result> { - anyhow::bail!("remote artifact byte access lands in M2") + Ok(Box::new(std::io::Cursor::new(self.bytes.clone()))) } fn walk(&self) -> Result> + '_>> { - anyhow::bail!("remote artifact walk lands in M2") + Ok(Box::new(hcore::hartifactcontent::tar::TarWalker::new( + std::io::Cursor::new(self.bytes.clone()), + )?)) } fn hashout(&self) -> Result { Ok(self.hashout.clone()) } fn byte_size(&self) -> Option { - Some(self.byte_size) + Some(self.bytes.len() as u64) } } From 6507008943cd1434e956c5d1de6fd6036ad95f16 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 15:36:54 +0200 Subject: [PATCH 06/44] feat(plugin): RawDef serialization contract for cross-boundary round-trip [M2] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A driver's parse() produces a concrete Arc that run()/ apply_transitive() read back via downcast. RawDef was serialize-only (erased), so a target def shipped to a remote plugin couldn't be turned back into the driver's concrete config type. This adds the missing contract: - hplugin: RawDefBytes — a RawDef that carries the serialized value and lazily deserializes + caches it into the requested type. Serializes transparently as the wrapped value so re-shipping is lossless. - hplugin: TargetDef::def_de::() — downcasts the concrete value in-process (zero cost) or materializes it from a RawDefBytes. def::() (downcast-only) is unchanged, since pluginexec stores a nested TargetDef (not Deserialize) as its raw_def. - plugin-abi: convert::raw_def_to_blob / raw_def_from_blob bridge to pb::RawDefBlob (JSON). Remote-capable driver def types derive Serialize+Deserialize and read config via def_de. Tests cover in-proc downcast, serialized materialization, and lossless re-serialization. This unblocks RemoteDriver. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 2 + crates/plugin-abi/Cargo.toml | 5 +- crates/plugin-abi/src/convert.rs | 37 ++++++++ crates/plugin/Cargo.toml | 1 + crates/plugin/src/driver.rs | 146 +++++++++++++++++++++++++++++++ 5 files changed, 190 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 5bc3bc42..a8c834f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3546,6 +3546,8 @@ dependencies = [ "prost 0.14.4", "proto-gen", "rkyv", + "serde", + "serde_json", "tokio", "tracing", ] diff --git a/crates/plugin-abi/Cargo.toml b/crates/plugin-abi/Cargo.toml index 54fbba7f..1976b2ac 100644 --- a/crates/plugin-abi/Cargo.toml +++ b/crates/plugin-abi/Cargo.toml @@ -15,6 +15,7 @@ rkyv = "0.8" hplugin = { package = "plugin", path = "../plugin", optional = true } hmodel = { package = "model", path = "../model", optional = true } hcore = { package = "core", path = "../core", optional = true } +serde_json = { version = "1.0", optional = true } # transport feature: generic length-prefixed Frame framing + bidirectional mux # over any AsyncRead/AsyncWrite. Shared by the host adapter and the guest SDK. prost = { version = "0.14", optional = true } @@ -23,8 +24,10 @@ async-trait = { version = "0.1.89", optional = true } tracing = { version = "0.1", optional = true } [features] -convert = ["dep:hplugin", "dep:hmodel", "dep:hcore"] +convert = ["dep:hplugin", "dep:hmodel", "dep:hcore", "dep:serde_json"] transport = ["dep:prost", "dep:tokio", "dep:async-trait", "dep:tracing"] [dev-dependencies] rkyv = "0.8" +serde = { version = "1", features = ["derive"] } +serde_json = "1.0" diff --git a/crates/plugin-abi/src/convert.rs b/crates/plugin-abi/src/convert.rs index 64aefef7..932d4f6e 100644 --- a/crates/plugin-abi/src/convert.rs +++ b/crates/plugin-abi/src/convert.rs @@ -12,9 +12,11 @@ use hmodel::htaddr::Addr; use hmodel::htmatcher::Matcher; use hmodel::htpkg::PkgBuf; use hplugin::driver::sandbox::{Dep, Env, EnvValue, Mode, Sandbox, Tool}; +use hplugin::driver::targetdef::{RawDef, RawDefBytes}; use hplugin::driver::TargetAddr; use hplugin::provider::{State, TargetSpec}; use std::collections::BTreeMap; +use std::sync::Arc; // ---- Addr ---- @@ -284,6 +286,25 @@ pub fn matcher_from_pb(m: pb::Matcher) -> Matcher { } } +// ---- raw_def (opaque driver blob) ---- + +/// Serialize a driver's `raw_def` to a wire blob (JSON). Works on any `RawDef`, +/// whether a concrete value (in-process) or a round-tripped [`RawDefBytes`]. +pub fn raw_def_to_blob(raw: &Arc) -> anyhow::Result { + let data = serde_json::to_vec(&**raw)?; + Ok(pb::RawDefBlob { + driver: String::new(), + format: pb::raw_def_blob::Format::Json as i32, + data: data.into(), + }) +} + +/// Reconstruct a `raw_def` from a wire blob as a [`RawDefBytes`] carrier. The +/// receiving driver reads its concrete config via `TargetDef::def_de`. +pub fn raw_def_from_blob(blob: &pb::RawDefBlob) -> anyhow::Result> { + Ok(Arc::new(RawDefBytes::from_json_slice(&blob.data)?)) +} + #[cfg(test)] mod tests { use super::*; @@ -354,4 +375,20 @@ mod tests { ]); assert_eq!(matcher_from_pb(matcher_to_pb(&m)), m); } + + #[test] + fn raw_def_blob_roundtrip() { + #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] + struct D { + x: u32, + } + let raw: Arc = Arc::new(D { x: 5 }); + let blob = raw_def_to_blob(&raw).expect("to blob"); + let back = raw_def_from_blob(&blob).expect("from blob"); + // The reconstructed RawDefBytes re-serializes to the original value. + assert_eq!( + serde_json::to_value(&*back).expect("reserialize"), + serde_json::json!({"x": 5}) + ); + } } diff --git a/crates/plugin/Cargo.toml b/crates/plugin/Cargo.toml index f317526b..04d13fe8 100644 --- a/crates/plugin/Cargo.toml +++ b/crates/plugin/Cargo.toml @@ -14,6 +14,7 @@ hsandboxfuse = { package = "sandboxfuse", path = "../sandboxfuse", default-featu htspec-derive = { path = "../htspec-derive" } anyhow = "1.0.102" serde = { version = "1", features = ["derive"] } +serde_json = "1.0" serde_yaml = "0.9" async-trait = "0.1.89" erased-serde = "0.4" diff --git a/crates/plugin/src/driver.rs b/crates/plugin/src/driver.rs index a015de4b..7ab72be1 100644 --- a/crates/plugin/src/driver.rs +++ b/crates/plugin/src/driver.rs @@ -422,6 +422,61 @@ pub mod targetdef { erased_serde::serialize(&**d, s) } + /// A `RawDef` carried as serialized bytes rather than a concrete value. + /// + /// `RawDef` is serialize-only (erased), which is enough to ship a target def + /// out of the process, but a driver's `run`/`apply_transitive` needs its + /// concrete config type back. This carrier closes that loop: it holds the + /// serialized value and lazily deserializes it into the requested type on + /// the first [`TargetDef::def_de`] call, caching the materialized value so a + /// `&T` can be handed out (same ergonomics as the in-process downcast path). + /// + /// This is how a `TargetDef` round-trips across the plugin boundary (proto / + /// shm / wasm): the host wraps the wire blob in a `RawDefBytes`, and the + /// driver reads it with `def_de::()`. + pub struct RawDefBytes { + value: serde_json::Value, + cell: std::sync::OnceLock>, + } + + impl RawDefBytes { + /// Wrap an already-parsed JSON value. + pub fn from_value(value: serde_json::Value) -> Self { + Self { + value, + cell: std::sync::OnceLock::new(), + } + } + + /// Wrap a JSON-encoded raw-def blob (as produced by serializing a + /// `RawDef`). + pub fn from_json_slice(bytes: &[u8]) -> anyhow::Result { + Ok(Self::from_value(serde_json::from_slice(bytes)?)) + } + + fn materialize(&self) -> &T + where + T: serde::de::DeserializeOwned + Send + Sync + 'static, + { + let boxed = self.cell.get_or_init(|| { + let v: T = serde_json::from_value(self.value.clone()) + .expect("RawDefBytes: deserialize into requested raw_def type"); + Box::new(v) as Box + }); + boxed + .downcast_ref::() + .expect("RawDefBytes: materialized raw_def type mismatch") + } + } + + // Serialize transparently as the wrapped value, so re-serializing a + // round-tripped `TargetDef` yields the original blob. + impl serde::Serialize for RawDefBytes { + fn serialize(&self, s: S) -> Result { + self.value.serialize(s) + } + } + /// Per-target cache configuration. /// /// `enabled` gates local caching; `remote_enabled` gates the remote cache @@ -484,6 +539,26 @@ pub mod targetdef { .downcast_ref::() .expect("TargetDef raw_def type mismatch: wrong type T requested") } + + /// Like [`def`](Self::def), but also works when the raw_def crossed a + /// process boundary and arrived as a serialized [`RawDefBytes`]: it + /// downcasts the concrete value in-process (zero cost), or deserializes + /// the carried blob into `T` on first use (cached). Remote-capable + /// drivers read their config with this instead of `def`, and their + /// def type must be `Serialize + DeserializeOwned`. + pub fn def_de(&self) -> &T + where + T: serde::de::DeserializeOwned + Send + Sync + 'static, + { + let any = RawDef::as_any(self.raw_def.as_ref()); + if let Some(v) = any.downcast_ref::() { + return v; + } + any.downcast_ref::() + .expect("TargetDef raw_def: neither the concrete type nor a serialized RawDefBytes") + .materialize::() + } + pub fn set_def(&mut self, def: T) { self.raw_def = Arc::new(def); } @@ -497,6 +572,77 @@ pub mod targetdef { } } + #[cfg(test)] + mod raw_def_tests { + use super::*; + use hmodel::htpkg::PkgBuf; + + #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] + struct MyDef { + name: String, + n: u32, + } + + fn td(raw: Arc) -> TargetDef { + TargetDef { + addr: Addr::new(PkgBuf::from("//x"), "y".to_string(), Default::default()), + labels: vec![], + raw_def: raw, + inputs: vec![], + outputs: vec![], + support_files: vec![], + cache: CacheConfig::off(), + pty: false, + hash: vec![], + transparent: false, + } + } + + #[test] + fn def_de_downcasts_in_process() { + let t = td(Arc::new(MyDef { + name: "a".to_string(), + n: 7, + })); + assert_eq!( + t.def_de::(), + &MyDef { + name: "a".to_string(), + n: 7 + } + ); + } + + #[test] + fn def_de_materializes_serialized_blob() { + let original = MyDef { + name: "a".to_string(), + n: 7, + }; + // Serialize as a RawDef would cross the wire, then wrap as bytes. + let bytes = serde_json::to_vec(&original).expect("serialize"); + let t = td(Arc::new( + RawDefBytes::from_json_slice(&bytes).expect("wrap"), + )); + assert_eq!(t.def_de::(), &original); + } + + #[test] + fn raw_def_bytes_reserializes_to_original() { + let original = MyDef { + name: "a".to_string(), + n: 7, + }; + let bytes = serde_json::to_vec(&original).expect("serialize"); + let t = td(Arc::new( + RawDefBytes::from_json_slice(&bytes).expect("wrap"), + )); + // Re-serializing the (round-tripped) raw_def yields the original. + let reser = serde_json::to_value(&*t.raw_def).expect("reserialize"); + assert_eq!(reser, serde_json::json!({"name": "a", "n": 7})); + } + } + #[derive(Clone, Hash, Serialize)] pub struct Input { pub r#ref: TargetAddr, From 0bc688f656eac8ce780dc9fe357b986ba2045609 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 19:24:07 +0200 Subject: [PATCH 07/44] feat(plugin-remote): RemoteDriver parse/apply_transitive + guest driver serving [M2] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - plugin-abi: driver-path conversions (TargetDef/Input/Output/Path/ CacheConfig + raw_def via the RawDefBytes contract). - plugin-remote: RemoteDriver impls hplugin::Driver — config/schema/ parse/apply_transitive over the mux; run/run_shell deferred (need the driver-support ManagedDriver sandbox materialization). - plugin-sdk: serve generalized to provider and/or driver (serve_driver/serve_plugin); guest dispatch for ParseReq/ ApplyTransitiveReq. - e2e: a TestDriver's TargetDef (incl. opaque raw_def) round-trips through parse + apply_transitive; def_de reconstructs the concrete config on both host and guest. Green; clippy -D warnings clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + crates/plugin-abi/src/convert.rs | 146 ++++++++++++++++++++++++ crates/plugin-remote/Cargo.toml | 2 + crates/plugin-remote/src/driver.rs | 117 +++++++++++++++++++ crates/plugin-remote/src/lib.rs | 2 + crates/plugin-remote/tests/proto_e2e.rs | 136 ++++++++++++++++++++++ crates/plugin-sdk/src/lib.rs | 2 +- crates/plugin-sdk/src/serve.rs | 131 +++++++++++++++++++-- 8 files changed, 527 insertions(+), 10 deletions(-) create mode 100644 crates/plugin-remote/src/driver.rs diff --git a/Cargo.lock b/Cargo.lock index a8c834f7..d481668e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3693,6 +3693,7 @@ dependencies = [ "plugin-abi", "plugin-sdk", "prost 0.14.4", + "serde", "tokio", "tracing", "walk", diff --git a/crates/plugin-abi/src/convert.rs b/crates/plugin-abi/src/convert.rs index 932d4f6e..4e685e75 100644 --- a/crates/plugin-abi/src/convert.rs +++ b/crates/plugin-abi/src/convert.rs @@ -286,6 +286,152 @@ pub fn matcher_from_pb(m: pb::Matcher) -> Matcher { } } +// ---- TargetDef and its parts (driver path) ---- + +use hplugin::driver::targetdef::path::{CodegenMode, Content as PathContent, Path}; +use hplugin::driver::targetdef::{CacheConfig, Input, InputMode, Output, TargetDef}; + +fn input_mode_to_pb(m: &InputMode) -> pb::InputMode { + match m { + InputMode::Standard => pb::InputMode::Standard, + InputMode::Link => pb::InputMode::Link, + InputMode::Tool => pb::InputMode::Tool, + } +} + +fn input_mode_from_pb(m: i32) -> InputMode { + match pb::InputMode::try_from(m).unwrap_or(pb::InputMode::Standard) { + pb::InputMode::Link => InputMode::Link, + pb::InputMode::Tool => InputMode::Tool, + _ => InputMode::Standard, + } +} + +fn input_to_pb(i: &Input) -> pb::Input { + pb::Input { + r#ref: Some(target_addr_to_pb(&i.r#ref)), + mode: input_mode_to_pb(&i.mode) as i32, + origin_id: i.origin_id.clone(), + annotations: i.annotations.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + hashed: i.hashed, + runtime: i.runtime, + } +} + +fn input_from_pb(i: pb::Input) -> Input { + Input { + r#ref: target_addr_from_pb(i.r#ref.unwrap_or_default()), + mode: input_mode_from_pb(i.mode), + origin_id: i.origin_id, + annotations: i.annotations.into_iter().collect(), + hashed: i.hashed, + runtime: i.runtime, + } +} + +fn codegen_to_pb(c: &CodegenMode) -> pb::CodegenMode { + match c { + CodegenMode::None => pb::CodegenMode::None, + CodegenMode::Copy => pb::CodegenMode::Copy, + CodegenMode::InPlace => pb::CodegenMode::InPlace, + } +} + +fn codegen_from_pb(c: i32) -> CodegenMode { + match pb::CodegenMode::try_from(c).unwrap_or(pb::CodegenMode::None) { + pb::CodegenMode::Copy => CodegenMode::Copy, + pb::CodegenMode::InPlace => CodegenMode::InPlace, + _ => CodegenMode::None, + } +} + +fn path_to_pb(p: &Path) -> pb::Path { + let content = match &p.content { + PathContent::FilePath(s) => pb::path::Content::FilePath(s.clone()), + PathContent::DirPath(s) => pb::path::Content::DirPath(s.clone()), + PathContent::Glob(s) => pb::path::Content::Glob(s.clone()), + }; + pb::Path { + content: Some(content), + codegen_tree: codegen_to_pb(&p.codegen_tree) as i32, + collect: p.collect, + } +} + +fn path_from_pb(p: pb::Path) -> Path { + let content = match p.content { + Some(pb::path::Content::FilePath(s)) => PathContent::FilePath(s), + Some(pb::path::Content::DirPath(s)) => PathContent::DirPath(s), + Some(pb::path::Content::Glob(s)) => PathContent::Glob(s), + None => PathContent::FilePath(String::new()), + }; + Path { + content, + codegen_tree: codegen_from_pb(p.codegen_tree), + collect: p.collect, + } +} + +fn output_to_pb(o: &Output) -> pb::Output { + pb::Output { + group: o.group.clone(), + paths: o.paths.iter().map(path_to_pb).collect(), + } +} + +fn output_from_pb(o: pb::Output) -> Output { + Output { + group: o.group, + paths: o.paths.into_iter().map(path_from_pb).collect(), + } +} + +fn cache_config_to_pb(c: &CacheConfig) -> pb::CacheConfig { + pb::CacheConfig { + enabled: c.enabled, + remote_enabled: c.remote_enabled, + history: c.history, + } +} + +fn cache_config_from_pb(c: pb::CacheConfig) -> CacheConfig { + CacheConfig { + enabled: c.enabled, + remote_enabled: c.remote_enabled, + history: c.history, + } +} + +pub fn target_def_to_pb(td: &TargetDef) -> anyhow::Result { + Ok(pb::TargetDef { + addr: Some(addr_to_pb(&td.addr)), + labels: td.labels.clone(), + raw_def: Some(raw_def_to_blob(&td.raw_def)?), + inputs: td.inputs.iter().map(input_to_pb).collect(), + outputs: td.outputs.iter().map(output_to_pb).collect(), + support_files: td.support_files.iter().map(path_to_pb).collect(), + cache: Some(cache_config_to_pb(&td.cache)), + pty: td.pty, + hash: td.hash.clone().into(), + transparent: td.transparent, + }) +} + +pub fn target_def_from_pb(td: pb::TargetDef) -> anyhow::Result { + Ok(TargetDef { + addr: addr_from_pb(td.addr.unwrap_or_default()), + labels: td.labels, + raw_def: raw_def_from_blob(&td.raw_def.unwrap_or_default())?, + inputs: td.inputs.into_iter().map(input_from_pb).collect(), + outputs: td.outputs.into_iter().map(output_from_pb).collect(), + support_files: td.support_files.into_iter().map(path_from_pb).collect(), + cache: cache_config_from_pb(td.cache.unwrap_or_default()), + pty: td.pty, + hash: td.hash.to_vec(), + transparent: td.transparent, + }) +} + // ---- raw_def (opaque driver blob) ---- /// Serialize a driver's `raw_def` to a wire blob (JSON). Works on any `RawDef`, diff --git a/crates/plugin-remote/Cargo.toml b/crates/plugin-remote/Cargo.toml index a2a3ae4f..85f24118 100644 --- a/crates/plugin-remote/Cargo.toml +++ b/crates/plugin-remote/Cargo.toml @@ -27,4 +27,6 @@ wasm = [] [dev-dependencies] plugin-sdk = { path = "../plugin-sdk" } +async-trait = "0.1.89" +serde = { version = "1", features = ["derive"] } tokio = { version = "1.52", features = ["rt-multi-thread", "macros", "net", "time"] } diff --git a/crates/plugin-remote/src/driver.rs b/crates/plugin-remote/src/driver.rs new file mode 100644 index 00000000..950dee37 --- /dev/null +++ b/crates/plugin-remote/src/driver.rs @@ -0,0 +1,117 @@ +//! `RemoteDriver`: implements [`hplugin::driver::Driver`] by forwarding to an +//! out-of-process plugin over a [`Mux`]. +//! +//! `parse`/`apply_transitive` (the target-def path) are wired; they round-trip +//! `TargetDef` including the opaque `raw_def` via the serialization contract +//! (`RawDefBytes` + `def_de`). `run`/`run_shell` are deferred: they need the +//! `driver-support` `ManagedDriver` sandbox-materialization integration (inputs +//! written into the shared sandbox dir, outputs collected as paths) — the next +//! slice. + +use crate::host::{HostCallbackHandler, HostInner}; +use async_trait::async_trait; +use hcore::hasync::Cancellable; +use hplugin::driver::{ + ApplyTransitiveRequest, ApplyTransitiveResponse, ConfigRequest, ConfigResponse, Driver, + DriverSchema, ParseRequest, ParseResponse, RunRequest, RunResponse, +}; +use plugin_abi::convert; +use plugin_abi::mux::{Body, Mux}; +use plugin_abi::pb; +use std::sync::Arc; +use tokio::io::{AsyncRead, AsyncWrite}; + +pub struct RemoteDriver { + name: String, + mux: Arc, + _inner: Arc, +} + +impl RemoteDriver { + /// Connect over an established duplex byte stream. `name` is the driver name + /// (from the handshake) returned by `config`. + pub fn connect(read: R, write: W, name: impl Into) -> Self + where + R: AsyncRead + Unpin + Send + 'static, + W: AsyncWrite + Unpin + Send + 'static, + { + let inner = Arc::new(HostInner::default()); + let handler = Arc::new(HostCallbackHandler { + inner: Arc::clone(&inner), + }); + let mux = Mux::start(read, write, handler); + Self { + name: name.into(), + mux, + _inner: inner, + } + } +} + +#[async_trait] +impl Driver for RemoteDriver { + fn config(&self, _req: ConfigRequest) -> anyhow::Result { + Ok(ConfigResponse { + name: self.name.clone(), + }) + } + + fn schema(&self) -> DriverSchema { + // Driver config schema is for BUILD-file tooling; not surfaced remotely + // yet. A config-less default is correct for execution paths. + DriverSchema::default() + } + + async fn parse( + &self, + req: ParseRequest, + ctoken: &(dyn Cancellable + Send + Sync), + ) -> anyhow::Result { + let body = Body::ParseReq(pb::ParseRequest { + request_id: req.request_id, + target_spec: Some(convert::target_spec_to_pb(req.target_spec.as_ref())), + }); + match self.mux.call_cancellable(body, ctoken.cancelled()).await? { + Body::ParseResp(pr) => Ok(ParseResponse { + target_def: convert::target_def_from_pb(pr.target_def.unwrap_or_default())?, + }), + other => anyhow::bail!("unexpected parse response: {other:?}"), + } + } + + async fn apply_transitive( + &self, + req: ApplyTransitiveRequest, + ctoken: &(dyn Cancellable + Send + Sync), + ) -> anyhow::Result { + let body = Body::ApplyTransitiveReq(pb::ApplyTransitiveRequest { + request_id: req.request_id, + target_def: Some(convert::target_def_to_pb(&req.target_def)?), + sandbox: Some(convert::sandbox_to_pb(&req.sandbox)), + }); + match self.mux.call_cancellable(body, ctoken.cancelled()).await? { + Body::ApplyTransitiveResp(r) => Ok(ApplyTransitiveResponse { + target_def: convert::target_def_from_pb(r.target_def.unwrap_or_default())?, + }), + other => anyhow::bail!("unexpected apply_transitive response: {other:?}"), + } + } + + async fn run<'a, 'io>( + &self, + _req: RunRequest<'a, 'io>, + _ctoken: &(dyn Cancellable + Send + Sync), + ) -> anyhow::Result { + anyhow::bail!( + "remote driver run() not yet implemented — needs ManagedDriver sandbox materialization" + ) + } + + async fn run_shell<'a, 'io>( + &self, + _req: RunRequest<'a, 'io>, + _ctoken: &(dyn Cancellable + Send + Sync), + ) -> anyhow::Result { + anyhow::bail!("remote driver run_shell() not yet implemented") + } +} diff --git a/crates/plugin-remote/src/lib.rs b/crates/plugin-remote/src/lib.rs index a59d1fcc..ea0eacd3 100644 --- a/crates/plugin-remote/src/lib.rs +++ b/crates/plugin-remote/src/lib.rs @@ -8,11 +8,13 @@ pub mod lease; +mod driver; mod host; mod provider; #[cfg(unix)] mod spawn; +pub use driver::RemoteDriver; pub use provider::RemoteProvider; #[cfg(unix)] pub use spawn::{spawn_plugin, PLUGIN_FD}; diff --git a/crates/plugin-remote/tests/proto_e2e.rs b/crates/plugin-remote/tests/proto_e2e.rs index cbb2213b..e79f810b 100644 --- a/crates/plugin-remote/tests/proto_e2e.rs +++ b/crates/plugin-remote/tests/proto_e2e.rs @@ -278,3 +278,139 @@ async fn get_cancellation_returns_error() { }; assert!(msg.contains("cancelled"), "unexpected error: {msg}"); } + +// ---- driver path: parse + apply_transitive round-trip raw_def via def_de ---- + +#[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] +struct MyCfg { + msg: String, + n: u32, +} + +struct TestDriver; + +#[async_trait::async_trait] +impl hplugin::driver::Driver for TestDriver { + fn config( + &self, + _req: hplugin::driver::ConfigRequest, + ) -> anyhow::Result { + Ok(hplugin::driver::ConfigResponse { + name: "td".to_string(), + }) + } + + fn schema(&self) -> hplugin::driver::DriverSchema { + hplugin::driver::DriverSchema::default() + } + + async fn parse( + &self, + req: hplugin::driver::ParseRequest, + _ctoken: &(dyn hcore::hasync::Cancellable + Send + Sync), + ) -> anyhow::Result { + let target_def = hplugin::driver::targetdef::TargetDef { + addr: req.target_spec.addr.clone(), + labels: vec![], + raw_def: Arc::new(MyCfg { + msg: "hi".to_string(), + n: 9, + }), + inputs: vec![], + outputs: vec![], + support_files: vec![], + cache: hplugin::driver::targetdef::CacheConfig::off(), + pty: false, + hash: vec![], + transparent: false, + }; + Ok(hplugin::driver::ParseResponse { target_def }) + } + + async fn apply_transitive( + &self, + req: hplugin::driver::ApplyTransitiveRequest, + _ctoken: &(dyn hcore::hasync::Cancellable + Send + Sync), + ) -> anyhow::Result { + // Guest reads the wire raw_def back as the concrete type via def_de. + let cfg = req.target_def.def_de::(); + anyhow::ensure!(cfg.msg == "hi" && cfg.n == 9, "raw_def lost in transit"); + Ok(hplugin::driver::ApplyTransitiveResponse { + target_def: req.target_def, + }) + } + + async fn run<'a, 'io>( + &self, + _req: hplugin::driver::RunRequest<'a, 'io>, + _ctoken: &(dyn hcore::hasync::Cancellable + Send + Sync), + ) -> anyhow::Result { + anyhow::bail!("test driver does not run") + } + + async fn run_shell<'a, 'io>( + &self, + _req: hplugin::driver::RunRequest<'a, 'io>, + _ctoken: &(dyn hcore::hasync::Cancellable + Send + Sync), + ) -> anyhow::Result { + anyhow::bail!("test driver does not run") + } +} + +#[tokio::test] +async fn driver_parse_and_apply_transitive_roundtrip() { + let (a, b) = UnixStream::pair().expect("socketpair"); + let (ar, aw) = a.into_split(); + let (br, bw) = b.into_split(); + let host = plugin_remote::RemoteDriver::connect(ar, aw, "td"); + let _guest = plugin_sdk::serve_driver(Arc::new(TestDriver), br, bw); + + use hplugin::driver::Driver; + assert_eq!( + host.config(hplugin::driver::ConfigRequest {}) + .expect("config") + .name, + "td" + ); + + let ctoken = StdCancellationToken::new(); + + // parse: TargetDef (incl. opaque raw_def) crosses back; def_de reconstructs. + let spec = Arc::new(hplugin::provider::TargetSpec { + addr: addr("//x", "y"), + driver: "td".to_string(), + ..Default::default() + }); + let parsed = host + .parse( + hplugin::driver::ParseRequest { + request_id: "r1".to_string(), + target_spec: spec, + }, + &ctoken, + ) + .await + .expect("parse"); + assert_eq!( + parsed.target_def.def_de::(), + &MyCfg { + msg: "hi".to_string(), + n: 9 + } + ); + + // apply_transitive: ship the round-tripped TargetDef back; the guest reads + // its raw_def via def_de (proving the contract works guest-side too). + let applied = host + .apply_transitive( + hplugin::driver::ApplyTransitiveRequest { + request_id: "r2".to_string(), + target_def: parsed.target_def, + sandbox: hplugin::driver::sandbox::Sandbox::default(), + }, + &ctoken, + ) + .await + .expect("apply_transitive"); + assert_eq!(applied.target_def.def_de::().n, 9); +} diff --git a/crates/plugin-sdk/src/lib.rs b/crates/plugin-sdk/src/lib.rs index a9b3e2a1..0a3aef0d 100644 --- a/crates/plugin-sdk/src/lib.rs +++ b/crates/plugin-sdk/src/lib.rs @@ -19,7 +19,7 @@ pub mod serve; pub use ctx::Ctx; pub use host::HostClient; -pub use serve::serve; +pub use serve::{serve, serve_driver, serve_plugin}; #[cfg(unix)] pub use serve::serve_inherited; diff --git a/crates/plugin-sdk/src/serve.rs b/crates/plugin-sdk/src/serve.rs index 003b342b..b185bd29 100644 --- a/crates/plugin-sdk/src/serve.rs +++ b/crates/plugin-sdk/src/serve.rs @@ -8,6 +8,9 @@ use futures::future::BoxFuture; use hcore::hartifactcontent::{Content, WalkEntry}; use hcore::hasync::StdCancellationToken; use hplugin::eresult::{ArtifactMeta, EResult}; +use hplugin::driver::{ + ApplyTransitiveRequest, ConfigRequest as DriverConfigRequest, Driver, ParseRequest, +}; use hplugin::provider::{ ConfigRequest, GetError, GetRequest, ListPackagesRequest, ListRequest, ProbeRequest, Provider, ProviderExecutor, @@ -27,12 +30,36 @@ use std::sync::{Arc, Mutex}; /// handle; keep it alive for the lifetime of the connection (the read/write /// tasks run in the background and end on EOF). pub fn serve(provider: Arc, read: R, write: W) -> Arc +where + R: tokio::io::AsyncRead + Unpin + Send + 'static, + W: tokio::io::AsyncWrite + Unpin + Send + 'static, +{ + serve_plugin(Some(provider), None, read, write) +} + +/// Serve a driver-only plugin. +pub fn serve_driver(driver: Arc, read: R, write: W) -> Arc +where + R: tokio::io::AsyncRead + Unpin + Send + 'static, + W: tokio::io::AsyncWrite + Unpin + Send + 'static, +{ + serve_plugin(None, Some(driver), read, write) +} + +/// Serve a plugin that may export a provider, a driver, or both. +pub fn serve_plugin( + provider: Option>, + driver: Option>, + read: R, + write: W, +) -> Arc where R: tokio::io::AsyncRead + Unpin + Send + 'static, W: tokio::io::AsyncWrite + Unpin + Send + 'static, { let handler = Arc::new(GuestHandler { provider, + driver, tokens: Mutex::new(HashMap::new()), }); Mux::start(read, write, handler) @@ -55,7 +82,8 @@ pub async fn serve_inherited(provider: Arc) -> anyhow::Result<()> } struct GuestHandler { - provider: Arc, + provider: Option>, + driver: Option>, // frame-id -> cancellation token for the in-flight method call tokens: Mutex>>, } @@ -82,27 +110,44 @@ fn err_frame(message: String) -> Body { impl InboundHandler for GuestHandler { async fn handle(&self, id: u64, body: Body, mux: Arc) { match body { - Body::ConfigReq(_) => match self.provider.config(ConfigRequest {}) { - Ok(resp) => mux.send_body(id, Body::ConfigResp(pb::ConfigResponse { name: resp.name })), - Err(e) => mux.send_body(id, err_frame(e.to_string())), - }, + Body::ConfigReq(_) => { + let name = if let Some(p) = &self.provider { + p.config(ConfigRequest {}).map(|r| r.name) + } else if let Some(d) = &self.driver { + d.config(DriverConfigRequest {}).map(|r| r.name) + } else { + Err(anyhow::anyhow!("plugin exports neither provider nor driver")) + }; + match name { + Ok(name) => mux.send_body(id, Body::ConfigResp(pb::ConfigResponse { name })), + Err(e) => mux.send_body(id, err_frame(e.to_string())), + } + } Body::ListReq(req) => { + let Some(provider) = self.provider.clone() else { + mux.send_body(id, stream_err("plugin has no provider".to_string())); + return; + }; let tok = self.new_token(id); let lreq = ListRequest { request_id: req.request_id, package: PkgBuf::from(req.package), states: req.states.into_iter().map(convert::state_from_pb).collect(), }; - let res = self.provider.list(lreq, &*tok).await; + let res = provider.list(lreq, &*tok).await; self.stream_addrs(id, res, &mux); self.drop_token(id); } Body::ListPackagesReq(req) => { + let Some(provider) = self.provider.clone() else { + mux.send_body(id, stream_err("plugin has no provider".to_string())); + return; + }; let tok = self.new_token(id); let lreq = ListPackagesRequest { prefix: PkgBuf::from(req.prefix), }; - let res = self.provider.list_packages(lreq, &*tok).await; + let res = provider.list_packages(lreq, &*tok).await; match res { Ok(iter) => { for item in iter { @@ -130,6 +175,10 @@ impl InboundHandler for GuestHandler { self.drop_token(id); } Body::GetReq(req) => { + let Some(provider) = self.provider.clone() else { + mux.send_body(id, err_frame("plugin has no provider".to_string())); + return; + }; let tok = self.new_token(id); let executor: Arc = Arc::new(MuxExecutor { mux: Arc::clone(&mux), @@ -141,7 +190,7 @@ impl InboundHandler for GuestHandler { states: req.states.into_iter().map(convert::state_from_pb).collect(), executor, }; - match self.provider.get(greq, &*tok).await { + match provider.get(greq, &*tok).await { Ok(gr) => mux.send_body( id, Body::GetResp(pb::GetResponse { @@ -166,12 +215,16 @@ impl InboundHandler for GuestHandler { self.drop_token(id); } Body::ProbeReq(req) => { + let Some(provider) = self.provider.clone() else { + mux.send_body(id, err_frame("plugin has no provider".to_string())); + return; + }; let tok = self.new_token(id); let preq = ProbeRequest { request_id: req.request_id, package: PkgBuf::from(req.package), }; - match self.provider.probe(preq, &*tok).await { + match provider.probe(preq, &*tok).await { Ok(pr) => mux.send_body( id, Body::ProbeResp(pb::ProbeResponse { @@ -182,6 +235,66 @@ impl InboundHandler for GuestHandler { } self.drop_token(id); } + Body::ParseReq(req) => { + let Some(driver) = self.driver.clone() else { + mux.send_body(id, err_frame("plugin has no driver".to_string())); + return; + }; + let tok = self.new_token(id); + let preq = ParseRequest { + request_id: req.request_id, + target_spec: Arc::new(convert::target_spec_from_pb( + req.target_spec.unwrap_or_default(), + )), + }; + match driver.parse(preq, &*tok).await { + Ok(resp) => match convert::target_def_to_pb(&resp.target_def) { + Ok(td) => mux.send_body( + id, + Body::ParseResp(pb::ParseResponse { + target_def: Some(td), + }), + ), + Err(e) => mux.send_body(id, err_frame(e.to_string())), + }, + Err(e) => mux.send_body(id, err_frame(e.to_string())), + } + self.drop_token(id); + } + Body::ApplyTransitiveReq(req) => { + let Some(driver) = self.driver.clone() else { + mux.send_body(id, err_frame("plugin has no driver".to_string())); + return; + }; + let tok = self.new_token(id); + let target_def = match convert::target_def_from_pb(req.target_def.unwrap_or_default()) + { + Ok(td) => td, + Err(e) => { + mux.send_body(id, err_frame(e.to_string())); + self.drop_token(id); + return; + } + }; + let areq = ApplyTransitiveRequest { + request_id: req.request_id, + target_def, + sandbox: convert::sandbox_from_pb(req.sandbox.unwrap_or_default()), + }; + match driver.apply_transitive(areq, &*tok).await { + Ok(resp) => match convert::target_def_to_pb(&resp.target_def) { + Ok(td) => mux.send_body( + id, + Body::ApplyTransitiveResp(pb::ApplyTransitiveResponse { + target_def: Some(td), + }), + ), + Err(e) => mux.send_body(id, err_frame(e.to_string())), + }, + Err(e) => mux.send_body(id, err_frame(e.to_string())), + } + self.drop_token(id); + } Body::Cancel(c) => { if let Some(tok) = self.tokens.lock().expect("tokens").get(&c.request_id) { tok.cancel(); From 995ad2f548de5c788125806e8c469c07e7b494cf Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 20:07:01 +0200 Subject: [PATCH 08/44] feat(plugin-remote): remote target execution via RemoteManagedDriver [M2] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run() now works over the wire, plugged in at the driver-support ManagedDriver layer so the host reuses all existing sandbox setup + input materialization: - proto: ManagedRunRequest/Input/Response (paths + metadata only — inputs are on the shared filesystem, never streamed) + envelope frames. - plugin-abi: OutputArtifact <-> OutputArtifactRef conversions. - plugin-remote: RemoteManagedDriver impls ManagedDriver; the host ManagedDriverBridge materializes the sandbox, then forwards the run to the out-of-process plugin; outputs come back as paths. - plugin-sdk: serve_managed_driver + guest reconstruction of the ManagedRunRequest (NullContent for on-disk inputs) -> executes the plugin's ManagedDriver. - fix: add ManagedRunResp to mux is_response() (it was misrouted as an inbound request, deadlocking the call). - e2e: guest writes an output file into the host-prepared sandbox; host gets the artifact back and the file exists on disk. 6 proto tests + subprocess test green; clippy -D warnings clean. Stdio proxying for the run streams is the remaining run() follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 3 + crates/plugin-abi/src/convert.rs | 73 ++++++++++ crates/plugin-abi/src/mux.rs | 1 + crates/plugin-remote/Cargo.toml | 2 + crates/plugin-remote/src/lib.rs | 2 + crates/plugin-remote/src/managed.rs | 176 ++++++++++++++++++++++++ crates/plugin-remote/tests/proto_e2e.rs | 114 +++++++++++++++ crates/plugin-sdk/Cargo.toml | 1 + crates/plugin-sdk/src/lib.rs | 2 +- crates/plugin-sdk/src/serve.rs | 161 ++++++++++++++++++++-- proto/plugin/v1/driver.proto | 40 ++++++ proto/plugin/v1/envelope.proto | 2 + 12 files changed, 565 insertions(+), 12 deletions(-) create mode 100644 crates/plugin-remote/src/managed.rs diff --git a/Cargo.lock b/Cargo.lock index d481668e..029c5d47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3686,6 +3686,7 @@ dependencies = [ "anyhow", "async-trait", "core", + "driver-support", "futures", "libc", "model", @@ -3694,6 +3695,7 @@ dependencies = [ "plugin-sdk", "prost 0.14.4", "serde", + "tempfile", "tokio", "tracing", "walk", @@ -3706,6 +3708,7 @@ dependencies = [ "anyhow", "async-trait", "core", + "driver-support", "futures", "model", "plugin", diff --git a/crates/plugin-abi/src/convert.rs b/crates/plugin-abi/src/convert.rs index 4e685e75..0eee763f 100644 --- a/crates/plugin-abi/src/convert.rs +++ b/crates/plugin-abi/src/convert.rs @@ -432,6 +432,79 @@ pub fn target_def_from_pb(td: pb::TargetDef) -> anyhow::Result { }) } +// ---- OutputArtifact (driver run outputs) ---- + +use hplugin::driver::outputartifact::{Content as OaContent, ContentFile, ContentRaw, OutputArtifact, Type as OaType}; + +fn oa_type_to_pb(t: &OaType) -> pb::ArtifactType { + match t { + OaType::Output => pb::ArtifactType::Output, + OaType::Log => pb::ArtifactType::Log, + OaType::SupportFile => pb::ArtifactType::SupportFile, + } +} + +fn oa_type_from_pb(t: i32) -> OaType { + match pb::ArtifactType::try_from(t).unwrap_or(pb::ArtifactType::Output) { + pb::ArtifactType::Log => OaType::Log, + pb::ArtifactType::SupportFile => OaType::SupportFile, + _ => OaType::Output, + } +} + +pub fn output_artifact_to_pb(oa: &OutputArtifact) -> pb::OutputArtifactRef { + let content = match &oa.content { + OaContent::File(f) => pb::output_artifact_ref::Content::File(pb::ContentFile { + source_path: f.source_path.clone(), + out_path: f.out_path.clone(), + x: f.x, + }), + OaContent::Raw(r) => pb::output_artifact_ref::Content::Raw(pb::ContentRaw { + data: r.data.clone().into(), + path: r.path.clone(), + x: r.x, + }), + OaContent::TarPath(p) => pb::output_artifact_ref::Content::TarPath(p.clone()), + OaContent::CpioPath(p) => pb::output_artifact_ref::Content::CpioPath(p.clone()), + }; + pb::OutputArtifactRef { + group: oa.group.clone(), + name: oa.name.clone(), + r#type: oa_type_to_pb(&oa.r#type) as i32, + content: Some(content), + hashout: oa.hashout.clone(), + } +} + +pub fn output_artifact_from_pb(oa: pb::OutputArtifactRef) -> OutputArtifact { + let content = match oa.content { + Some(pb::output_artifact_ref::Content::File(f)) => OaContent::File(ContentFile { + source_path: f.source_path, + out_path: f.out_path, + x: f.x, + }), + Some(pb::output_artifact_ref::Content::Raw(r)) => OaContent::Raw(ContentRaw { + data: r.data.to_vec(), + path: r.path, + x: r.x, + }), + Some(pb::output_artifact_ref::Content::TarPath(p)) => OaContent::TarPath(p), + Some(pb::output_artifact_ref::Content::CpioPath(p)) => OaContent::CpioPath(p), + None => OaContent::Raw(ContentRaw { + data: vec![], + path: String::new(), + x: false, + }), + }; + OutputArtifact { + group: oa.group, + name: oa.name, + r#type: oa_type_from_pb(oa.r#type), + content, + hashout: oa.hashout, + } +} + // ---- raw_def (opaque driver blob) ---- /// Serialize a driver's `raw_def` to a wire blob (JSON). Works on any `RawDef`, diff --git a/crates/plugin-abi/src/mux.rs b/crates/plugin-abi/src/mux.rs index 6f264584..6cf6da0e 100644 --- a/crates/plugin-abi/src/mux.rs +++ b/crates/plugin-abi/src/mux.rs @@ -49,6 +49,7 @@ fn is_response(body: &Body) -> bool { | Body::ParseResp(_) | Body::ApplyTransitiveResp(_) | Body::RunResp(_) + | Body::ManagedRunResp(_) | Body::ResultResp(_) | Body::NoteDepResp(_) | Body::QueryResp(_) diff --git a/crates/plugin-remote/Cargo.toml b/crates/plugin-remote/Cargo.toml index 85f24118..7230a0ae 100644 --- a/crates/plugin-remote/Cargo.toml +++ b/crates/plugin-remote/Cargo.toml @@ -11,6 +11,7 @@ hplugin = { package = "plugin", path = "../plugin" } hmodel = { package = "model", path = "../model" } hcore = { package = "core", path = "../core" } hwalk = { package = "walk", path = "../walk" } +hdriver-support = { package = "driver-support", path = "../driver-support" } plugin-abi = { path = "../plugin-abi", features = ["convert", "transport"] } anyhow = "1.0.102" async-trait = "0.1.89" @@ -29,4 +30,5 @@ wasm = [] plugin-sdk = { path = "../plugin-sdk" } async-trait = "0.1.89" serde = { version = "1", features = ["derive"] } +tempfile = "3" tokio = { version = "1.52", features = ["rt-multi-thread", "macros", "net", "time"] } diff --git a/crates/plugin-remote/src/lib.rs b/crates/plugin-remote/src/lib.rs index ea0eacd3..c5c57d36 100644 --- a/crates/plugin-remote/src/lib.rs +++ b/crates/plugin-remote/src/lib.rs @@ -10,11 +10,13 @@ pub mod lease; mod driver; mod host; +mod managed; mod provider; #[cfg(unix)] mod spawn; pub use driver::RemoteDriver; +pub use managed::RemoteManagedDriver; pub use provider::RemoteProvider; #[cfg(unix)] pub use spawn::{spawn_plugin, PLUGIN_FD}; diff --git a/crates/plugin-remote/src/managed.rs b/crates/plugin-remote/src/managed.rs new file mode 100644 index 00000000..172910c2 --- /dev/null +++ b/crates/plugin-remote/src/managed.rs @@ -0,0 +1,176 @@ +//! `RemoteManagedDriver`: the host side of remote target execution. +//! +//! It implements driver-support's `ManagedDriver`, so the host's +//! `ManagedDriverBridge`/`ManagedDriverFuse` performs all sandbox setup + +//! input materialization (as for in-process managed drivers) and then calls +//! `run` here. We forward the already-materialized run — only paths + metadata, +//! never input bytes (the guest shares the filesystem) — to the out-of-process +//! plugin and return its output artifacts. +//! +//! Stdio is not yet proxied (build drivers like go's don't use the run streams); +//! streaming the target's stdin/out/err is a follow-up. + +use crate::host::{HostCallbackHandler, HostInner}; +use async_trait::async_trait; +use hcore::hasync::Cancellable; +use hdriver_support::driver_managed::{ + ManagedDriver, ManagedRunInput, ManagedRunRequest, ManagedRunResponse, +}; +use hplugin::driver::inputartifact; +use hplugin::driver::{ + ApplyTransitiveRequest, ApplyTransitiveResponse, ConfigRequest, ConfigResponse, DriverSchema, + ParseRequest, ParseResponse, +}; +use plugin_abi::convert; +use plugin_abi::mux::{Body, Mux}; +use plugin_abi::pb; +use std::sync::Arc; +use tokio::io::{AsyncRead, AsyncWrite}; + +pub struct RemoteManagedDriver { + name: String, + mux: Arc, + _inner: Arc, +} + +impl RemoteManagedDriver { + pub fn connect(read: R, write: W, name: impl Into) -> Self + where + R: AsyncRead + Unpin + Send + 'static, + W: AsyncWrite + Unpin + Send + 'static, + { + let inner = Arc::new(HostInner::default()); + let handler = Arc::new(HostCallbackHandler { + inner: Arc::clone(&inner), + }); + let mux = Mux::start(read, write, handler); + Self { + name: name.into(), + mux, + _inner: inner, + } + } +} + +fn managed_input_to_pb(mi: &ManagedRunInput) -> pb::ManagedRunInput { + let ty = match mi.input.artifact.r#type { + inputartifact::Type::Dep => pb::InputArtifactType::Dep, + inputartifact::Type::Support => pb::InputArtifactType::Support, + }; + pb::ManagedRunInput { + r#type: ty as i32, + origin_id: mi.input.origin_id.clone(), + source_addr: Some(convert::addr_to_pb(&mi.input.source_addr)), + filters: mi.input.filters.clone(), + annotations: mi + .input + .annotations + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + unpack_root: mi.unpack_root.to_string_lossy().into_owned(), + list_path: mi + .list_path + .as_ref() + .map(|p| p.to_string_lossy().into_owned()), + } +} + +#[async_trait] +impl ManagedDriver for RemoteManagedDriver { + fn config(&self, _req: ConfigRequest) -> anyhow::Result { + Ok(ConfigResponse { + name: self.name.clone(), + }) + } + + fn schema(&self) -> DriverSchema { + DriverSchema::default() + } + + async fn parse( + &self, + req: ParseRequest, + ctoken: &(dyn Cancellable + Send + Sync), + ) -> anyhow::Result { + let body = Body::ParseReq(pb::ParseRequest { + request_id: req.request_id, + target_spec: Some(convert::target_spec_to_pb(req.target_spec.as_ref())), + }); + match self.mux.call_cancellable(body, ctoken.cancelled()).await? { + Body::ParseResp(pr) => Ok(ParseResponse { + target_def: convert::target_def_from_pb(pr.target_def.unwrap_or_default())?, + }), + other => anyhow::bail!("unexpected parse response: {other:?}"), + } + } + + async fn apply_transitive( + &self, + req: ApplyTransitiveRequest, + ctoken: &(dyn Cancellable + Send + Sync), + ) -> anyhow::Result { + let body = Body::ApplyTransitiveReq(pb::ApplyTransitiveRequest { + request_id: req.request_id, + target_def: Some(convert::target_def_to_pb(&req.target_def)?), + sandbox: Some(convert::sandbox_to_pb(&req.sandbox)), + }); + match self.mux.call_cancellable(body, ctoken.cancelled()).await? { + Body::ApplyTransitiveResp(r) => Ok(ApplyTransitiveResponse { + target_def: convert::target_def_from_pb(r.target_def.unwrap_or_default())?, + }), + other => anyhow::bail!("unexpected apply_transitive response: {other:?}"), + } + } + + fn supports_shell(&self) -> bool { + true + } + + async fn run<'a, 'io>( + &self, + req: ManagedRunRequest<'a, 'io>, + ctoken: &(dyn Cancellable + Send + Sync), + ) -> anyhow::Result { + self.dispatch_run(req, ctoken, false).await + } + + async fn run_shell<'a, 'io>( + &self, + req: ManagedRunRequest<'a, 'io>, + ctoken: &(dyn Cancellable + Send + Sync), + ) -> anyhow::Result { + self.dispatch_run(req, ctoken, true).await + } +} + +impl RemoteManagedDriver { + async fn dispatch_run( + &self, + req: ManagedRunRequest<'_, '_>, + ctoken: &(dyn Cancellable + Send + Sync), + shell: bool, + ) -> anyhow::Result { + let body = Body::ManagedRunReq(pb::ManagedRunRequest { + request_id: req.request.request_id.clone(), + target: Some(convert::target_def_to_pb(req.request.target)?), + tree_root_path: req.request.tree_root_path.to_string_lossy().into_owned(), + hashin: req.request.hashin.to_string(), + sandbox_dir: req.sandbox_dir.to_string_lossy().into_owned(), + sandbox_ws_dir: req.sandbox_ws_dir.to_string_lossy().into_owned(), + sandbox_pkg_dir: req.sandbox_pkg_dir.to_string_lossy().into_owned(), + inputs: req.inputs.iter().map(managed_input_to_pb).collect(), + shell, + }); + match self.mux.call_cancellable(body, ctoken.cancelled()).await? { + Body::ManagedRunResp(r) => Ok(ManagedRunResponse { + artifacts: r + .artifacts + .into_iter() + .map(convert::output_artifact_from_pb) + .collect(), + }), + other => anyhow::bail!("unexpected managed run response: {other:?}"), + } + } +} diff --git a/crates/plugin-remote/tests/proto_e2e.rs b/crates/plugin-remote/tests/proto_e2e.rs index e79f810b..50ff712d 100644 --- a/crates/plugin-remote/tests/proto_e2e.rs +++ b/crates/plugin-remote/tests/proto_e2e.rs @@ -414,3 +414,117 @@ async fn driver_parse_and_apply_transitive_roundtrip() { .expect("apply_transitive"); assert_eq!(applied.target_def.def_de::().n, 9); } + +// ---- managed run: host materializes, guest executes in the shared sandbox ---- + +struct TestManagedDriver; + +#[async_trait::async_trait] +impl hdriver_support::driver_managed::ManagedDriver for TestManagedDriver { + fn config( + &self, + _req: hplugin::driver::ConfigRequest, + ) -> anyhow::Result { + Ok(hplugin::driver::ConfigResponse { + name: "tmd".to_string(), + }) + } + + fn schema(&self) -> hplugin::driver::DriverSchema { + hplugin::driver::DriverSchema::default() + } + + async fn parse( + &self, + _req: hplugin::driver::ParseRequest, + _ctoken: &(dyn hcore::hasync::Cancellable + Send + Sync), + ) -> anyhow::Result { + anyhow::bail!("not used") + } + + async fn apply_transitive( + &self, + _req: hplugin::driver::ApplyTransitiveRequest, + _ctoken: &(dyn hcore::hasync::Cancellable + Send + Sync), + ) -> anyhow::Result { + anyhow::bail!("not used") + } + + async fn run<'a, 'io>( + &self, + req: hdriver_support::driver_managed::ManagedRunRequest<'a, 'io>, + _ctoken: &(dyn hcore::hasync::Cancellable + Send + Sync), + ) -> anyhow::Result { + // Execute in the (host-prepared) sandbox: write an output file. + let out = req.sandbox_pkg_dir.join("out.txt"); + std::fs::write(&out, b"built")?; + Ok(hdriver_support::driver_managed::ManagedRunResponse { + artifacts: vec![hplugin::driver::outputartifact::OutputArtifact { + group: "out".to_string(), + name: "out.txt".to_string(), + r#type: hplugin::driver::outputartifact::Type::Output, + content: hplugin::driver::outputartifact::Content::File( + hplugin::driver::outputartifact::ContentFile { + source_path: out.to_string_lossy().into_owned(), + out_path: "out.txt".to_string(), + x: false, + }, + ), + hashout: "h".to_string(), + }], + }) + } +} + +#[tokio::test] +async fn managed_run_executes_remotely() { + let dir = tempfile::tempdir().expect("tempdir"); + let sandbox = dir.path().to_path_buf(); + + let (a, b) = UnixStream::pair().expect("socketpair"); + let (ar, aw) = a.into_split(); + let (br, bw) = b.into_split(); + let host = plugin_remote::RemoteManagedDriver::connect(ar, aw, "tmd"); + let _guest = plugin_sdk::serve_managed_driver(Arc::new(TestManagedDriver), br, bw); + + let ctoken = StdCancellationToken::new(); + let request_id = "r1".to_string(); + let target = hplugin::driver::targetdef::TargetDef { + addr: addr("//x", "y"), + labels: vec![], + raw_def: Arc::new(()), + inputs: vec![], + outputs: vec![], + support_files: vec![], + cache: hplugin::driver::targetdef::CacheConfig::off(), + pty: false, + hash: vec![], + transparent: false, + }; + let hashin = "hash".to_string(); + let rr = hplugin::driver::RunRequest { + request_id: &request_id, + target: &target, + tree_root_path: sandbox.clone(), + inputs: vec![], + hashin: hashin.as_str(), + stdin: None, + stdout: None, + stderr: None, + sandbox_dir: sandbox.clone(), + }; + let mrr = hdriver_support::driver_managed::ManagedRunRequest { + request: rr, + sandbox_dir: sandbox.clone(), + sandbox_ws_dir: sandbox.clone(), + sandbox_pkg_dir: sandbox.clone(), + inputs: vec![], + }; + + use hdriver_support::driver_managed::ManagedDriver; + let resp = host.run(mrr, &ctoken).await.expect("managed run"); + assert_eq!(resp.artifacts.len(), 1); + assert_eq!(resp.artifacts[0].name, "out.txt"); + // The guest actually wrote the file into the shared sandbox. + assert!(sandbox.join("out.txt").exists()); +} diff --git a/crates/plugin-sdk/Cargo.toml b/crates/plugin-sdk/Cargo.toml index 5f4acf5c..2d934799 100644 --- a/crates/plugin-sdk/Cargo.toml +++ b/crates/plugin-sdk/Cargo.toml @@ -10,6 +10,7 @@ workspace = true hplugin = { package = "plugin", path = "../plugin" } hmodel = { package = "model", path = "../model" } hcore = { package = "core", path = "../core" } +hdriver-support = { package = "driver-support", path = "../driver-support" } plugin-abi = { path = "../plugin-abi", features = ["convert", "transport"] } anyhow = "1.0.102" async-trait = "0.1.89" diff --git a/crates/plugin-sdk/src/lib.rs b/crates/plugin-sdk/src/lib.rs index 0a3aef0d..c008a641 100644 --- a/crates/plugin-sdk/src/lib.rs +++ b/crates/plugin-sdk/src/lib.rs @@ -19,7 +19,7 @@ pub mod serve; pub use ctx::Ctx; pub use host::HostClient; -pub use serve::{serve, serve_driver, serve_plugin}; +pub use serve::{serve, serve_driver, serve_managed_driver, serve_plugin}; #[cfg(unix)] pub use serve::serve_inherited; diff --git a/crates/plugin-sdk/src/serve.rs b/crates/plugin-sdk/src/serve.rs index b185bd29..fae486b8 100644 --- a/crates/plugin-sdk/src/serve.rs +++ b/crates/plugin-sdk/src/serve.rs @@ -8,9 +8,12 @@ use futures::future::BoxFuture; use hcore::hartifactcontent::{Content, WalkEntry}; use hcore::hasync::StdCancellationToken; use hplugin::eresult::{ArtifactMeta, EResult}; +use hdriver_support::driver_managed::{ManagedDriver, ManagedRunInput, ManagedRunRequest}; use hplugin::driver::{ - ApplyTransitiveRequest, ConfigRequest as DriverConfigRequest, Driver, ParseRequest, + inputartifact, ApplyTransitiveRequest, ConfigRequest as DriverConfigRequest, Driver, + ParseRequest, RunInput, RunRequest, }; +use std::path::PathBuf; use hplugin::provider::{ ConfigRequest, GetError, GetRequest, ListPackagesRequest, ListRequest, ProbeRequest, Provider, ProviderExecutor, @@ -46,6 +49,23 @@ where serve_plugin(None, Some(driver), read, write) } +/// Serve a managed-driver plugin (target execution: parse/apply_transitive/run +/// at the driver-support `ManagedDriver` layer; the host materializes the +/// sandbox). +pub fn serve_managed_driver(managed: Arc, read: R, write: W) -> Arc +where + R: tokio::io::AsyncRead + Unpin + Send + 'static, + W: tokio::io::AsyncWrite + Unpin + Send + 'static, +{ + let handler = Arc::new(GuestHandler { + provider: None, + driver: None, + managed: Some(managed), + tokens: Mutex::new(HashMap::new()), + }); + Mux::start(read, write, handler) +} + /// Serve a plugin that may export a provider, a driver, or both. pub fn serve_plugin( provider: Option>, @@ -60,6 +80,7 @@ where let handler = Arc::new(GuestHandler { provider, driver, + managed: None, tokens: Mutex::new(HashMap::new()), }); Mux::start(read, write, handler) @@ -84,6 +105,7 @@ pub async fn serve_inherited(provider: Arc) -> anyhow::Result<()> struct GuestHandler { provider: Option>, driver: Option>, + managed: Option>, // frame-id -> cancellation token for the in-flight method call tokens: Mutex>>, } @@ -115,6 +137,8 @@ impl InboundHandler for GuestHandler { p.config(ConfigRequest {}).map(|r| r.name) } else if let Some(d) = &self.driver { d.config(DriverConfigRequest {}).map(|r| r.name) + } else if let Some(m) = &self.managed { + m.config(DriverConfigRequest {}).map(|r| r.name) } else { Err(anyhow::anyhow!("plugin exports neither provider nor driver")) }; @@ -236,10 +260,6 @@ impl InboundHandler for GuestHandler { self.drop_token(id); } Body::ParseReq(req) => { - let Some(driver) = self.driver.clone() else { - mux.send_body(id, err_frame("plugin has no driver".to_string())); - return; - }; let tok = self.new_token(id); let preq = ParseRequest { request_id: req.request_id, @@ -247,7 +267,12 @@ impl InboundHandler for GuestHandler { req.target_spec.unwrap_or_default(), )), }; - match driver.parse(preq, &*tok).await { + let result = match (self.driver.clone(), self.managed.clone()) { + (Some(d), _) => d.parse(preq, &*tok).await, + (None, Some(m)) => m.parse(preq, &*tok).await, + (None, None) => Err(anyhow::anyhow!("plugin has no driver")), + }; + match result { Ok(resp) => match convert::target_def_to_pb(&resp.target_def) { Ok(td) => mux.send_body( id, @@ -262,10 +287,6 @@ impl InboundHandler for GuestHandler { self.drop_token(id); } Body::ApplyTransitiveReq(req) => { - let Some(driver) = self.driver.clone() else { - mux.send_body(id, err_frame("plugin has no driver".to_string())); - return; - }; let tok = self.new_token(id); let target_def = match convert::target_def_from_pb(req.target_def.unwrap_or_default()) { @@ -281,7 +302,12 @@ impl InboundHandler for GuestHandler { target_def, sandbox: convert::sandbox_from_pb(req.sandbox.unwrap_or_default()), }; - match driver.apply_transitive(areq, &*tok).await { + let result = match (self.driver.clone(), self.managed.clone()) { + (Some(d), _) => d.apply_transitive(areq, &*tok).await, + (None, Some(m)) => m.apply_transitive(areq, &*tok).await, + (None, None) => Err(anyhow::anyhow!("plugin has no driver")), + }; + match result { Ok(resp) => match convert::target_def_to_pb(&resp.target_def) { Ok(td) => mux.send_body( id, @@ -295,6 +321,15 @@ impl InboundHandler for GuestHandler { } self.drop_token(id); } + Body::ManagedRunReq(req) => { + let Some(managed) = self.managed.clone() else { + mux.send_body(id, err_frame("plugin has no managed driver".to_string())); + return; + }; + let tok = self.new_token(id); + self.handle_managed_run(id, req, managed, &*tok, &mux).await; + self.drop_token(id); + } Body::Cancel(c) => { if let Some(tok) = self.tokens.lock().expect("tokens").get(&c.request_id) { tok.cancel(); @@ -305,7 +340,111 @@ impl InboundHandler for GuestHandler { } } +fn run_input_from_pb(mi: &pb::ManagedRunInput) -> RunInput { + let ty = match pb::InputArtifactType::try_from(mi.r#type).unwrap_or(pb::InputArtifactType::Dep) { + pb::InputArtifactType::Support => inputartifact::Type::Support, + _ => inputartifact::Type::Dep, + }; + RunInput { + artifact: inputartifact::InputArtifact { + r#type: ty, + origin_id: mi.origin_id.clone(), + // Input bytes are on the shared filesystem (unpack_root); the guest + // reads from disk, never from this Content. + content: Arc::new(NullContent), + }, + origin_id: mi.origin_id.clone(), + source_addr: convert::addr_from_pb(mi.source_addr.clone().unwrap_or_default()), + filters: mi.filters.clone(), + annotations: mi + .annotations + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + } +} + +fn managed_input_from_pb(mi: pb::ManagedRunInput) -> ManagedRunInput { + let input = run_input_from_pb(&mi); + ManagedRunInput { + input, + list_path: mi.list_path.map(PathBuf::from), + unpack_root: PathBuf::from(mi.unpack_root), + } +} + +/// Placeholder Content for materialized run inputs — the bytes live on the +/// shared filesystem (unpack_root), so this is never read. +struct NullContent; + +impl Content for NullContent { + fn reader(&self) -> Result> { + anyhow::bail!("managed run input content is on disk (unpack_root), not streamed") + } + fn walk(&self) -> Result> + '_>> { + anyhow::bail!("managed run input content is on disk (unpack_root), not streamed") + } + fn hashout(&self) -> Result { + Ok(String::new()) + } +} + impl GuestHandler { + async fn handle_managed_run( + &self, + id: u64, + req: pb::ManagedRunRequest, + managed: Arc, + ctoken: &(dyn hcore::hasync::Cancellable + Send + Sync), + mux: &Arc, + ) { + let target = match convert::target_def_from_pb(req.target.unwrap_or_default()) { + Ok(t) => t, + Err(e) => { + mux.send_body(id, err_frame(e.to_string())); + return; + } + }; + let request_id = req.request_id; + let hashin = req.hashin; + let sandbox_dir = PathBuf::from(req.sandbox_dir); + let run_inputs: Vec = req.inputs.iter().map(run_input_from_pb).collect(); + let managed_inputs: Vec = + req.inputs.into_iter().map(managed_input_from_pb).collect(); + let rr = RunRequest { + request_id: &request_id, + target: &target, + tree_root_path: PathBuf::from(req.tree_root_path), + inputs: run_inputs, + hashin: hashin.as_str(), + stdin: None, + stdout: None, + stderr: None, + sandbox_dir: sandbox_dir.clone(), + }; + let mrr = ManagedRunRequest { + request: rr, + sandbox_dir, + sandbox_ws_dir: PathBuf::from(req.sandbox_ws_dir), + sandbox_pkg_dir: PathBuf::from(req.sandbox_pkg_dir), + inputs: managed_inputs, + }; + let result = if req.shell { + managed.run_shell(mrr, ctoken).await + } else { + managed.run(mrr, ctoken).await + }; + match result { + Ok(resp) => mux.send_body( + id, + Body::ManagedRunResp(pb::ManagedRunResponse { + artifacts: resp.artifacts.iter().map(convert::output_artifact_to_pb).collect(), + }), + ), + Err(e) => mux.send_body(id, err_frame(e.to_string())), + } + } + fn stream_addrs( &self, id: u64, diff --git a/proto/plugin/v1/driver.proto b/proto/plugin/v1/driver.proto index 61a01045..515d28a4 100644 --- a/proto/plugin/v1/driver.proto +++ b/proto/plugin/v1/driver.proto @@ -56,3 +56,43 @@ message RunRequest { message RunResponse { repeated OutputArtifactRef artifacts = 1; } + +// ---- Managed run (driver-support ManagedDriver layer) ---- +// The host's ManagedDriverBridge already materialized inputs into the shared +// sandbox dirs, so a managed run carries only paths + metadata — never input +// bytes. The guest executes in the sandbox and returns output artifacts. + +enum InputArtifactType { + INPUT_ARTIFACT_TYPE_UNSPECIFIED = 0; + INPUT_ARTIFACT_TYPE_DEP = 1; + INPUT_ARTIFACT_TYPE_SUPPORT = 2; +} + +message ManagedRunInput { + InputArtifactType type = 1; + string origin_id = 2; + Addr source_addr = 3; + repeated string filters = 4; + map annotations = 5; + // Where this input was unpacked on the shared filesystem. + string unpack_root = 6; + // List file for Dep inputs (absent for Support inputs). + optional string list_path = 7; +} + +message ManagedRunRequest { + string request_id = 1; + TargetDef target = 2; + string tree_root_path = 3; + string hashin = 4; + string sandbox_dir = 5; + string sandbox_ws_dir = 6; + string sandbox_pkg_dir = 7; + repeated ManagedRunInput inputs = 8; + // true => run_shell + bool shell = 9; +} + +message ManagedRunResponse { + repeated OutputArtifactRef artifacts = 1; +} diff --git a/proto/plugin/v1/envelope.proto b/proto/plugin/v1/envelope.proto index 64fce724..a23759f9 100644 --- a/proto/plugin/v1/envelope.proto +++ b/proto/plugin/v1/envelope.proto @@ -98,6 +98,8 @@ message Frame { ApplyTransitiveResponse apply_transitive_resp = 28; RunRequest run_req = 29; RunResponse run_resp = 30; + ManagedRunRequest managed_run_req = 53; + ManagedRunResponse managed_run_resp = 54; // host-exported callbacks (plugin->host request / host->plugin response) ResultRequest result_req = 40; From 63e9eee814937893e9e90b2a3d8bd74c74fcccd3 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 20:20:13 +0200 Subject: [PATCH 09/44] test(e2e): RemoteProvider registers + serves through a real Engine [M2] Validates the core claim: a RemoteProvider (backed by an out-of-process plugin served via the SDK over proto) registers on a real Engine through the normal register_provider factory hook, and an engine get_spec query routes to the remote plugin and returns its spec. The engine is unaware the plugin is remote. Green; clippy clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 6 ++ crates/e2e/Cargo.toml | 8 ++- crates/e2e/tests/remote_plugin.rs | 110 ++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 crates/e2e/tests/remote_plugin.rs diff --git a/Cargo.lock b/Cargo.lock index 029c5d47..2f174363 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1291,7 +1291,13 @@ name = "e2e" version = "0.1.0" dependencies = [ "anyhow", + "core", + "futures", "heph", + "model", + "plugin", + "plugin-remote", + "plugin-sdk", "tempfile", "testkit", "tokio", diff --git a/crates/e2e/Cargo.toml b/crates/e2e/Cargo.toml index c40e3f22..b265375d 100644 --- a/crates/e2e/Cargo.toml +++ b/crates/e2e/Cargo.toml @@ -10,6 +10,12 @@ workspace = true [dev-dependencies] heph = { path = "../.." } htestkit = { package = "testkit", path = "../testkit" } -tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] } +plugin-remote = { path = "../plugin-remote" } +plugin-sdk = { path = "../plugin-sdk" } +hplugin = { package = "plugin", path = "../plugin" } +hmodel = { package = "model", path = "../model" } +hcore = { package = "core", path = "../core" } +futures = "0.3" +tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "net"] } anyhow = "1" tempfile = "3" diff --git a/crates/e2e/tests/remote_plugin.rs b/crates/e2e/tests/remote_plugin.rs new file mode 100644 index 00000000..1fb9e988 --- /dev/null +++ b/crates/e2e/tests/remote_plugin.rs @@ -0,0 +1,110 @@ +//! Engine-level integration: a `RemoteProvider` (backed by an out-of-process +//! plugin served via the SDK over the proto transport) registers on a real +//! `Engine` through the normal `register_provider` factory hook and serves a +//! `get_spec` query — proving the engine is unaware the plugin is remote. + +use futures::future::BoxFuture; +use hcore::hasync::Cancellable; +use hmodel::htaddr::Addr; +use hmodel::htpkg::PkgBuf; +use hplugin::provider::{ + ConfigRequest, ConfigResponse, GetError, GetRequest, GetResponse, ListPackageResponse, + ListPackagesRequest, ListRequest, ListResponse, ProbeRequest, ProbeResponse, Provider, + TargetSpec, +}; +use std::collections::BTreeMap; +use std::sync::Arc; +use tokio::net::UnixStream; + +struct TestProvider; + +fn addr(pkg: &str, name: &str) -> Addr { + Addr::new(PkgBuf::from(pkg), name.to_string(), BTreeMap::new()) +} + +impl Provider for TestProvider { + fn config(&self, _req: ConfigRequest) -> anyhow::Result { + Ok(ConfigResponse { + name: "remote-test".to_string(), + }) + } + + fn list<'a>( + &'a self, + _req: ListRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> + Send>>> + { + Box::pin(async move { + let items = vec![Ok(ListResponse { + addr: addr("pkg", "a"), + })]; + Ok(Box::new(items.into_iter()) + as Box> + Send>) + }) + } + + fn list_packages<'a>( + &'a self, + _req: ListPackagesRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture< + 'a, + anyhow::Result> + Send>>, + > { + Box::pin(async move { + let items = vec![Ok(ListPackageResponse { + pkg: PkgBuf::from("pkg"), + })]; + Ok(Box::new(items.into_iter()) + as Box> + Send>) + }) + } + + fn get<'a>( + &'a self, + req: GetRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + if req.addr.name != "a" { + return Err(GetError::NotFound); + } + let spec = TargetSpec { + addr: req.addr, + driver: "exec".to_string(), + ..Default::default() + }; + Ok(GetResponse { target_spec: spec }) + }) + } + + fn probe<'a>( + &'a self, + _req: ProbeRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> { + Box::pin(async move { Ok(ProbeResponse { states: vec![] }) }) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn remote_provider_serves_get_spec_through_engine() -> anyhow::Result<()> { + let (a, b) = UnixStream::pair()?; + let (ar, aw) = a.into_split(); + let (br, bw) = b.into_split(); + // Guest plugin served over the proto transport (kept alive for the test). + let _guest = plugin_sdk::serve(Arc::new(TestProvider), br, bw); + let remote = plugin_remote::RemoteProvider::connect(ar, aw, "remote-test"); + + // Register the remote provider on a real Engine via the normal hook. + let ws = htestkit::WorkspaceBuilder::new()? + .with_provider(move |_| Box::new(remote)) + .build()?; + + // Resolve a target spec through the engine — it routes to the remote plugin. + let spec = ws.get_spec("//pkg:a").await?; + assert_eq!(spec.driver, "exec"); + + Ok(()) +} From ebcc38ef737dfd795dcfbacb0e2691c3f27fb35b Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 20:29:08 +0200 Subject: [PATCH 10/44] feat(plugin-remote): RemotePlugin + multi-driver routing on one connection [M2] A real plugin (plugin-go) exposes a provider plus several named drivers (golist/embed/testmain) over one process. Support that: - proto: a `driver` selector field on Parse/ApplyTransitive/ManagedRun requests (empty => the sole driver). - plugin-remote: RemotePlugin owns one connection; .provider(name)/ .managed_driver(name)/.driver(name) hand out handles that share the mux and tag requests with the component name. Remote* gain from_parts. - plugin-sdk: guest holds a name->ManagedDriver map; serve_components registers several; requests route by the selector (single-driver fallback when empty). - e2e: two named managed drivers over one connection, parse routed to the correct one. 7 plugin-remote tests green; clippy -D warnings clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-remote/src/conn.rs | 51 ++++++++++++++ crates/plugin-remote/src/driver.rs | 8 ++- crates/plugin-remote/src/lib.rs | 2 + crates/plugin-remote/src/managed.rs | 12 +++- crates/plugin-remote/src/provider.rs | 10 +-- crates/plugin-remote/tests/proto_e2e.rs | 93 +++++++++++++++++++++++++ crates/plugin-sdk/src/lib.rs | 2 +- crates/plugin-sdk/src/serve.rs | 79 +++++++++++++++------ proto/plugin/v1/driver.proto | 5 ++ 9 files changed, 232 insertions(+), 30 deletions(-) create mode 100644 crates/plugin-remote/src/conn.rs diff --git a/crates/plugin-remote/src/conn.rs b/crates/plugin-remote/src/conn.rs new file mode 100644 index 00000000..5fae281c --- /dev/null +++ b/crates/plugin-remote/src/conn.rs @@ -0,0 +1,51 @@ +//! A single connection to an out-of-process plugin that exposes several +//! components (a provider and/or multiple named drivers) over one socket. +//! +//! All derived handles share the one mux, so every component multiplexes over +//! the same connection; each request carries the component name so the guest +//! routes it. This matches a real plugin like plugin-go (one process: a +//! provider plus the golist/embed/testmain drivers). + +use crate::driver::RemoteDriver; +use crate::host::{HostCallbackHandler, HostInner}; +use crate::managed::RemoteManagedDriver; +use crate::provider::RemoteProvider; +use plugin_abi::mux::Mux; +use std::sync::Arc; +use tokio::io::{AsyncRead, AsyncWrite}; + +pub struct RemotePlugin { + mux: Arc, + inner: Arc, +} + +impl RemotePlugin { + /// Open a plugin connection over an established duplex byte stream. + pub fn connect(read: R, write: W) -> Self + where + R: AsyncRead + Unpin + Send + 'static, + W: AsyncWrite + Unpin + Send + 'static, + { + let inner = Arc::new(HostInner::default()); + let handler = Arc::new(HostCallbackHandler { + inner: Arc::clone(&inner), + }); + let mux = Mux::start(read, write, handler); + Self { mux, inner } + } + + /// A handle to this plugin's provider (`name` is its registered name). + pub fn provider(&self, name: impl Into) -> RemoteProvider { + RemoteProvider::from_parts(Arc::clone(&self.mux), Arc::clone(&self.inner), name.into()) + } + + /// A handle to one of this plugin's managed drivers, selected by `name`. + pub fn managed_driver(&self, name: impl Into) -> RemoteManagedDriver { + RemoteManagedDriver::from_parts(Arc::clone(&self.mux), Arc::clone(&self.inner), name.into()) + } + + /// A handle to one of this plugin's (non-managed) drivers, selected by `name`. + pub fn driver(&self, name: impl Into) -> RemoteDriver { + RemoteDriver::from_parts(Arc::clone(&self.mux), Arc::clone(&self.inner), name.into()) + } +} diff --git a/crates/plugin-remote/src/driver.rs b/crates/plugin-remote/src/driver.rs index 950dee37..9c78f424 100644 --- a/crates/plugin-remote/src/driver.rs +++ b/crates/plugin-remote/src/driver.rs @@ -40,8 +40,12 @@ impl RemoteDriver { inner: Arc::clone(&inner), }); let mux = Mux::start(read, write, handler); + Self::from_parts(mux, inner, name.into()) + } + + pub(crate) fn from_parts(mux: Arc, inner: Arc, name: String) -> Self { Self { - name: name.into(), + name, mux, _inner: inner, } @@ -70,6 +74,7 @@ impl Driver for RemoteDriver { let body = Body::ParseReq(pb::ParseRequest { request_id: req.request_id, target_spec: Some(convert::target_spec_to_pb(req.target_spec.as_ref())), + driver: self.name.clone(), }); match self.mux.call_cancellable(body, ctoken.cancelled()).await? { Body::ParseResp(pr) => Ok(ParseResponse { @@ -88,6 +93,7 @@ impl Driver for RemoteDriver { request_id: req.request_id, target_def: Some(convert::target_def_to_pb(&req.target_def)?), sandbox: Some(convert::sandbox_to_pb(&req.sandbox)), + driver: self.name.clone(), }); match self.mux.call_cancellable(body, ctoken.cancelled()).await? { Body::ApplyTransitiveResp(r) => Ok(ApplyTransitiveResponse { diff --git a/crates/plugin-remote/src/lib.rs b/crates/plugin-remote/src/lib.rs index c5c57d36..3b9aa51b 100644 --- a/crates/plugin-remote/src/lib.rs +++ b/crates/plugin-remote/src/lib.rs @@ -8,6 +8,7 @@ pub mod lease; +mod conn; mod driver; mod host; mod managed; @@ -15,6 +16,7 @@ mod provider; #[cfg(unix)] mod spawn; +pub use conn::RemotePlugin; pub use driver::RemoteDriver; pub use managed::RemoteManagedDriver; pub use provider::RemoteProvider; diff --git a/crates/plugin-remote/src/managed.rs b/crates/plugin-remote/src/managed.rs index 172910c2..f8283adb 100644 --- a/crates/plugin-remote/src/managed.rs +++ b/crates/plugin-remote/src/managed.rs @@ -44,8 +44,15 @@ impl RemoteManagedDriver { inner: Arc::clone(&inner), }); let mux = Mux::start(read, write, handler); + Self::from_parts(mux, inner, name.into()) + } + + /// Build from a shared connection (used by [`crate::RemotePlugin`] when a + /// plugin exposes several named drivers over one connection). `name` selects + /// which guest driver handles each request. + pub(crate) fn from_parts(mux: Arc, inner: Arc, name: String) -> Self { Self { - name: name.into(), + name, mux, _inner: inner, } @@ -96,6 +103,7 @@ impl ManagedDriver for RemoteManagedDriver { let body = Body::ParseReq(pb::ParseRequest { request_id: req.request_id, target_spec: Some(convert::target_spec_to_pb(req.target_spec.as_ref())), + driver: self.name.clone(), }); match self.mux.call_cancellable(body, ctoken.cancelled()).await? { Body::ParseResp(pr) => Ok(ParseResponse { @@ -114,6 +122,7 @@ impl ManagedDriver for RemoteManagedDriver { request_id: req.request_id, target_def: Some(convert::target_def_to_pb(&req.target_def)?), sandbox: Some(convert::sandbox_to_pb(&req.sandbox)), + driver: self.name.clone(), }); match self.mux.call_cancellable(body, ctoken.cancelled()).await? { Body::ApplyTransitiveResp(r) => Ok(ApplyTransitiveResponse { @@ -161,6 +170,7 @@ impl RemoteManagedDriver { sandbox_pkg_dir: req.sandbox_pkg_dir.to_string_lossy().into_owned(), inputs: req.inputs.iter().map(managed_input_to_pb).collect(), shell, + driver: self.name.clone(), }); match self.mux.call_cancellable(body, ctoken.cancelled()).await? { Body::ManagedRunResp(r) => Ok(ManagedRunResponse { diff --git a/crates/plugin-remote/src/provider.rs b/crates/plugin-remote/src/provider.rs index 7ce0b8fe..d8c583e3 100644 --- a/crates/plugin-remote/src/provider.rs +++ b/crates/plugin-remote/src/provider.rs @@ -39,11 +39,11 @@ impl RemoteProvider { inner: Arc::clone(&inner), }); let mux = Mux::start(read, write, handler); - Self { - name: name.into(), - mux, - inner, - } + Self::from_parts(mux, inner, name.into()) + } + + pub(crate) fn from_parts(mux: Arc, inner: Arc, name: String) -> Self { + Self { name, mux, inner } } } diff --git a/crates/plugin-remote/tests/proto_e2e.rs b/crates/plugin-remote/tests/proto_e2e.rs index 50ff712d..a17da74b 100644 --- a/crates/plugin-remote/tests/proto_e2e.rs +++ b/crates/plugin-remote/tests/proto_e2e.rs @@ -528,3 +528,96 @@ async fn managed_run_executes_remotely() { // The guest actually wrote the file into the shared sandbox. assert!(sandbox.join("out.txt").exists()); } + +// ---- multi-driver routing: several named managed drivers, one connection ---- + +struct NamedDriver(String); + +#[async_trait::async_trait] +impl hdriver_support::driver_managed::ManagedDriver for NamedDriver { + fn config( + &self, + _req: hplugin::driver::ConfigRequest, + ) -> anyhow::Result { + Ok(hplugin::driver::ConfigResponse { + name: self.0.clone(), + }) + } + fn schema(&self) -> hplugin::driver::DriverSchema { + hplugin::driver::DriverSchema::default() + } + async fn parse( + &self, + req: hplugin::driver::ParseRequest, + _ctoken: &(dyn hcore::hasync::Cancellable + Send + Sync), + ) -> anyhow::Result { + // Encode which driver handled this in the raw_def, to assert routing. + let target_def = hplugin::driver::targetdef::TargetDef { + addr: req.target_spec.addr.clone(), + labels: vec![], + raw_def: Arc::new(MyCfg { + msg: self.0.clone(), + n: 0, + }), + inputs: vec![], + outputs: vec![], + support_files: vec![], + cache: hplugin::driver::targetdef::CacheConfig::off(), + pty: false, + hash: vec![], + transparent: false, + }; + Ok(hplugin::driver::ParseResponse { target_def }) + } + async fn apply_transitive( + &self, + req: hplugin::driver::ApplyTransitiveRequest, + _ctoken: &(dyn hcore::hasync::Cancellable + Send + Sync), + ) -> anyhow::Result { + Ok(hplugin::driver::ApplyTransitiveResponse { + target_def: req.target_def, + }) + } + async fn run<'a, 'io>( + &self, + _req: hdriver_support::driver_managed::ManagedRunRequest<'a, 'io>, + _ctoken: &(dyn hcore::hasync::Cancellable + Send + Sync), + ) -> anyhow::Result { + anyhow::bail!("not used") + } +} + +#[tokio::test] +async fn multi_driver_routing_one_connection() { + use hdriver_support::driver_managed::ManagedDriver; + use std::collections::HashMap; + + let (a, b) = UnixStream::pair().expect("socketpair"); + let (ar, aw) = a.into_split(); + let (br, bw) = b.into_split(); + + let mut map: HashMap> = HashMap::new(); + map.insert("alpha".to_string(), Arc::new(NamedDriver("alpha".to_string()))); + map.insert("beta".to_string(), Arc::new(NamedDriver("beta".to_string()))); + let _guest = plugin_sdk::serve_components(None, map, br, bw); + + // One connection, two driver handles selected by name. + let plugin = plugin_remote::RemotePlugin::connect(ar, aw); + let alpha = plugin.managed_driver("alpha"); + let beta = plugin.managed_driver("beta"); + + let ctoken = StdCancellationToken::new(); + let mk_req = || hplugin::driver::ParseRequest { + request_id: "r".to_string(), + target_spec: Arc::new(hplugin::provider::TargetSpec { + addr: addr("//x", "y"), + ..Default::default() + }), + }; + + let pa = alpha.parse(mk_req(), &ctoken).await.expect("alpha parse"); + assert_eq!(pa.target_def.def_de::().msg, "alpha"); + + let pb_ = beta.parse(mk_req(), &ctoken).await.expect("beta parse"); + assert_eq!(pb_.target_def.def_de::().msg, "beta"); +} diff --git a/crates/plugin-sdk/src/lib.rs b/crates/plugin-sdk/src/lib.rs index c008a641..670db327 100644 --- a/crates/plugin-sdk/src/lib.rs +++ b/crates/plugin-sdk/src/lib.rs @@ -19,7 +19,7 @@ pub mod serve; pub use ctx::Ctx; pub use host::HostClient; -pub use serve::{serve, serve_driver, serve_managed_driver, serve_plugin}; +pub use serve::{serve, serve_components, serve_driver, serve_managed_driver, serve_plugin}; #[cfg(unix)] pub use serve::serve_inherited; diff --git a/crates/plugin-sdk/src/serve.rs b/crates/plugin-sdk/src/serve.rs index fae486b8..b6ad1820 100644 --- a/crates/plugin-sdk/src/serve.rs +++ b/crates/plugin-sdk/src/serve.rs @@ -49,27 +49,42 @@ where serve_plugin(None, Some(driver), read, write) } -/// Serve a managed-driver plugin (target execution: parse/apply_transitive/run -/// at the driver-support `ManagedDriver` layer; the host materializes the -/// sandbox). +/// Serve a single managed-driver plugin (target execution at the driver-support +/// `ManagedDriver` layer; the host materializes the sandbox). pub fn serve_managed_driver(managed: Arc, read: R, write: W) -> Arc +where + R: tokio::io::AsyncRead + Unpin + Send + 'static, + W: tokio::io::AsyncWrite + Unpin + Send + 'static, +{ + serve_components(None, HashMap::from([(String::new(), managed)]), read, write) +} + +/// Serve a plugin that may export a provider, a driver, or both (single driver). +pub fn serve_plugin( + provider: Option>, + driver: Option>, + read: R, + write: W, +) -> Arc where R: tokio::io::AsyncRead + Unpin + Send + 'static, W: tokio::io::AsyncWrite + Unpin + Send + 'static, { let handler = Arc::new(GuestHandler { - provider: None, - driver: None, - managed: Some(managed), + provider, + driver, + managed: HashMap::new(), tokens: Mutex::new(HashMap::new()), }); Mux::start(read, write, handler) } -/// Serve a plugin that may export a provider, a driver, or both. -pub fn serve_plugin( +/// Serve a plugin that exposes a provider and/or several named managed drivers +/// over one connection (the plugin-go shape: provider + golist/embed/testmain). +/// Driver-targeted requests carry the driver name to route here. +pub fn serve_components( provider: Option>, - driver: Option>, + managed: HashMap>, read: R, write: W, ) -> Arc @@ -79,8 +94,8 @@ where { let handler = Arc::new(GuestHandler { provider, - driver, - managed: None, + driver: None, + managed, tokens: Mutex::new(HashMap::new()), }); Mux::start(read, write, handler) @@ -105,11 +120,25 @@ pub async fn serve_inherited(provider: Arc) -> anyhow::Result<()> struct GuestHandler { provider: Option>, driver: Option>, - managed: Option>, + managed: HashMap>, // frame-id -> cancellation token for the in-flight method call tokens: Mutex>>, } +impl GuestHandler { + /// Pick a managed driver by selector; fall back to the sole one when the + /// selector is empty or there is exactly one registered. + fn pick_managed(&self, sel: &str) -> Option> { + if let Some(d) = self.managed.get(sel) { + return Some(Arc::clone(d)); + } + if self.managed.len() == 1 { + return self.managed.values().next().cloned(); + } + None + } +} + impl GuestHandler { fn new_token(&self, id: u64) -> Arc { let tok = Arc::new(StdCancellationToken::new()); @@ -137,7 +166,7 @@ impl InboundHandler for GuestHandler { p.config(ConfigRequest {}).map(|r| r.name) } else if let Some(d) = &self.driver { d.config(DriverConfigRequest {}).map(|r| r.name) - } else if let Some(m) = &self.managed { + } else if let Some(m) = self.managed.values().next() { m.config(DriverConfigRequest {}).map(|r| r.name) } else { Err(anyhow::anyhow!("plugin exports neither provider nor driver")) @@ -261,16 +290,19 @@ impl InboundHandler for GuestHandler { } Body::ParseReq(req) => { let tok = self.new_token(id); + let sel = req.driver.clone(); let preq = ParseRequest { request_id: req.request_id, target_spec: Arc::new(convert::target_spec_from_pb( req.target_spec.unwrap_or_default(), )), }; - let result = match (self.driver.clone(), self.managed.clone()) { - (Some(d), _) => d.parse(preq, &*tok).await, - (None, Some(m)) => m.parse(preq, &*tok).await, - (None, None) => Err(anyhow::anyhow!("plugin has no driver")), + let result = if let Some(d) = self.driver.clone() { + d.parse(preq, &*tok).await + } else if let Some(m) = self.pick_managed(&sel) { + m.parse(preq, &*tok).await + } else { + Err(anyhow::anyhow!("plugin has no driver")) }; match result { Ok(resp) => match convert::target_def_to_pb(&resp.target_def) { @@ -297,15 +329,18 @@ impl InboundHandler for GuestHandler { return; } }; + let sel = req.driver.clone(); let areq = ApplyTransitiveRequest { request_id: req.request_id, target_def, sandbox: convert::sandbox_from_pb(req.sandbox.unwrap_or_default()), }; - let result = match (self.driver.clone(), self.managed.clone()) { - (Some(d), _) => d.apply_transitive(areq, &*tok).await, - (None, Some(m)) => m.apply_transitive(areq, &*tok).await, - (None, None) => Err(anyhow::anyhow!("plugin has no driver")), + let result = if let Some(d) = self.driver.clone() { + d.apply_transitive(areq, &*tok).await + } else if let Some(m) = self.pick_managed(&sel) { + m.apply_transitive(areq, &*tok).await + } else { + Err(anyhow::anyhow!("plugin has no driver")) }; match result { Ok(resp) => match convert::target_def_to_pb(&resp.target_def) { @@ -322,7 +357,7 @@ impl InboundHandler for GuestHandler { self.drop_token(id); } Body::ManagedRunReq(req) => { - let Some(managed) = self.managed.clone() else { + let Some(managed) = self.pick_managed(&req.driver) else { mux.send_body(id, err_frame("plugin has no managed driver".to_string())); return; }; diff --git a/proto/plugin/v1/driver.proto b/proto/plugin/v1/driver.proto index 515d28a4..2a6f0ee9 100644 --- a/proto/plugin/v1/driver.proto +++ b/proto/plugin/v1/driver.proto @@ -11,6 +11,9 @@ import "plugin/v1/callback.proto"; message ParseRequest { string request_id = 1; TargetSpec target_spec = 2; + // Selects which driver on the plugin handles this (a plugin may expose + // several named drivers over one connection). Empty => the sole driver. + string driver = 3; } message ParseResponse { TargetDef target_def = 1; @@ -20,6 +23,7 @@ message ApplyTransitiveRequest { string request_id = 1; TargetDef target_def = 2; Sandbox sandbox = 3; + string driver = 4; } message ApplyTransitiveResponse { TargetDef target_def = 1; @@ -91,6 +95,7 @@ message ManagedRunRequest { repeated ManagedRunInput inputs = 8; // true => run_shell bool shell = 9; + string driver = 10; } message ManagedRunResponse { From cc8012cf8587fa21cf15422651515b744e504bbf Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 20:35:01 +0200 Subject: [PATCH 11/44] =?UTF-8?q?feat(plugin-go):=20heph-plugin-go=20binar?= =?UTF-8?q?y=20=E2=80=94=20plugin-go=20served=20out-of-process=20[M2]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit plugin-go stays Rust; this binary relocates its construction (the `go` provider + golist/embed/testmain managed drivers) into a process served over the plugin transport via the SDK, instead of in-process engine registration. It still shells `go list` internally, unchanged. - crates/plugin-go: new bin heph-plugin-go using serve_components_inherited (provider + 3 named managed drivers over fd 3). Builds its own CachedWalker (the process has FS access) and reads root / go-bin / walk-db from env. - plugin-sdk: serve_components_inherited helper (multi-component over fd 3). Compiles + clippy clean. Bootstrap flip (spawn + register the remote provider/drivers) and a go-toolchain e2e are the remaining wiring. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + crates/plugin-go/Cargo.toml | 5 +++ crates/plugin-go/src/bin/heph-plugin-go.rs | 46 ++++++++++++++++++++++ crates/plugin-sdk/src/lib.rs | 2 +- crates/plugin-sdk/src/serve.rs | 27 +++++++++++-- 5 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 crates/plugin-go/src/bin/heph-plugin-go.rs diff --git a/Cargo.lock b/Cargo.lock index 2f174363..eae8af2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3644,6 +3644,7 @@ dependencies = [ "plugin", "plugin-exec", "plugin-query", + "plugin-sdk", "proc", "serde", "serde_json", diff --git a/crates/plugin-go/Cargo.toml b/crates/plugin-go/Cargo.toml index ef85921e..dcc691db 100644 --- a/crates/plugin-go/Cargo.toml +++ b/crates/plugin-go/Cargo.toml @@ -16,6 +16,7 @@ hbuiltins = { package = "builtins", path = "../builtins" } hdriver-support = { package = "driver-support", path = "../driver-support" } hplugin-exec = { package = "plugin-exec", path = "../plugin-exec" } hplugin-query = { package = "plugin-query", path = "../plugin-query" } +plugin-sdk = { path = "../plugin-sdk" } anyhow = "1.0.102" serde = { version = "1", features = ["derive"] } serde_json = "1.0" @@ -29,5 +30,9 @@ glob = "0.3" xxhash-rust = { version = "0.8.15", features = ["xxh3", "std"] } tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "io-util", "process", "macros", "sync", "fs"] } +[[bin]] +name = "heph-plugin-go" +path = "src/bin/heph-plugin-go.rs" + [dev-dependencies] tempfile = "3" diff --git a/crates/plugin-go/src/bin/heph-plugin-go.rs b/crates/plugin-go/src/bin/heph-plugin-go.rs new file mode 100644 index 00000000..6754722e --- /dev/null +++ b/crates/plugin-go/src/bin/heph-plugin-go.rs @@ -0,0 +1,46 @@ +//! The Go plugin as an out-of-process plugin. +//! +//! plugin-go stays Rust — this binary just relocates its construction (the `go` +//! provider plus the golist/embed/testmain managed drivers) into a process +//! served over the heph plugin transport via the SDK, instead of registering +//! them in-process on the engine. It still shells out to `go list` internally, +//! exactly as before. +//! +//! Config is passed by the host at spawn via env vars: +//! - `HEPH_PLUGIN_GO_ROOT` — workspace root (default: cwd) +//! - `HEPH_PLUGIN_GO_BIN` — addr of the go toolchain (default `//@heph/bin:go`) +//! - `HEPH_PLUGIN_GO_WALK_DB` — fs-walk cache db path (default: under root) + +use hdriver_support::driver_managed::ManagedDriver; +use plugin_go::plugingo::{GoEmbedDriver, GoGolistDriver, GoTestmainDriver, Provider}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { + let root = std::env::var_os("HEPH_PLUGIN_GO_ROOT") + .map(PathBuf::from) + .unwrap_or(std::env::current_dir()?); + let go_bin = + std::env::var("HEPH_PLUGIN_GO_BIN").unwrap_or_else(|_| "//@heph/bin:go".to_string()); + let walk_db = std::env::var_os("HEPH_PLUGIN_GO_WALK_DB") + .map(PathBuf::from) + .unwrap_or_else(|| root.join(".heph-plugin-go-fswalk.db")); + + // The plugin process has filesystem access, so it owns its own walker rather + // than proxying the host's. + let walker = Arc::new(hwalk::CachedWalker::open(&walk_db)); + let opts = hplugin::config::Options::new(); + let provider = Provider::from_options(root, &[], &[], &opts, walker)?; + + let mut managed: HashMap> = HashMap::new(); + managed.insert( + "go_golist".to_string(), + Arc::new(GoGolistDriver::new(go_bin)), + ); + managed.insert("go_embed".to_string(), Arc::new(GoEmbedDriver)); + managed.insert("go_testmain".to_string(), Arc::new(GoTestmainDriver)); + + plugin_sdk::serve_components_inherited(Some(Arc::new(provider)), managed).await +} diff --git a/crates/plugin-sdk/src/lib.rs b/crates/plugin-sdk/src/lib.rs index 670db327..95b7a512 100644 --- a/crates/plugin-sdk/src/lib.rs +++ b/crates/plugin-sdk/src/lib.rs @@ -21,7 +21,7 @@ pub use ctx::Ctx; pub use host::HostClient; pub use serve::{serve, serve_components, serve_driver, serve_managed_driver, serve_plugin}; #[cfg(unix)] -pub use serve::serve_inherited; +pub use serve::{serve_components_inherited, serve_inherited}; /// Re-export of the author-facing contract so a plugin depends only on the SDK. pub use hplugin::{driver, eresult, provider}; diff --git a/crates/plugin-sdk/src/serve.rs b/crates/plugin-sdk/src/serve.rs index b6ad1820..a4fa21d3 100644 --- a/crates/plugin-sdk/src/serve.rs +++ b/crates/plugin-sdk/src/serve.rs @@ -106,15 +106,34 @@ where /// `main` typically does just `serve_inherited(Arc::new(MyProvider)).await`. #[cfg(unix)] pub async fn serve_inherited(provider: Arc) -> anyhow::Result<()> { + let (r, w) = inherited_fd3()?; + serve(provider, r, w).wait_closed().await; + Ok(()) +} + +/// Guest entry point for a multi-component plugin (a provider and/or several +/// named managed drivers — the plugin-go shape) over the inherited fd 3. +#[cfg(unix)] +pub async fn serve_components_inherited( + provider: Option>, + managed: HashMap>, +) -> anyhow::Result<()> { + let (r, w) = inherited_fd3()?; + serve_components(provider, managed, r, w).wait_closed().await; + Ok(()) +} + +#[cfg(unix)] +fn inherited_fd3() -> anyhow::Result<( + tokio::net::unix::OwnedReadHalf, + tokio::net::unix::OwnedWriteHalf, +)> { use std::os::unix::io::FromRawFd; // SAFETY: fd 3 is the protocol socket the host passed us at spawn; we own it. let std_stream = unsafe { std::os::unix::net::UnixStream::from_raw_fd(3) }; std_stream.set_nonblocking(true)?; let stream = tokio::net::UnixStream::from_std(std_stream)?; - let (r, w) = stream.into_split(); - let mux = serve(provider, r, w); - mux.wait_closed().await; - Ok(()) + Ok(stream.into_split()) } struct GuestHandler { From ca05bd9898748be9b22ded85fbc155d14cfe83de Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 20:45:08 +0200 Subject: [PATCH 12/44] feat(bootstrap): opt-in remote go plugin (HEPH_REMOTE_GO) [M2] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire heph-plugin-go into the engine: when HEPH_REMOTE_GO is set, spawn one heph-plugin-go process (next to the heph binary; root/go-bin passed via env) and register the `go` provider + go_golist/go_embed/go_testmain managed drivers as RemotePlugin handles sharing that connection — through the same factory hooks, with the same config opt-in semantics. Default stays fully in-process, so production is unchanged. - plugin-remote: spawn_streams (env + fd-3 socket) + RemotePlugin: Clone. - root: depend on plugin-remote; bootstrap gates go registration on the flag (+ a runtime-present check), falling back to in-process. Engine core untouched. End-to-end validation needs the go toolchain (set HEPH_REMOTE_GO and run a go build); the wiring compiles + clippy clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + Cargo.toml | 1 + crates/plugin-remote/src/conn.rs | 1 + crates/plugin-remote/src/lib.rs | 2 +- crates/plugin-remote/src/spawn.rs | 32 +++++++++-- src/commands/bootstrap.rs | 95 ++++++++++++++++++++++++------- 6 files changed, 104 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eae8af2d..3ed0a99c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1971,6 +1971,7 @@ dependencies = [ "plugin-go", "plugin-nix", "plugin-query", + "plugin-remote", "posthog-rs", "pprof", "proc", diff --git a/Cargo.toml b/Cargo.toml index 5f56d832..70a8f33e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,6 +110,7 @@ hplugin-exec = { package = "plugin-exec", path = "crates/plugin-exec" } hplugin-nix = { package = "plugin-nix", path = "crates/plugin-nix" } hplugin-query = { package = "plugin-query", path = "crates/plugin-query" } hplugin-go = { package = "plugin-go", path = "crates/plugin-go" } +hplugin-remote = { package = "plugin-remote", path = "crates/plugin-remote" } htelemetry = { package = "telemetry", path = "crates/telemetry" } htui = { package = "tui", path = "crates/tui" } hlock = { package = "lock", path = "crates/lock" } diff --git a/crates/plugin-remote/src/conn.rs b/crates/plugin-remote/src/conn.rs index 5fae281c..8400fd81 100644 --- a/crates/plugin-remote/src/conn.rs +++ b/crates/plugin-remote/src/conn.rs @@ -14,6 +14,7 @@ use plugin_abi::mux::Mux; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; +#[derive(Clone)] pub struct RemotePlugin { mux: Arc, inner: Arc, diff --git a/crates/plugin-remote/src/lib.rs b/crates/plugin-remote/src/lib.rs index 3b9aa51b..e610d093 100644 --- a/crates/plugin-remote/src/lib.rs +++ b/crates/plugin-remote/src/lib.rs @@ -21,7 +21,7 @@ pub use driver::RemoteDriver; pub use managed::RemoteManagedDriver; pub use provider::RemoteProvider; #[cfg(unix)] -pub use spawn::{spawn_plugin, PLUGIN_FD}; +pub use spawn::{spawn_plugin, spawn_streams, PLUGIN_FD}; #[cfg(feature = "shm")] pub mod shm; diff --git a/crates/plugin-remote/src/spawn.rs b/crates/plugin-remote/src/spawn.rs index 5e2c5f53..99323bd7 100644 --- a/crates/plugin-remote/src/spawn.rs +++ b/crates/plugin-remote/src/spawn.rs @@ -17,19 +17,29 @@ use std::process::{Child, Command}; /// The fd the plugin child inherits the protocol socket on. pub const PLUGIN_FD: RawFd = 3; -/// Spawn `program` as a plugin and connect to it over proto. Returns the host -/// adapter plus the child handle (kill/wait it to control its lifetime). -pub fn spawn_plugin( +/// The split tokio halves of a spawned plugin's protocol socket. +pub type PluginStreams = ( + tokio::net::unix::OwnedReadHalf, + tokio::net::unix::OwnedWriteHalf, +); + +/// Spawn `program` with `args` + `env`, passing the protocol socket on fd 3. +/// Returns the tokio half-streams + the child handle. Higher-level helpers +/// ([`spawn_plugin`], [`crate::RemotePlugin::spawn`]) build adapters on top. +pub fn spawn_streams( program: &Path, args: &[String], - name: impl Into, -) -> anyhow::Result<(RemoteProvider, Child)> { + env: &[(String, String)], +) -> anyhow::Result<(PluginStreams, Child)> { let (parent, child_end) = std::os::unix::net::UnixStream::pair()?; parent.set_nonblocking(true)?; let child_fd = child_end.as_raw_fd(); let mut cmd = Command::new(program); cmd.args(args); + for (k, v) in env { + cmd.env(k, v); + } // Runs post-fork, pre-exec. dup2 clears CLOEXEC on the new fd so fd 3 // survives exec, while the original (CLOEXEC) socketpair fd closes. let pre = move || -> std::io::Result<()> { @@ -51,6 +61,16 @@ pub fn spawn_plugin( drop(child_end); let tokio_parent = tokio::net::UnixStream::from_std(parent)?; - let (r, w) = tokio_parent.into_split(); + Ok((tokio_parent.into_split(), child)) +} + +/// Spawn `program` as a single-provider plugin and connect over proto. Returns +/// the host adapter plus the child handle (kill/wait it to control its lifetime). +pub fn spawn_plugin( + program: &Path, + args: &[String], + name: impl Into, +) -> anyhow::Result<(RemoteProvider, Child)> { + let ((r, w), child) = spawn_streams(program, args, &[])?; Ok((RemoteProvider::connect(r, w, name), child)) } diff --git a/src/commands/bootstrap.rs b/src/commands/bootstrap.rs index f1879876..2199fc51 100644 --- a/src/commands/bootstrap.rs +++ b/src/commands/bootstrap.rs @@ -109,15 +109,36 @@ pub fn new_engine() -> anyhow::Result<(Arc, ShutdownTrigger)> { .with_walker(init.walker.clone()), )) })?; - e.register_provider_factory("go", |init, opts| { - Ok(Box::new(plugingo::Provider::from_options( - init.root.to_path_buf(), - &init.skip_dirs, - &init.skip_globs, - opts, - init.walker.clone(), - )?)) - })?; + // The go plugin can run out-of-process (opt-in via HEPH_REMOTE_GO): one + // heph-plugin-go process serves the `go` provider + golist/embed/testmain + // drivers over the plugin transport. Default stays in-process. + let remote_go = std::env::var_os("HEPH_REMOTE_GO").is_some() + && tokio::runtime::Handle::try_current().is_ok(); + if remote_go { + register_remote_go(&mut e, &root)?; + } else { + e.register_provider_factory("go", |init, opts| { + Ok(Box::new(plugingo::Provider::from_options( + init.root.to_path_buf(), + &init.skip_dirs, + &init.skip_globs, + opts, + init.walker.clone(), + )?)) + })?; + e.register_managed_driver_factory("go_golist", |_init, opts| { + config_yaml::deny_unknown("go_golist driver", opts, &[])?; + Ok(Box::new(plugingo::GoGolistDriver::new("//@heph/bin:go"))) + })?; + e.register_managed_driver_factory("go_embed", |_init, opts| { + config_yaml::deny_unknown("go_embed driver", opts, &[])?; + Ok(Box::new(plugingo::GoEmbedDriver)) + })?; + e.register_managed_driver_factory("go_testmain", |_init, opts| { + config_yaml::deny_unknown("go_testmain driver", opts, &[])?; + Ok(Box::new(plugingo::GoTestmainDriver)) + })?; + } e.register_managed_driver_factory("exec", |_init, opts| { Ok(Box::new(pluginexec::Driver::from_options_exec(opts)?)) @@ -128,18 +149,6 @@ pub fn new_engine() -> anyhow::Result<(Arc, ShutdownTrigger)> { e.register_managed_driver_factory("sh", |_init, opts| { Ok(Box::new(pluginexec::Driver::from_options_sh(opts)?)) })?; - e.register_managed_driver_factory("go_golist", |_init, opts| { - config_yaml::deny_unknown("go_golist driver", opts, &[])?; - Ok(Box::new(plugingo::GoGolistDriver::new("//@heph/bin:go"))) - })?; - e.register_managed_driver_factory("go_embed", |_init, opts| { - config_yaml::deny_unknown("go_embed driver", opts, &[])?; - Ok(Box::new(plugingo::GoEmbedDriver)) - })?; - e.register_managed_driver_factory("go_testmain", |_init, opts| { - config_yaml::deny_unknown("go_testmain driver", opts, &[])?; - Ok(Box::new(plugingo::GoTestmainDriver)) - })?; e.apply_config(&file.providers, &file.drivers)?; @@ -398,3 +407,47 @@ drivers: ); } } + +/// Spawn `heph-plugin-go` (next to the heph executable) and register its `go` +/// provider + golist/embed/testmain managed drivers as remote handles sharing +/// one connection. Opt-in via `HEPH_REMOTE_GO`. plugin-go stays Rust — this just +/// runs it out-of-process. +#[cfg(unix)] +fn register_remote_go(e: &mut engine::Engine, root: &std::path::Path) -> anyhow::Result<()> { + let exe = std::env::current_exe().context("locate heph executable")?; + let bin = exe + .parent() + .map(|p| p.join("heph-plugin-go")) + .context("heph-plugin-go directory")?; + let env = vec![ + ( + "HEPH_PLUGIN_GO_ROOT".to_string(), + root.to_string_lossy().into_owned(), + ), + ("HEPH_PLUGIN_GO_BIN".to_string(), "//@heph/bin:go".to_string()), + ]; + let ((r, w), child) = hplugin_remote::spawn_streams(&bin, &[], &env) + .with_context(|| format!("spawn {}", bin.display()))?; + // The plugin self-exits when our end of the socket closes (engine drop), so + // we don't reap the child eagerly. + std::mem::forget(child); + let plugin = hplugin_remote::RemotePlugin::connect(r, w); + // Register as factories (same opt-in semantics as in-process): only + // activated when the config lists `go` / `go_*`. + { + let p = plugin.clone(); + e.register_provider_factory("go", move |_init, _opts| Ok(Box::new(p.provider("go"))))?; + } + for name in ["go_golist", "go_embed", "go_testmain"] { + let p = plugin.clone(); + e.register_managed_driver_factory(name, move |_init, _opts| { + Ok(Box::new(p.managed_driver(name))) + })?; + } + Ok(()) +} + +#[cfg(not(unix))] +fn register_remote_go(_e: &mut engine::Engine, _root: &std::path::Path) -> anyhow::Result<()> { + anyhow::bail!("HEPH_REMOTE_GO is only supported on unix") +} From b121139c71bf942276eb374c62bc17edaeca3331 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 21:13:19 +0200 Subject: [PATCH 13/44] feat(plugin-abi): shm transport over iceoryx2 (byte-pipe under Mux) [M3] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shm tier: an iceoryx2 shared-memory byte-pipe wrapped as AsyncRead/AsyncWrite, so the entire existing Mux + plugin protocol run over shared memory unchanged — no protocol reimplementation. - plugin-abi `shm` feature: two iceoryx2 pub-sub services (one per direction) carry the Frame byte stream; a single dedicated io-thread owns the !Send ports and bridges to async via channels. - ShmReadHalf/ShmWriteHalf impl AsyncRead/AsyncWrite, so RemotePlugin::connect / serve work over shm with no new API. - plugin-remote `shm` feature enables it; full plugin-over-shm e2e (config + get round-trip through iceoryx2) + a byte-pipe round-trip test. Both green; clippy -D warnings clean. Avoids the per-message syscall (the UDS cost). Follow-up perf-hardening: rkyv zero-copy payload in the loaned sample, event-driven wakeup (Listener/WaitSet) instead of the polling io-thread, note_dep batching, and a benchmark vs the UDS baseline. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 463 +++++++++++++++++++++++++- crates/plugin-abi/Cargo.toml | 3 + crates/plugin-abi/src/lib.rs | 3 + crates/plugin-abi/src/shm.rs | 243 ++++++++++++++ crates/plugin-remote/Cargo.toml | 2 +- crates/plugin-remote/tests/shm_e2e.rs | 131 ++++++++ 6 files changed, 841 insertions(+), 4 deletions(-) create mode 100644 crates/plugin-abi/src/shm.rs create mode 100644 crates/plugin-remote/tests/shm_e2e.rs diff --git a/Cargo.lock b/Cargo.lock index 3ed0a99c..478a52fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -363,6 +363,26 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.13.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.117", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -587,6 +607,15 @@ dependencies = [ "shlex 2.0.1", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -630,6 +659,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.6.1" @@ -1395,6 +1435,26 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "enum-iterator" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4549325971814bda7a44061bf3fe7e487d447cba01e4220a4b454d630d7a016" +dependencies = [ + "enum-iterator-derive", +] + +[[package]] +name = "enum-iterator-derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "685adfa4d6f3d765a26bc5dbc936577de9abf756c1feeb3089b01dd395034842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "env_filter" version = "1.0.1" @@ -2165,6 +2225,270 @@ dependencies = [ "cc", ] +[[package]] +name = "iceoryx2" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa70a19527472fb5c7ec3f76b0f77e241ce390302eee8a39c0f859c038360f15" +dependencies = [ + "iceoryx2-bb-concurrency", + "iceoryx2-bb-container", + "iceoryx2-bb-derive-macros", + "iceoryx2-bb-elementary", + "iceoryx2-bb-elementary-traits", + "iceoryx2-bb-lock-free", + "iceoryx2-bb-loggers", + "iceoryx2-bb-memory", + "iceoryx2-bb-posix", + "iceoryx2-bb-print", + "iceoryx2-bb-system-types", + "iceoryx2-cal", + "iceoryx2-log", + "iceoryx2-pal-configuration", + "serde", + "tiny-fn", + "toml", +] + +[[package]] +name = "iceoryx2-bb-concurrency" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "232b2c9276d276974ced52f9470e73636524eecb07628b03b3f6b306cfc645d3" +dependencies = [ + "iceoryx2-bb-elementary-traits", + "iceoryx2-pal-concurrency-sync", +] + +[[package]] +name = "iceoryx2-bb-container" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5dd1a9f404bd5789a98614363f7f5ad7fa757a2cf8b20a83b6b1580f5b8c0" +dependencies = [ + "iceoryx2-bb-concurrency", + "iceoryx2-bb-derive-macros", + "iceoryx2-bb-elementary", + "iceoryx2-bb-elementary-traits", + "iceoryx2-log", + "serde", +] + +[[package]] +name = "iceoryx2-bb-derive-macros" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a8f63bbb07a2a4092f154da74516d3238899b4d3498f6156bd4b7d9170368ef" +dependencies = [ + "iceoryx2-bb-elementary", + "iceoryx2-bb-elementary-traits", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "iceoryx2-bb-elementary" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8655f486ccddb8cfaf8c3a4811ec7ca055f49186779f388928a6a741c99a6b" +dependencies = [ + "iceoryx2-bb-elementary-traits", + "iceoryx2-pal-concurrency-sync", +] + +[[package]] +name = "iceoryx2-bb-elementary-traits" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487b7bb44cb35688a037610e3d367ce8d9d31720ce1b0d3ad0619ec19b95b6f4" + +[[package]] +name = "iceoryx2-bb-linux" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bcc939836845926763f027518c33a7fa03c36907d12330387bc72b7df8e686e" +dependencies = [ + "iceoryx2-bb-concurrency", + "iceoryx2-bb-container", + "iceoryx2-bb-posix", + "iceoryx2-bb-system-types", + "iceoryx2-log", + "iceoryx2-pal-os-api", + "iceoryx2-pal-posix", +] + +[[package]] +name = "iceoryx2-bb-lock-free" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "908daf57ca2fad15af8df83e1a571aeeaffb0392432bcc588356b15b496f32c0" +dependencies = [ + "iceoryx2-bb-concurrency", + "iceoryx2-bb-elementary", + "iceoryx2-bb-elementary-traits", + "iceoryx2-log", +] + +[[package]] +name = "iceoryx2-bb-loggers" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e447da3f13cc7bcdfa208dea63a9cc24fd0dccea80943040b3e074fbf54dbc00" +dependencies = [ + "iceoryx2-bb-print", + "iceoryx2-log-types", + "iceoryx2-pal-concurrency-sync", + "iceoryx2-pal-posix", +] + +[[package]] +name = "iceoryx2-bb-memory" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cca425d22230f7e3ac4944002349cbce90ef76f2358a0961eb9c259c68b321b" +dependencies = [ + "iceoryx2-bb-concurrency", + "iceoryx2-bb-elementary", + "iceoryx2-bb-elementary-traits", + "iceoryx2-bb-lock-free", + "iceoryx2-bb-posix", + "iceoryx2-log", +] + +[[package]] +name = "iceoryx2-bb-posix" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e03397688249c464214f3534546f6c231a1368ac12a4fe89ea33b4a33a692c8" +dependencies = [ + "enum-iterator", + "iceoryx2-bb-concurrency", + "iceoryx2-bb-container", + "iceoryx2-bb-derive-macros", + "iceoryx2-bb-elementary", + "iceoryx2-bb-elementary-traits", + "iceoryx2-bb-print", + "iceoryx2-bb-system-types", + "iceoryx2-log", + "iceoryx2-pal-configuration", + "iceoryx2-pal-posix", + "serde", + "tiny-fn", +] + +[[package]] +name = "iceoryx2-bb-print" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433a64f537be4df813ef00e71884a2062df90bf5c55447e4ffafb4827eafe173" +dependencies = [ + "iceoryx2-pal-print", +] + +[[package]] +name = "iceoryx2-bb-system-types" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6573fdf227176a90a97a8b49efe41216f971d099ac5797d902249dee7e6806" +dependencies = [ + "iceoryx2-bb-container", + "iceoryx2-bb-derive-macros", + "iceoryx2-bb-elementary", + "iceoryx2-bb-elementary-traits", + "iceoryx2-log", + "iceoryx2-pal-configuration", + "iceoryx2-pal-posix", + "serde", +] + +[[package]] +name = "iceoryx2-cal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f527b34fc6ace4ea1a9a539c234ff07b51cdb281d677481e5f7f7e15c809214e" +dependencies = [ + "iceoryx2-bb-concurrency", + "iceoryx2-bb-container", + "iceoryx2-bb-derive-macros", + "iceoryx2-bb-elementary", + "iceoryx2-bb-elementary-traits", + "iceoryx2-bb-linux", + "iceoryx2-bb-lock-free", + "iceoryx2-bb-memory", + "iceoryx2-bb-posix", + "iceoryx2-bb-system-types", + "iceoryx2-log", + "postcard", + "serde", + "sha1_smol", + "tiny-fn", + "toml", +] + +[[package]] +name = "iceoryx2-log" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f55e205c7d00876d8f8259cc5c259e150aad7eb93167def6aa2bfa6dc2c7537" +dependencies = [ + "iceoryx2-log-types", + "iceoryx2-pal-concurrency-sync", +] + +[[package]] +name = "iceoryx2-log-types" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f0fc0134ef60a5b7d8bfb700cf35af76fc40180f46aaee7fc0152145442ede" + +[[package]] +name = "iceoryx2-pal-concurrency-sync" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a4e7f4a2e8ca8bb94892254bd59b811eecd6e6cc78b0fd27eea91ca4dd6aab" + +[[package]] +name = "iceoryx2-pal-configuration" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9664fb2564d62f6af5babece4c8b2170f2c3605e93578eb38161d6c4bd6db5d" + +[[package]] +name = "iceoryx2-pal-os-api" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a4d7c695c8caf78a6ffd254733be993b0bece78e0b9330d80cf87cc905fff3d" +dependencies = [ + "bindgen", + "cc", + "iceoryx2-pal-posix", + "libc", +] + +[[package]] +name = "iceoryx2-pal-posix" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c318f15cc0ec35f15b6f9767fa8af9c0a07f8f1a0a60d3adb40b7f03945343" +dependencies = [ + "bindgen", + "cc", + "iceoryx2-pal-concurrency-sync", + "iceoryx2-pal-configuration", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "iceoryx2-pal-print" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe15e255ef982e229d5919959e02c7c80296e31a44a2d8f13fa5bc4654a8d9f4" +dependencies = [ + "iceoryx2-pal-posix", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -2494,6 +2818,16 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -3548,6 +3882,7 @@ dependencies = [ "anyhow", "async-trait", "core", + "iceoryx2", "model", "plugin", "prost 0.14.4", @@ -4885,6 +5220,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4921,6 +5265,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -5581,6 +5931,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" +[[package]] +name = "tiny-fn" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9659b108631d1e1cf3e8e489f894bee40bc9d68fd6cc67ec4d4ce9b72d565228" + [[package]] name = "tinystr" version = "0.8.3" @@ -5657,6 +6013,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "serde_core", + "serde_spanned", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_datetime" version = "1.1.1+spec-1.1.0" @@ -5673,9 +6052,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ "indexmap", - "toml_datetime", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow", + "winnow 1.0.3", ] [[package]] @@ -5684,9 +6063,15 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow", + "winnow 1.0.3", ] +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + [[package]] name = "tower" version = "0.5.3" @@ -6368,6 +6753,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -6404,6 +6798,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -6437,6 +6846,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -6449,6 +6864,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -6461,6 +6882,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -6485,6 +6912,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -6497,6 +6930,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -6509,6 +6948,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -6521,6 +6966,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -6533,6 +6984,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + [[package]] name = "winnow" version = "1.0.3" diff --git a/crates/plugin-abi/Cargo.toml b/crates/plugin-abi/Cargo.toml index 1976b2ac..2ec49803 100644 --- a/crates/plugin-abi/Cargo.toml +++ b/crates/plugin-abi/Cargo.toml @@ -22,10 +22,13 @@ prost = { version = "0.14", optional = true } tokio = { version = "1.52", features = ["io-util", "rt", "sync", "macros"], optional = true } async-trait = { version = "0.1.89", optional = true } tracing = { version = "0.1", optional = true } +# shm tier (iceoryx2 byte-pipe under the existing Mux) +iceoryx2 = { version = "0.9", optional = true } [features] convert = ["dep:hplugin", "dep:hmodel", "dep:hcore", "dep:serde_json"] transport = ["dep:prost", "dep:tokio", "dep:async-trait", "dep:tracing"] +shm = ["transport", "dep:iceoryx2"] [dev-dependencies] rkyv = "0.8" diff --git a/crates/plugin-abi/src/lib.rs b/crates/plugin-abi/src/lib.rs index c492f1cd..7ce4023a 100644 --- a/crates/plugin-abi/src/lib.rs +++ b/crates/plugin-abi/src/lib.rs @@ -29,3 +29,6 @@ pub mod convert; pub mod frame; #[cfg(feature = "transport")] pub mod mux; + +#[cfg(feature = "shm")] +pub mod shm; diff --git a/crates/plugin-abi/src/shm.rs b/crates/plugin-abi/src/shm.rs new file mode 100644 index 00000000..3f347751 --- /dev/null +++ b/crates/plugin-abi/src/shm.rs @@ -0,0 +1,243 @@ +//! shm transport: an iceoryx2 shared-memory byte-pipe wrapped as +//! `AsyncRead`/`AsyncWrite`, so the existing [`crate::mux::Mux`] (and thus the +//! whole plugin protocol) runs over it unchanged. +//! +//! Two iceoryx2 publish-subscribe services carry the Frame byte stream, one per +//! direction; each side publishes on its "send" service and subscribes on its +//! "recv" service (host and guest pass them swapped). Ordering is FIFO per +//! single-publisher service, so the byte stream is preserved. +//! +//! iceoryx2 ports are `!Send`, so a single dedicated io-thread owns the node, +//! publisher and subscriber and bridges to the async world over channels: +//! `AsyncWrite` -> std mpsc -> publish; subscribe -> tokio mpsc -> `AsyncRead`. +//! +//! v1 copies bytes into/out of samples and the io-thread polls; that already +//! avoids the per-message syscall (the UDS cost). True zero-copy (rkyv payload +//! in the loaned sample) and event-driven wakeup (`Listener`/`WaitSet`) are +//! follow-up optimizations. + +use iceoryx2::prelude::*; +use std::io; +use std::pin::Pin; +use std::sync::mpsc as smpsc; +use std::task::{Context, Poll}; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::sync::mpsc; + +/// Max bytes per iceoryx2 sample; larger writes are split across samples. +const MAX_CHUNK: usize = 64 * 1024; + +pub struct ShmReadHalf { + rx: mpsc::UnboundedReceiver>, + leftover: Vec, + pos: usize, +} + +pub struct ShmWriteHalf { + tx: smpsc::Sender>, +} + +/// Open a shm byte-pipe. `send_service`/`recv_service` are the iceoryx2 service +/// names for this side (host and guest pass them swapped). +pub fn connect( + send_service: &str, + recv_service: &str, +) -> anyhow::Result<(ShmReadHalf, ShmWriteHalf)> { + let (wtx, wrx) = smpsc::channel::>(); + let (rtx, rrx) = mpsc::unbounded_channel::>(); + let (setup_tx, setup_rx) = smpsc::channel::>(); + + let send_service = send_service.to_string(); + let recv_service = recv_service.to_string(); + + // All iceoryx2 handles (!Send) live on this one thread. + std::thread::spawn(move || { + let ports = (|| -> anyhow::Result<_> { + let node = NodeBuilder::new().create::()?; + let send_svc = node + .service_builder(&send_service.as_str().try_into()?) + .publish_subscribe::<[u8]>() + .open_or_create()?; + let recv_svc = node + .service_builder(&recv_service.as_str().try_into()?) + .publish_subscribe::<[u8]>() + .open_or_create()?; + let publisher = send_svc + .publisher_builder() + .initial_max_slice_len(MAX_CHUNK) + .create()?; + let subscriber = recv_svc.subscriber_builder().create()?; + Ok((node, publisher, subscriber)) + })(); + + match ports { + Ok((_node, publisher, subscriber)) => { + if setup_tx.send(Ok(())).is_err() { + return; + } + run_loop(&publisher, &subscriber, &wrx, &rtx); + } + Err(e) => { + drop(setup_tx.send(Err(e.to_string()))); + } + } + }); + + setup_rx + .recv() + .map_err(|_e| anyhow::anyhow!("shm io-thread died during setup"))? + .map_err(|e| anyhow::anyhow!("shm setup: {e}"))?; + + Ok(( + ShmReadHalf { + rx: rrx, + leftover: Vec::new(), + pos: 0, + }, + ShmWriteHalf { tx: wtx }, + )) +} + +type Publisher = iceoryx2::port::publisher::Publisher; +type Subscriber = iceoryx2::port::subscriber::Subscriber; + +fn run_loop( + publisher: &Publisher, + subscriber: &Subscriber, + wrx: &smpsc::Receiver>, + rtx: &mpsc::UnboundedSender>, +) { + loop { + let mut idle = true; + + // Outbound: publish queued chunks. + loop { + match wrx.try_recv() { + Ok(chunk) => { + idle = false; + if publish(publisher, &chunk).is_err() { + return; + } + } + Err(smpsc::TryRecvError::Empty) => break, + Err(smpsc::TryRecvError::Disconnected) => return, + } + } + + // Inbound: forward received payloads. + loop { + match subscriber.receive() { + Ok(Some(sample)) => { + idle = false; + if rtx.send(sample.payload().to_vec()).is_err() { + return; + } + } + Ok(None) => break, + Err(_) => return, + } + } + + if idle { + std::thread::sleep(Duration::from_micros(50)); + } + } +} + +fn publish(publisher: &Publisher, chunk: &[u8]) -> anyhow::Result<()> { + let sample = publisher.loan_slice_uninit(chunk.len())?; + let sample = sample.write_from_slice(chunk); + sample.send()?; + Ok(()) +} + +impl AsyncRead for ShmReadHalf { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + if self.pos < self.leftover.len() { + let start = self.pos; + let end = (start + buf.remaining()).min(self.leftover.len()); + if let Some(s) = self.leftover.get(start..end) { + buf.put_slice(s); + self.pos = end; + } + return Poll::Ready(Ok(())); + } + match self.rx.poll_recv(cx) { + Poll::Ready(Some(chunk)) => { + let n = chunk.len().min(buf.remaining()); + if let Some(s) = chunk.get(..n) { + buf.put_slice(s); + } + if n < chunk.len() { + self.leftover = chunk; + self.pos = n; + } + Poll::Ready(Ok(())) + } + Poll::Ready(None) => Poll::Ready(Ok(())), + Poll::Pending => Poll::Pending, + } + } +} + +impl AsyncWrite for ShmWriteHalf { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + for chunk in buf.chunks(MAX_CHUNK) { + self.tx + .send(chunk.to_vec()) + .map_err(|_e| io::Error::new(io::ErrorKind::BrokenPipe, "shm writer closed"))?; + } + Poll::Ready(Ok(buf.len())) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU32, Ordering}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + static N: AtomicU32 = AtomicU32::new(0); + + #[tokio::test] + async fn shm_byte_pipe_roundtrip() { + let id = format!( + "heph_shm_{}_{}", + std::process::id(), + N.fetch_add(1, Ordering::Relaxed) + ); + let h2g = format!("{id}_h2g"); + let g2h = format!("{id}_g2h"); + + // Host: send on h2g, recv on g2h. Guest: the reverse. + let (mut host_r, mut host_w) = connect(&h2g, &g2h).expect("host shm"); + let (mut guest_r, mut guest_w) = connect(&g2h, &h2g).expect("guest shm"); + + host_w.write_all(b"ping").await.expect("host write"); + let mut a = [0u8; 4]; + guest_r.read_exact(&mut a).await.expect("guest read"); + assert_eq!(&a, b"ping"); + + guest_w.write_all(b"pong").await.expect("guest write"); + let mut b = [0u8; 4]; + host_r.read_exact(&mut b).await.expect("host read"); + assert_eq!(&b, b"pong"); + } +} diff --git a/crates/plugin-remote/Cargo.toml b/crates/plugin-remote/Cargo.toml index 7230a0ae..6ef9f7c1 100644 --- a/crates/plugin-remote/Cargo.toml +++ b/crates/plugin-remote/Cargo.toml @@ -23,7 +23,7 @@ libc = "0.2" [features] # heavy transports, off by default; proto (UDS) is always available -shm = [] +shm = ["plugin-abi/shm"] wasm = [] [dev-dependencies] diff --git a/crates/plugin-remote/tests/shm_e2e.rs b/crates/plugin-remote/tests/shm_e2e.rs new file mode 100644 index 00000000..8da61156 --- /dev/null +++ b/crates/plugin-remote/tests/shm_e2e.rs @@ -0,0 +1,131 @@ +//! Full plugin protocol over the shm (iceoryx2) transport: a guest provider +//! served on shm byte-pipe halves, driven by a host RemoteProvider over the +//! other end. Proves the entire Mux/protocol runs over shared memory unchanged. +//! +//! Gated on the `shm` feature (heavy iceoryx2 dep): `cargo test -p plugin-remote +//! --features shm --test shm_e2e`. +#![cfg(feature = "shm")] + +use futures::future::BoxFuture; +use hcore::hasync::{Cancellable, StdCancellationToken}; +use hmodel::htaddr::Addr; +use hmodel::htpkg::PkgBuf; +use hplugin::provider::{ + ConfigRequest, ConfigResponse, GetError, GetRequest, GetResponse, ListPackageResponse, + ListPackagesRequest, ListRequest, ListResponse, ProbeRequest, ProbeResponse, Provider, + ProviderExecutor, TargetSpec, +}; +use hmodel::htmatcher::Matcher; +use std::collections::BTreeMap; +use std::sync::Arc; + +fn addr(pkg: &str, name: &str) -> Addr { + Addr::new(PkgBuf::from(pkg), name.to_string(), BTreeMap::new()) +} + +struct TestProvider; + +impl Provider for TestProvider { + fn config(&self, _req: ConfigRequest) -> anyhow::Result { + Ok(ConfigResponse { + name: "shm-test".to_string(), + }) + } + fn list<'a>( + &'a self, + _req: ListRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> + Send>>> + { + Box::pin(async move { + let items = vec![Ok(ListResponse { + addr: addr("pkg", "a"), + })]; + Ok(Box::new(items.into_iter()) + as Box> + Send>) + }) + } + fn list_packages<'a>( + &'a self, + _req: ListPackagesRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture< + 'a, + anyhow::Result> + Send>>, + > { + Box::pin(async move { + Ok(Box::new(std::iter::empty()) + as Box> + Send>) + }) + } + fn get<'a>( + &'a self, + req: GetRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let spec = TargetSpec { + addr: req.addr, + driver: "exec".to_string(), + ..Default::default() + }; + Ok(GetResponse { target_spec: spec }) + }) + } + fn probe<'a>( + &'a self, + _req: ProbeRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> { + Box::pin(async move { Ok(ProbeResponse { states: vec![] }) }) + } +} + +struct NoopExec; +impl ProviderExecutor for NoopExec { + fn result<'a>( + &'a self, + _addr: &'a Addr, + ) -> BoxFuture<'a, anyhow::Result>> { + Box::pin(async move { Ok(Arc::new(hplugin::eresult::EResult::default())) }) + } + fn query<'a>( + &'a self, + _m: &'a Matcher, + _s: &'a [String], + ) -> BoxFuture<'a, anyhow::Result>> { + Box::pin(async move { Ok(vec![]) }) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn provider_over_shm() { + let id = format!("heph_shm_e2e_{}", std::process::id()); + let h2g = format!("{id}_h2g"); + let g2h = format!("{id}_g2h"); + + // Host: send h2g / recv g2h. Guest: the reverse. + let (hr, hw) = plugin_abi::shm::connect(&h2g, &g2h).expect("host shm"); + let (gr, gw) = plugin_abi::shm::connect(&g2h, &h2g).expect("guest shm"); + + let _guest = plugin_sdk::serve(Arc::new(TestProvider), gr, gw); + let host = plugin_remote::RemoteProvider::connect(hr, hw, "shm-test"); + + assert_eq!(host.config(ConfigRequest {}).expect("config").name, "shm-test"); + + let ctoken = StdCancellationToken::new(); + let executor: Arc = Arc::new(NoopExec); + let resp = host + .get( + GetRequest { + request_id: "r1".to_string(), + addr: addr("//pkg", "a"), + states: vec![], + executor, + }, + &ctoken, + ) + .await + .expect("get over shm"); + assert_eq!(resp.target_spec.driver, "exec"); +} From b7c4cb94a45b27868175f039f6f7281423f22adf Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 21:47:28 +0200 Subject: [PATCH 14/44] feat(plugin): note_dep edge-only API + typed wire errors (no message matching) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit note_dep: ProviderExecutor gains note_dep (register parent->addr edge, cycle-check, no execute); default falls back to result, the engine overrides with a direct track_dep. Host serves the cache-hit path with it instead of a full result. Typed transport errors (per review: never match by message): - proto Error.Kind += CYCLE; GetError.Kind += CYCLE/CANCELLED. - plugin-abi: WireError{kind,message} (impl Error); the mux raises it on an Error frame, carrying the kind. - host classifies errors by downcasting to the real engine types (CycleError/CancelledError) -> typed kind on Error/GetErr/NoteDepResp. - guest maps the kind back to a typed error (CycleError/CancelledError) for plugin code — result/note_dep/get all reconstruct the type. Test: a host-detected CycleError survives the full round trip (result callback -> Error{Cycle} -> guest -> GetErr{Cycle} -> host) and arrives as a typed CycleError at get(). 8 proto tests + engine e2e green; clippy -D warnings clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/e2e/tests/remote_plugin.rs | 1 + crates/engine/src/engine/result.rs | 9 ++ crates/plugin-abi/src/error.rs | 69 ++++++++++++++ crates/plugin-abi/src/lib.rs | 2 + crates/plugin-abi/src/mux.rs | 2 +- crates/plugin-remote/src/host.rs | 49 ++++++---- crates/plugin-remote/src/provider.rs | 19 +++- crates/plugin-remote/tests/proto_e2e.rs | 115 ++++++++++++++++++++++++ crates/plugin-sdk/src/serve.rs | 77 +++++++++++++++- crates/plugin/src/provider.rs | 12 +++ proto/plugin/v1/envelope.proto | 1 + proto/plugin/v1/provider.proto | 2 + 12 files changed, 333 insertions(+), 25 deletions(-) create mode 100644 crates/plugin-abi/src/error.rs diff --git a/crates/e2e/tests/remote_plugin.rs b/crates/e2e/tests/remote_plugin.rs index 1fb9e988..bf05cf3f 100644 --- a/crates/e2e/tests/remote_plugin.rs +++ b/crates/e2e/tests/remote_plugin.rs @@ -108,3 +108,4 @@ async fn remote_provider_serves_get_spec_through_engine() -> anyhow::Result<()> Ok(()) } + diff --git a/crates/engine/src/engine/result.rs b/crates/engine/src/engine/result.rs index 607ec26c..34eea949 100644 --- a/crates/engine/src/engine/result.rs +++ b/crates/engine/src/engine/result.rs @@ -71,6 +71,15 @@ impl ProviderExecutor for EngineProviderExecutor { }) } + fn note_dep<'a>( + &'a self, + addr: &'a Addr, + ) -> futures::future::BoxFuture<'a, anyhow::Result<()>> { + // Edge-only: register parent → addr (the synchronous cycle check) without + // executing. parent is already set on `rs` by the enclosing result_addr. + Box::pin(async move { self.rs.track_dep(addr).map_err(anyhow::Error::new) }) + } + fn query<'a>( &'a self, m: &'a Matcher, diff --git a/crates/plugin-abi/src/error.rs b/crates/plugin-abi/src/error.rs new file mode 100644 index 00000000..f0513f28 --- /dev/null +++ b/crates/plugin-abi/src/error.rs @@ -0,0 +1,69 @@ +//! Typed transport error, carried as the wire `Error` frame's `kind`. Callers +//! match on the kind (a well-known code), never on the message string. The +//! host classifies an error into a kind by downcasting to the real engine error +//! types; the guest maps the kind back to a typed error for plugin code. + +use crate::pb; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WireErrorKind { + Other, + Cancelled, + NotFound, + Cycle, +} + +impl WireErrorKind { + pub fn from_pb(kind: i32) -> Self { + match pb::error::Kind::try_from(kind).unwrap_or(pb::error::Kind::Other) { + pb::error::Kind::Cancelled => Self::Cancelled, + pb::error::Kind::NotFound => Self::NotFound, + pb::error::Kind::Cycle => Self::Cycle, + pb::error::Kind::Other | pb::error::Kind::Unspecified => Self::Other, + } + } + + pub fn as_pb(self) -> pb::error::Kind { + match self { + Self::Cancelled => pb::error::Kind::Cancelled, + Self::NotFound => pb::error::Kind::NotFound, + Self::Cycle => pb::error::Kind::Cycle, + Self::Other => pb::error::Kind::Other, + } + } +} + +/// An error received over the transport, with its kind preserved. +#[derive(Debug, Clone)] +pub struct WireError { + pub kind: WireErrorKind, + pub message: String, +} + +impl WireError { + pub fn from_pb(e: pb::Error) -> Self { + Self { + kind: WireErrorKind::from_pb(e.kind), + message: e.message, + } + } + + pub fn to_pb(&self) -> pb::Error { + pb::Error { + kind: self.as_pb_kind() as i32, + message: self.message.clone(), + } + } + + fn as_pb_kind(&self) -> pb::error::Kind { + self.kind.as_pb() + } +} + +impl std::fmt::Display for WireError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +impl std::error::Error for WireError {} diff --git a/crates/plugin-abi/src/lib.rs b/crates/plugin-abi/src/lib.rs index 7ce4023a..b7ca62b5 100644 --- a/crates/plugin-abi/src/lib.rs +++ b/crates/plugin-abi/src/lib.rs @@ -25,6 +25,8 @@ pub mod shm_types; #[cfg(feature = "convert")] pub mod convert; +#[cfg(feature = "transport")] +pub mod error; #[cfg(feature = "transport")] pub mod frame; #[cfg(feature = "transport")] diff --git a/crates/plugin-abi/src/mux.rs b/crates/plugin-abi/src/mux.rs index 6cf6da0e..ea8e0556 100644 --- a/crates/plugin-abi/src/mux.rs +++ b/crates/plugin-abi/src/mux.rs @@ -157,7 +157,7 @@ impl Mux { let resp = resp .map_err(|_e| anyhow::anyhow!("plugin connection closed before response"))?; if let Body::Error(e) = resp { - anyhow::bail!("plugin error: {}", e.message); + return Err(crate::error::WireError::from_pb(e).into()); } Ok(resp) } diff --git a/crates/plugin-remote/src/host.rs b/crates/plugin-remote/src/host.rs index ecc78d6c..6f5f8157 100644 --- a/crates/plugin-remote/src/host.rs +++ b/crates/plugin-remote/src/host.rs @@ -54,6 +54,27 @@ fn err_frame(message: String) -> Body { }) } +/// True if `e`'s chain contains a dependency-cycle error. +fn is_cycle(e: &anyhow::Error) -> bool { + hcore::hmemoizer::downcast_chain_ref::(e).is_some() +} + +/// Build an Error frame whose kind reflects the real error type (downcast, never +/// message-matched), so the guest can reconstruct a typed error. +fn err_frame_from(e: &anyhow::Error) -> Body { + let kind = if is_cycle(e) { + pb::error::Kind::Cycle + } else if hplugin::error::is_cancelled(e) { + pb::error::Kind::Cancelled + } else { + pb::error::Kind::Other + }; + Body::Error(pb::Error { + kind: kind as i32, + message: e.to_string(), + }) +} + #[async_trait] impl InboundHandler for HostCallbackHandler { async fn handle(&self, id: u64, body: Body, mux: Arc) { @@ -113,34 +134,28 @@ impl HostCallbackHandler { }), ); } - Err(e) => mux.send_body(id, err_frame(e.to_string())), + Err(e) => mux.send_body(id, err_frame_from(&e)), } } async fn handle_note_dep(&self, id: u64, req: pb::NoteDepRequest, mux: &Arc) { - // M1: fall back to a full result() to register the dep edge (the engine - // registers parent->addr before any await). The true edge-only fast - // path is an engine API added in M2/M3. let Some(scope) = self.inner.scope(&req.request_id) else { mux.send_body(id, err_frame(format!("unknown request scope {}", req.request_id))); return; }; let addr = convert::addr_from_pb(req.addr.unwrap_or_default()); - let resp = match scope.executor.result(&addr).await { - Ok(_) => pb::NoteDepResponse { + // Edge-only registration (cheap); cycle is detected by type, not message. + let resp = match scope.executor.note_dep(&addr).await { + Ok(()) => pb::NoteDepResponse { ok: true, cycle: false, message: String::new(), }, - Err(e) => { - let msg = e.to_string(); - let cycle = msg.to_lowercase().contains("cycle"); - pb::NoteDepResponse { - ok: false, - cycle, - message: msg, - } - } + Err(e) => pb::NoteDepResponse { + ok: false, + cycle: is_cycle(&e), + message: e.to_string(), + }, }; mux.send_body(id, Body::NoteDepResp(resp)); } @@ -158,7 +173,7 @@ impl HostCallbackHandler { addrs: addrs.iter().map(convert::addr_to_pb).collect(), }), ), - Err(e) => mux.send_body(id, err_frame(e.to_string())), + Err(e) => mux.send_body(id, err_frame_from(&e)), } } @@ -182,7 +197,7 @@ impl HostCallbackHandler { mux.send_body(id, Body::StreamItem(pb::StreamItem { item: buf.into() })); mux.send_body(id, Body::StreamEnd(pb::StreamEnd { error: None })); } - Ok(Err(e)) => mux.send_body(id, err_frame(e.to_string())), + Ok(Err(e)) => mux.send_body(id, err_frame_from(&e)), Err(e) => mux.send_body(id, err_frame(format!("artifact read task failed: {e}"))), } } diff --git a/crates/plugin-remote/src/provider.rs b/crates/plugin-remote/src/provider.rs index d8c583e3..e218dbd6 100644 --- a/crates/plugin-remote/src/provider.rs +++ b/crates/plugin-remote/src/provider.rs @@ -147,10 +147,21 @@ impl Provider for RemoteProvider { target_spec: convert::target_spec_from_pb(gr.target_spec.unwrap_or_default()), }), Ok(Body::GetErr(ge)) => { - if ge.kind == pb::get_error::Kind::NotFound as i32 { - Err(GetError::NotFound) - } else { - Err(GetError::Other(anyhow::anyhow!("{}", ge.message))) + // Reconstruct the typed error from the kind (never the message). + match pb::get_error::Kind::try_from(ge.kind) + .unwrap_or(pb::get_error::Kind::Other) + { + pb::get_error::Kind::NotFound => Err(GetError::NotFound), + pb::get_error::Kind::Cycle => Err(GetError::Other(anyhow::Error::new( + hplugin::error::CycleError { + from: req.addr.clone(), + to: req.addr.clone(), + }, + ))), + pb::get_error::Kind::Cancelled => { + Err(GetError::Other(anyhow::Error::new(hplugin::error::CancelledError))) + } + _ => Err(GetError::Other(anyhow::anyhow!("{}", ge.message))), } } Ok(other) => Err(GetError::Other(anyhow::anyhow!( diff --git a/crates/plugin-remote/tests/proto_e2e.rs b/crates/plugin-remote/tests/proto_e2e.rs index a17da74b..ae7fb012 100644 --- a/crates/plugin-remote/tests/proto_e2e.rs +++ b/crates/plugin-remote/tests/proto_e2e.rs @@ -621,3 +621,118 @@ async fn multi_driver_routing_one_connection() { let pb_ = beta.parse(mk_req(), &ctoken).await.expect("beta parse"); assert_eq!(pb_.target_def.def_de::().msg, "beta"); } + +// ---- typed cycle error propagates across the boundary (not by message) ---- + +/// Host executor that simulates the engine detecting a cycle: result/note_dep +/// return a typed CycleError. +struct CycleExec; +impl ProviderExecutor for CycleExec { + fn result<'a>(&'a self, addr: &'a Addr) -> BoxFuture<'a, anyhow::Result>> { + Box::pin(async move { + Err(anyhow::Error::new(hplugin::error::CycleError { + from: addr.clone(), + to: addr.clone(), + })) + }) + } + fn query<'a>( + &'a self, + _m: &'a Matcher, + _s: &'a [String], + ) -> BoxFuture<'a, anyhow::Result>> { + Box::pin(async move { Ok(vec![]) }) + } +} + +struct CycleTestProvider; +impl Provider for CycleTestProvider { + fn config(&self, _req: ConfigRequest) -> anyhow::Result { + Ok(ConfigResponse { + name: "cyc".to_string(), + }) + } + fn list<'a>( + &'a self, + _req: ListRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> + Send>>> + { + Box::pin(async move { + Ok(Box::new(std::iter::empty()) + as Box> + Send>) + }) + } + fn list_packages<'a>( + &'a self, + _req: ListPackagesRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture< + 'a, + anyhow::Result> + Send>>, + > { + Box::pin(async move { + Ok(Box::new(std::iter::empty()) + as Box> + Send>) + }) + } + fn get<'a>( + &'a self, + req: GetRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // Resolving a dep cycles host-side; the typed error must survive the + // round trip back to the host get(). + req.executor + .result(&addr("//dep", "x")) + .await + .map_err(GetError::Other)?; + Ok(GetResponse { + target_spec: TargetSpec { + addr: req.addr, + ..Default::default() + }, + }) + }) + } + fn probe<'a>( + &'a self, + _req: ProbeRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> { + Box::pin(async move { Ok(ProbeResponse { states: vec![] }) }) + } +} + +#[tokio::test] +async fn cycle_error_propagates_typed() { + let (a, b) = UnixStream::pair().expect("socketpair"); + let (ar, aw) = a.into_split(); + let (br, bw) = b.into_split(); + let _guest = plugin_sdk::serve(Arc::new(CycleTestProvider), br, bw); + let host = RemoteProvider::connect(ar, aw, "cyc"); + + let ctoken = StdCancellationToken::new(); + let executor: Arc = Arc::new(CycleExec); + let res = host + .get( + GetRequest { + request_id: "r1".to_string(), + addr: addr("//pkg", "a"), + states: vec![], + executor, + }, + &ctoken, + ) + .await; + + match res { + Ok(_) => panic!("expected a cycle error"), + Err(GetError::NotFound) => panic!("expected Other(CycleError), got NotFound"), + Err(GetError::Other(e)) => assert!( + e.downcast_ref::().is_some(), + "expected a typed CycleError (not message), got: {e:#}" + ), + } +} diff --git a/crates/plugin-sdk/src/serve.rs b/crates/plugin-sdk/src/serve.rs index a4fa21d3..810177fa 100644 --- a/crates/plugin-sdk/src/serve.rs +++ b/crates/plugin-sdk/src/serve.rs @@ -22,6 +22,7 @@ use hmodel::htaddr::Addr; use hmodel::htmatcher::Matcher; use hmodel::htpkg::PkgBuf; use plugin_abi::convert; +use plugin_abi::error::{WireError, WireErrorKind}; use plugin_abi::mux::{Body, InboundHandler, Mux}; use plugin_abi::pb; use prost::Message; @@ -279,7 +280,7 @@ impl InboundHandler for GuestHandler { Err(GetError::Other(e)) => mux.send_body( id, Body::GetErr(pb::GetError { - kind: pb::get_error::Kind::Other as i32, + kind: get_error_kind(&e) as i32, message: e.to_string(), }), ), @@ -540,7 +541,45 @@ fn stream_err(message: String) -> Body { }) } -/// A `ProviderExecutor` that forwards `result`/`query` to the host over the mux. +/// True if `e`'s chain carries a dependency-cycle error. +fn is_cycle(e: &anyhow::Error) -> bool { + hcore::hmemoizer::downcast_chain_ref::(e).is_some() +} + +/// Classify an anyhow error into the wire `GetError` kind, by type (not message). +fn get_error_kind(e: &anyhow::Error) -> pb::get_error::Kind { + if is_cycle(e) { + pb::get_error::Kind::Cycle + } else if hplugin::error::is_cancelled(e) { + pb::get_error::Kind::Cancelled + } else { + pb::get_error::Kind::Other + } +} + +/// Map a transport [`WireError`] back to the corresponding typed engine error, +/// by kind (never by message). `addr` is the target being resolved, used to +/// rebuild a `CycleError`. +fn map_wire_err(e: anyhow::Error, addr: &Addr) -> anyhow::Error { + if let Some(w) = e.downcast_ref::() { + match w.kind { + WireErrorKind::Cycle => { + return anyhow::Error::new(hplugin::error::CycleError { + from: addr.clone(), + to: addr.clone(), + }); + } + WireErrorKind::Cancelled => { + return anyhow::Error::new(hplugin::error::CancelledError); + } + WireErrorKind::NotFound | WireErrorKind::Other => {} + } + } + e +} + +/// A `ProviderExecutor` that forwards `result`/`query`/`note_dep` to the host +/// over the mux. struct MuxExecutor { mux: Arc, request_id: String, @@ -579,7 +618,12 @@ impl ProviderExecutor for MuxExecutor { request_id: self.request_id.clone(), addr: Some(convert::addr_to_pb(addr)), }); - match self.mux.call(body).await? { + let resp = self + .mux + .call(body) + .await + .map_err(|e| map_wire_err(e, addr))?; + match resp { Body::ResultResp(rr) => { // Eagerly pull each artifact's bytes over the mux. M2: whole // artifact in one fetch; lazy/offset chunking is M3. Most @@ -635,6 +679,33 @@ impl ProviderExecutor for MuxExecutor { } }) } + + fn note_dep<'a>(&'a self, addr: &'a Addr) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + // The host registers parent -> addr from its own request context, so + // `parent` here is informational only. + let body = Body::NoteDepReq(pb::NoteDepRequest { + request_id: self.request_id.clone(), + parent: None, + addr: Some(convert::addr_to_pb(addr)), + }); + match self.mux.call(body).await.map_err(|e| map_wire_err(e, addr))? { + Body::NoteDepResp(r) => { + if r.ok { + Ok(()) + } else if r.cycle { + Err(anyhow::Error::new(hplugin::error::CycleError { + from: addr.clone(), + to: addr.clone(), + })) + } else { + anyhow::bail!("{}", r.message) + } + } + other => anyhow::bail!("unexpected note_dep response: {other:?}"), + } + }) + } } /// A host artifact materialized on the guest side: the bytes are fetched eagerly diff --git a/crates/plugin/src/provider.rs b/crates/plugin/src/provider.rs index b7ce6d46..9470e326 100644 --- a/crates/plugin/src/provider.rs +++ b/crates/plugin/src/provider.rs @@ -96,6 +96,18 @@ pub trait ProviderExecutor: Send + Sync { m: &'a Matcher, extra_skip: &'a [String], ) -> BoxFuture<'a, anyhow::Result>>; + + /// Register a `parent → addr` dependency edge without resolving `addr`'s + /// result. The cache-hit fast path: a provider that already has `addr`'s + /// derived data still must register the edge (the synchronous cycle check), + /// but needn't pay for a full `result`. Returns a [`crate::error::CycleError`] + /// (in the error chain) when the edge closes a cycle. + /// + /// Default falls back to `result` (correct, just not cheap); the engine + /// overrides it with a direct edge insert. + fn note_dep<'a>(&'a self, addr: &'a Addr) -> BoxFuture<'a, anyhow::Result<()>> { + Box::pin(async move { self.result(addr).await.map(|_| ()) }) + } } pub struct GetRequest { diff --git a/proto/plugin/v1/envelope.proto b/proto/plugin/v1/envelope.proto index a23759f9..10f81fc5 100644 --- a/proto/plugin/v1/envelope.proto +++ b/proto/plugin/v1/envelope.proto @@ -57,6 +57,7 @@ message Error { KIND_OTHER = 1; KIND_CANCELLED = 2; KIND_NOT_FOUND = 3; + KIND_CYCLE = 4; } Kind kind = 1; string message = 2; diff --git a/proto/plugin/v1/provider.proto b/proto/plugin/v1/provider.proto index e0ce0f61..8b8329ed 100644 --- a/proto/plugin/v1/provider.proto +++ b/proto/plugin/v1/provider.proto @@ -46,6 +46,8 @@ message GetError { KIND_UNSPECIFIED = 0; KIND_NOT_FOUND = 1; KIND_OTHER = 2; + KIND_CYCLE = 3; + KIND_CANCELLED = 4; } Kind kind = 1; string message = 2; From 27c15dd8b0bcc3414b4a6717ce3eaddd80ff51e7 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 22:27:46 +0200 Subject: [PATCH 15/44] feat(plugin-exec): heph-plugin-exec out-of-process + wasm devenv toolchain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - crates/plugin-exec: new bin heph-plugin-exec serving exec/bash/sh as managed drivers over the plugin transport (serve_components_inherited). pluginexec stays Rust; the host ManagedDriverBridge materializes the sandbox, the guest executes the command and returns artifacts. - bootstrap: HEPH_REMOTE_EXEC opt-in spawns it + registers exec/bash/sh as RemotePlugin factories; default in-process. (Child handle dropped, not forgotten — std detaches; plugin self-exits on socket EOF.) - devenv.nix: wasm component toolchain (wasm32-wasip2 target, cargo-component, wasm-tools) to unblock the wasm transport; new plugin-* crates added to the fmt set. Builds + clippy -D warnings clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + crates/plugin-exec/Cargo.toml | 5 ++ .../plugin-exec/src/bin/heph-plugin-exec.rs | 23 ++++++++ devenv.nix | 8 ++- src/commands/bootstrap.rs | 58 +++++++++++++++---- 5 files changed, 83 insertions(+), 12 deletions(-) create mode 100644 crates/plugin-exec/src/bin/heph-plugin-exec.rs diff --git a/Cargo.lock b/Cargo.lock index 478a52fd..e742f3cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3951,6 +3951,7 @@ dependencies = [ "minijinja", "model", "plugin", + "plugin-sdk", "proc", "serde", "serde_json", diff --git a/crates/plugin-exec/Cargo.toml b/crates/plugin-exec/Cargo.toml index a7ff066d..7f60520d 100644 --- a/crates/plugin-exec/Cargo.toml +++ b/crates/plugin-exec/Cargo.toml @@ -12,6 +12,7 @@ hmodel = { package = "model", path = "../model" } hplugin = { package = "plugin", path = "../plugin" } hproc = { package = "proc", path = "../proc" } hdriver-support = { package = "driver-support", path = "../driver-support" } +plugin-sdk = { path = "../plugin-sdk" } anyhow = "1.0.102" serde = { version = "1", features = ["derive"] } serde_json = "1.0" @@ -24,6 +25,10 @@ minijinja = "2" crossterm = { version = "0.29", features = ["event-stream"] } tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "io-util", "io-std", "process", "macros", "sync", "signal", "fs", "net"] } +[[bin]] +name = "heph-plugin-exec" +path = "src/bin/heph-plugin-exec.rs" + [dev-dependencies] enclose = "1" serde_yaml = "0.9" diff --git a/crates/plugin-exec/src/bin/heph-plugin-exec.rs b/crates/plugin-exec/src/bin/heph-plugin-exec.rs new file mode 100644 index 00000000..93f4f886 --- /dev/null +++ b/crates/plugin-exec/src/bin/heph-plugin-exec.rs @@ -0,0 +1,23 @@ +//! The exec/bash/sh driver plugin as an out-of-process plugin. +//! +//! pluginexec stays Rust — this binary relocates its three managed drivers +//! (exec, bash, sh) into a process served over the heph plugin transport via +//! the SDK. The host's ManagedDriverBridge still materializes the sandbox; this +//! process executes the command in it and returns output artifacts. +//! +//! Served over the inherited fd 3 (set up by the host's spawn). + +use hdriver_support::driver_managed::ManagedDriver; +use plugin_exec::pluginexec::Driver; +use std::collections::HashMap; +use std::sync::Arc; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { + let opts = hplugin::config::Options::new(); + let mut managed: HashMap> = HashMap::new(); + managed.insert("exec".to_string(), Arc::new(Driver::from_options_exec(&opts)?)); + managed.insert("bash".to_string(), Arc::new(Driver::from_options_bash(&opts)?)); + managed.insert("sh".to_string(), Arc::new(Driver::from_options_sh(&opts)?)); + plugin_sdk::serve_components_inherited(None, managed).await +} diff --git a/devenv.nix b/devenv.nix index 02773166..ef435e63 100644 --- a/devenv.nix +++ b/devenv.nix @@ -2,7 +2,7 @@ let binLocation = "$HOME/.local/bin/heph3"; - qualityCrates = "-p heph -p e2e -p testkit -p plugingo-e2e -p htspec-derive -p core -p walk -p proc -p model -p sandboxfuse -p plugin -p builtins -p plugin-buildfile -p driver-support -p plugin-exec -p plugin-nix -p plugin-query -p plugin-go -p telemetry -p tui -p lock -p engine"; + qualityCrates = "-p heph -p e2e -p testkit -p plugingo-e2e -p htspec-derive -p core -p walk -p proc -p model -p sandboxfuse -p plugin -p plugin-abi -p plugin-sdk -p plugin-remote -p plugin-echo -p builtins -p plugin-buildfile -p driver-support -p plugin-exec -p plugin-nix -p plugin-query -p plugin-go -p telemetry -p tui -p lock -p engine"; in { # https://devenv.sh/basics/ @@ -18,6 +18,10 @@ in pkgs.cargo-zigbuild pkgs.tokio-console pkgs.sccache + # wasm component tooling for the wasm plugin transport (M4): build guest + # components + inspect/convert wasm. + pkgs.cargo-component + pkgs.wasm-tools # pkg-config + libfuse for the `fuse-sandbox` feature. # - Linux: `fuse3` ships headers/pc files fuser links against. # - macOS: `macfuse-stubs` provides the build-time `osxfuse.pc` per @@ -41,7 +45,7 @@ in enable = true; channel = "stable"; components = [ "rustc" "cargo" "clippy" "rustfmt" "rust-analyzer" ]; - targets = [ "x86_64-apple-darwin" "aarch64-apple-darwin" ] + targets = [ "x86_64-apple-darwin" "aarch64-apple-darwin" "wasm32-wasip2" ] ++ lib.optionals pkgs.stdenv.isLinux [ "x86_64-unknown-linux-gnu" "aarch64-unknown-linux-gnu" ]; }; diff --git a/src/commands/bootstrap.rs b/src/commands/bootstrap.rs index 2199fc51..dedff1fc 100644 --- a/src/commands/bootstrap.rs +++ b/src/commands/bootstrap.rs @@ -140,15 +140,22 @@ pub fn new_engine() -> anyhow::Result<(Arc, ShutdownTrigger)> { })?; } - e.register_managed_driver_factory("exec", |_init, opts| { - Ok(Box::new(pluginexec::Driver::from_options_exec(opts)?)) - })?; - e.register_managed_driver_factory("bash", |_init, opts| { - Ok(Box::new(pluginexec::Driver::from_options_bash(opts)?)) - })?; - e.register_managed_driver_factory("sh", |_init, opts| { - Ok(Box::new(pluginexec::Driver::from_options_sh(opts)?)) - })?; + // exec/bash/sh can likewise run out-of-process (opt-in via HEPH_REMOTE_EXEC). + let remote_exec = std::env::var_os("HEPH_REMOTE_EXEC").is_some() + && tokio::runtime::Handle::try_current().is_ok(); + if remote_exec { + register_remote_exec(&mut e)?; + } else { + e.register_managed_driver_factory("exec", |_init, opts| { + Ok(Box::new(pluginexec::Driver::from_options_exec(opts)?)) + })?; + e.register_managed_driver_factory("bash", |_init, opts| { + Ok(Box::new(pluginexec::Driver::from_options_bash(opts)?)) + })?; + e.register_managed_driver_factory("sh", |_init, opts| { + Ok(Box::new(pluginexec::Driver::from_options_sh(opts)?)) + })?; + } e.apply_config(&file.providers, &file.drivers)?; @@ -430,7 +437,9 @@ fn register_remote_go(e: &mut engine::Engine, root: &std::path::Path) -> anyhow: .with_context(|| format!("spawn {}", bin.display()))?; // The plugin self-exits when our end of the socket closes (engine drop), so // we don't reap the child eagerly. - std::mem::forget(child); + // Dropping the handle does not kill the child (std detaches); the plugin + // self-exits when our socket end closes at engine drop. + drop(child); let plugin = hplugin_remote::RemotePlugin::connect(r, w); // Register as factories (same opt-in semantics as in-process): only // activated when the config lists `go` / `go_*`. @@ -451,3 +460,32 @@ fn register_remote_go(e: &mut engine::Engine, root: &std::path::Path) -> anyhow: fn register_remote_go(_e: &mut engine::Engine, _root: &std::path::Path) -> anyhow::Result<()> { anyhow::bail!("HEPH_REMOTE_GO is only supported on unix") } + +/// Spawn `heph-plugin-exec` and register its exec/bash/sh managed drivers as +/// remote handles sharing one connection. Opt-in via `HEPH_REMOTE_EXEC`. +#[cfg(unix)] +fn register_remote_exec(e: &mut engine::Engine) -> anyhow::Result<()> { + let exe = std::env::current_exe().context("locate heph executable")?; + let bin = exe + .parent() + .map(|p| p.join("heph-plugin-exec")) + .context("heph-plugin-exec directory")?; + let ((r, w), child) = hplugin_remote::spawn_streams(&bin, &[], &[]) + .with_context(|| format!("spawn {}", bin.display()))?; + // Dropping the handle does not kill the child (std detaches); the plugin + // self-exits when our socket end closes at engine drop. + drop(child); + let plugin = hplugin_remote::RemotePlugin::connect(r, w); + for name in ["exec", "bash", "sh"] { + let p = plugin.clone(); + e.register_managed_driver_factory(name, move |_init, _opts| { + Ok(Box::new(p.managed_driver(name))) + })?; + } + Ok(()) +} + +#[cfg(not(unix))] +fn register_remote_exec(_e: &mut engine::Engine) -> anyhow::Result<()> { + anyhow::bail!("HEPH_REMOTE_EXEC is only supported on unix") +} From dab1ebc2c6963b6ce43dc10710e75e816c10d9e9 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 23:03:15 +0200 Subject: [PATCH 16/44] feat(plugin-remote): wasm transport vertical slice (wasmtime host + echo guest) [M4] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit De-risks the wasm tier end-to-end: a cargo-component guest component (crates/wasm-guests/echo) exporting `greet`, loaded and called through an in-process wasmtime host (plugin_remote::wasm, feature-gated `wasm`) with WASI linked. Proves the cargo-component ↔ wasmtime contract before the full provider/driver WIT + AbiHost host-imports are brought up. - echo guest: standalone workspace (kept out of host workspace), wasip2 component built via cargo-component; wit world echo { export greet }. - host: wasmtime 45 component bindgen + wasmtime-wasi p2; WasiView over WasiCtx+ResourceTable; instantiate_and_greet. - e2e test gated on --features wasm: builds the guest, instantiates, asserts greet("heph") == "hello heph". - devenv.nix: add wasm32-wasip1 target (cargo-component's build+adapt path). Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1034 ++++++++++++++++++++++- Cargo.toml | 3 + crates/plugin-remote/Cargo.toml | 4 +- crates/plugin-remote/src/host.rs | 23 +- crates/plugin-remote/src/lib.rs | 2 +- crates/plugin-remote/src/provider.rs | 10 +- crates/plugin-remote/src/wasm.rs | 72 +- crates/plugin-remote/tests/proto_e2e.rs | 38 +- crates/plugin-remote/tests/shm_e2e.rs | 15 +- crates/plugin-remote/tests/wasm_e2e.rs | 50 ++ crates/wasm-guests/echo/.gitignore | 1 + crates/wasm-guests/echo/Cargo.lock | 25 + crates/wasm-guests/echo/Cargo.toml | 20 + crates/wasm-guests/echo/src/bindings.rs | 119 +++ crates/wasm-guests/echo/src/lib.rs | 14 + crates/wasm-guests/echo/wit/world.wit | 7 + devenv.nix | 4 +- 17 files changed, 1393 insertions(+), 48 deletions(-) create mode 100644 crates/plugin-remote/tests/wasm_e2e.rs create mode 100644 crates/wasm-guests/echo/.gitignore create mode 100644 crates/wasm-guests/echo/Cargo.lock create mode 100644 crates/wasm-guests/echo/Cargo.toml create mode 100644 crates/wasm-guests/echo/src/bindings.rs create mode 100644 crates/wasm-guests/echo/src/lib.rs create mode 100644 crates/wasm-guests/echo/wit/world.wit diff --git a/Cargo.lock b/Cargo.lock index e742f3cb..cc16af6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,7 +18,16 @@ version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" dependencies = [ - "gimli", + "gimli 0.32.3", +] + +[[package]] +name = "addr2line" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59317f77929f0e679d39364702289274de2f0f0b22cbf50b2b8cff2169a0b27a" +dependencies = [ + "gimli 0.33.0", ] [[package]] @@ -99,6 +108,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -201,9 +216,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348" dependencies = [ - "object", + "object 0.37.3", ] +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + [[package]] name = "arrayref" version = "0.3.9" @@ -336,11 +357,11 @@ version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" dependencies = [ - "addr2line", + "addr2line 0.25.1", "cfg-if", "libc", "miniz_oxide", - "object", + "object 0.37.3", "rustc-demangle", "windows-link", ] @@ -524,6 +545,9 @@ name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +dependencies = [ + "allocator-api2", +] [[package]] name = "by_address" @@ -586,6 +610,74 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cap-fs-ext" +version = "3.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5528f85b1e134ae811704e41ef80930f56e795923f866813255bc342cc20654" +dependencies = [ + "cap-primitives", + "cap-std", + "io-lifetimes", + "windows-sys 0.59.0", +] + +[[package]] +name = "cap-net-ext" +version = "3.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20a158160765c6a7d0d8c072a53d772e4cb243f38b04bfcf6b4939cfbe7482e7" +dependencies = [ + "cap-primitives", + "cap-std", + "rustix", + "smallvec", +] + +[[package]] +name = "cap-primitives" +version = "3.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6cf3aea8a5081171859ef57bc1606b1df6999df4f1110f8eef68b30098d1d3a" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes", + "ipnet", + "maybe-owned", + "rustix", + "rustix-linux-procfs", + "windows-sys 0.59.0", + "winx", +] + +[[package]] +name = "cap-std" +version = "3.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6dc3090992a735d23219de5c204927163d922f42f575a0189b005c62d37549a" +dependencies = [ + "cap-primitives", + "io-extras", + "io-lifetimes", + "rustix", +] + +[[package]] +name = "cap-time-ext" +version = "3.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "def102506ce40c11710a9b16e614af0cde8e76ae51b1f48c04b8d79f4b671a80" +dependencies = [ + "ambient-authority", + "cap-primitives", + "iana-time-zone", + "once_cell", + "rustix", + "winx", +] + [[package]] name = "castaway" version = "0.2.4" @@ -906,6 +998,148 @@ dependencies = [ "libc", ] +[[package]] +name = "cranelift-assembler-x64" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bc293b86236abcc45f2f72e2d18e2bd636f2a08b75eb286bae31e71e1430c91" +dependencies = [ + "cranelift-assembler-x64-meta", +] + +[[package]] +name = "cranelift-assembler-x64-meta" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b954c826eddaf1b001402cb8aecf1764c6f6d637ba69fb9e3311f1ebac965be6" +dependencies = [ + "cranelift-srcgen", +] + +[[package]] +name = "cranelift-bforest" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4053fa2575ef4a5c35d2708533df2200400ae979226cea9cc92a578b811bd4e7" +dependencies = [ + "cranelift-entity", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-bitset" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d216663191014aa63e1d2cffd058e609eaf207646d40b739d88250f65b2c4f69" +dependencies = [ + "serde", + "serde_derive", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-codegen" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a5e7e7aad6a425a51da1ad7ab9e5d280ea97eb7c7c4545fafb567915a75aadb" +dependencies = [ + "bumpalo", + "cranelift-assembler-x64", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-control", + "cranelift-entity", + "cranelift-isle", + "gimli 0.33.0", + "hashbrown 0.17.1", + "libm", + "log", + "pulley-interpreter", + "regalloc2", + "rustc-hash", + "serde", + "smallvec", + "target-lexicon", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c421d80a9a85f806cb02a2983b5b5368a335c319795b1f1b4b771a24479af5b0" +dependencies = [ + "cranelift-assembler-x64-meta", + "cranelift-codegen-shared", + "cranelift-srcgen", + "heck", + "pulley-interpreter", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78fdb83ab012d0ee6a44ced7ca8788a444f17cf821c62f95d6ef87c9f0262518" + +[[package]] +name = "cranelift-control" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b75adc6eb7bb4ac6365106afb6cac4f12fe1ddfa02ddc9fd7015ca1469b471b" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "cranelift-entity" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668e56db75a54816cbdd7c7b7bfc558b08bf7b2cda9d0846491517e92f3b393b" +dependencies = [ + "cranelift-bitset", + "serde", + "serde_derive", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-frontend" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c63892dc1cc3ae48680183fa66997f60ffe7f1e200c8d390f8ee66edff4aef5a" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94eaf429c32a12715429c7c6ddfdd43c170f4cdd7e97bfa507bd68a652091087" + +[[package]] +name = "cranelift-native" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd77674904ae9be11c1e1efdba54788b59f3d6658d747b97534bfbba2909aacc" +dependencies = [ + "cranelift-codegen", + "libc", + "target-lexicon", +] + +[[package]] +name = "cranelift-srcgen" +version = "0.132.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cba7c0ff5941842c36653da155580ce41e675c204a67ac1b4e1c478a9347bbb7" + [[package]] name = "crc" version = "3.4.0" @@ -1239,6 +1473,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "directories-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dispatch2" version = "0.3.1" @@ -1367,6 +1622,15 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eef75b364b1baff88ff28dc34e4c7c0ebd138abd76f4e58e24e37d9b7f54b8f1" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -1699,6 +1963,17 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-set-times" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" +dependencies = [ + "io-lifetimes", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -1822,6 +2097,20 @@ dependencies = [ "byteorder", ] +[[package]] +name = "fxprof-processed-profile" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25234f20a3ec0a962a61770cfe39ecf03cb529a6e474ad8cff025ed497eda557" +dependencies = [ + "bitflags 2.13.0", + "debugid", + "rustc-hash", + "serde", + "serde_derive", + "serde_json", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1879,6 +2168,18 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "gimli" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" +dependencies = [ + "fnv", + "hashbrown 0.16.1", + "indexmap", + "stable_deref_trait", +] + [[package]] name = "glob" version = "0.3.3" @@ -1957,6 +2258,8 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", + "serde", + "serde_core", ] [[package]] @@ -2653,6 +2956,22 @@ dependencies = [ "rustversion", ] +[[package]] +name = "io-extras" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2285ddfe3054097ef4b2fe909ef8c3bcd1ea52a8f0d274416caebeef39f04a65" +dependencies = [ + "io-lifetimes", + "windows-sys 0.59.0", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" + [[package]] name = "ipnet" version = "2.12.0" @@ -2698,6 +3017,26 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "ittapi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b996fe614c41395cdaedf3cf408a9534851090959d90d54a535f675550b64b1" +dependencies = [ + "anyhow", + "ittapi-sys", + "log", +] + +[[package]] +name = "ittapi-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5385394064fa2c886205dba02598013ce83d3e92d33dbdc0c52fe0e7bf4fc" +dependencies = [ + "cc", +] + [[package]] name = "jni" version = "0.22.4" @@ -2806,6 +3145,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c83bff1d572d6b9aeef67ddfc8448e4a3737909cb28e81f97c791b9018703e52" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -2834,6 +3179,15 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libredox" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +dependencies = [ + "libc", +] + [[package]] name = "libsqlite3-sys" version = "0.37.0" @@ -3019,6 +3373,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "maplit" version = "1.0.2" @@ -3034,6 +3397,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + [[package]] name = "md-5" version = "0.10.6" @@ -3050,6 +3419,15 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +[[package]] +name = "memfd" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" +dependencies = [ + "rustix", +] + [[package]] name = "memmap2" version = "0.9.10" @@ -3527,6 +3905,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "crc32fast", + "hashbrown 0.17.1", + "indexmap", + "memchr", +] + [[package]] name = "object_store" version = "0.13.2" @@ -4043,6 +4433,8 @@ dependencies = [ "tokio", "tracing", "walk", + "wasmtime", + "wasmtime-wasi", ] [[package]] @@ -4339,6 +4731,29 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "pulley-interpreter" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d9880c1985ccccaed3646b0ef793dc39a4b117403ed4afc6fa3ef6027c5200f" +dependencies = [ + "cranelift-bitset", + "log", + "pulley-macros", + "wasmtime-internal-core", +] + +[[package]] +name = "pulley-macros" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee249346855ad102580e474da5463f86f8a7d449e6d49e00fefb304e448e2983" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "quick-xml" version = "0.39.4" @@ -4682,10 +5097,21 @@ dependencies = [ ] [[package]] -name = "ref-cast" -version = "1.0.25" +name = "redox_users" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ "ref-cast-impl", ] @@ -4701,6 +5127,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "regalloc2" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de2c52737737f8609e94f975dee22854a2d5c125772d4b1cf292120f4d45c186" +dependencies = [ + "allocator-api2", + "bumpalo", + "hashbrown 0.17.1", + "log", + "rustc-hash", + "smallvec", +] + [[package]] name = "regex" version = "1.12.4" @@ -4923,6 +5363,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustix-linux-procfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" +dependencies = [ + "once_cell", + "rustix", +] + [[package]] name = "rustls" version = "0.23.40" @@ -5150,6 +5600,10 @@ name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] [[package]] name = "sequence_trie" @@ -5383,6 +5837,9 @@ name = "smallvec" version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] [[package]] name = "smart-default" @@ -5738,6 +6195,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + [[package]] name = "telemetry" version = "0.1.0" @@ -5770,6 +6233,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminfo" version = "0.9.0" @@ -6020,6 +6492,7 @@ version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ + "indexmap", "serde_core", "serde_spanned", "toml_datetime 0.7.5+spec-1.1.0", @@ -6492,6 +6965,23 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-compose" +version = "0.248.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96ba953e2b9b4b4b52a31cf4e3ee1c1374c872b6e012cf2138d1c37cba00bfd6" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "log", + "petgraph", + "smallvec", + "wasm-encoder 0.248.0", + "wasmparser 0.248.0", + "wat", +] + [[package]] name = "wasm-encoder" version = "0.244.0" @@ -6499,7 +6989,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" dependencies = [ "leb128fmt", - "wasmparser", + "wasmparser 0.244.0", +] + +[[package]] +name = "wasm-encoder" +version = "0.248.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac92cf547bc18d27ecc521015c08c353b4f18b84ab388bb6d1b6b682c620d9b6" +dependencies = [ + "leb128fmt", + "wasmparser 0.248.0", +] + +[[package]] +name = "wasm-encoder" +version = "0.252.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8185ae345fa5687c054626ff9a50e7089797a343d9904d1dc9820eb4c4d3196f" +dependencies = [ + "leb128fmt", + "wasmparser 0.252.0", ] [[package]] @@ -6510,8 +7020,8 @@ checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", "indexmap", - "wasm-encoder", - "wasmparser", + "wasm-encoder 0.244.0", + "wasmparser 0.244.0", ] [[package]] @@ -6539,6 +7049,372 @@ dependencies = [ "semver", ] +[[package]] +name = "wasmparser" +version = "0.248.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4439c5eee9df71ee0c6efb37f63b1fcb1fec38f85f5142c54e7ed05d33091a" +dependencies = [ + "bitflags 2.13.0", + "hashbrown 0.17.1", + "indexmap", + "semver", + "serde", +] + +[[package]] +name = "wasmparser" +version = "0.252.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3eb099dcadcde5be9eef55e3a337128efd4e44b4c93122487e4d2e4e1c6627c" +dependencies = [ + "bitflags 2.13.0", + "indexmap", + "semver", +] + +[[package]] +name = "wasmprinter" +version = "0.248.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b264a5410b008d4d199a92bf536eae703cbd614482fc1ec53831cf19e1c183" +dependencies = [ + "anyhow", + "termcolor", + "wasmparser 0.248.0", +] + +[[package]] +name = "wasmtime" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7ce9aa2c67f75fadcfdc6aa9097d03e7c39485dfe316f2ed6a7c0fd186c527" +dependencies = [ + "addr2line 0.26.1", + "async-trait", + "bitflags 2.13.0", + "bumpalo", + "cc", + "cfg-if", + "encoding_rs", + "futures", + "fxprof-processed-profile", + "gimli 0.33.0", + "ittapi", + "libc", + "log", + "mach2", + "memfd", + "object 0.39.1", + "once_cell", + "postcard", + "pulley-interpreter", + "rayon", + "rustix", + "semver", + "serde", + "serde_derive", + "serde_json", + "smallvec", + "target-lexicon", + "tempfile", + "wasm-compose", + "wasm-encoder 0.248.0", + "wasmparser 0.248.0", + "wasmtime-environ", + "wasmtime-internal-cache", + "wasmtime-internal-component-macro", + "wasmtime-internal-component-util", + "wasmtime-internal-core", + "wasmtime-internal-cranelift", + "wasmtime-internal-fiber", + "wasmtime-internal-jit-debug", + "wasmtime-internal-jit-icache-coherence", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", + "wasmtime-internal-winch", + "wat", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-environ" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fb157bd1fbf689ac89d570433a700db6f33bdfcb5ffc30e3f1c49e4c70de71" +dependencies = [ + "anyhow", + "cpp_demangle", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-entity", + "gimli 0.33.0", + "hashbrown 0.17.1", + "indexmap", + "log", + "object 0.39.1", + "postcard", + "rustc-demangle", + "semver", + "serde", + "serde_derive", + "sha2", + "smallvec", + "target-lexicon", + "wasm-encoder 0.248.0", + "wasmparser 0.248.0", + "wasmprinter", + "wasmtime-internal-component-util", + "wasmtime-internal-core", +] + +[[package]] +name = "wasmtime-internal-cache" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d1a46c4a2360186b59c6ed7a74a1121ac97925ae9a18db1b2f146cc27ac0b7" +dependencies = [ + "base64 0.22.1", + "directories-next", + "log", + "postcard", + "rustix", + "serde", + "serde_derive", + "sha2", + "toml", + "wasmtime-environ", + "windows-sys 0.61.2", + "zstd", +] + +[[package]] +name = "wasmtime-internal-component-macro" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96c17f35fae2ab574667aba0c58fd56349a6f788ac42541a2e543116d5cfb91" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn 2.0.117", + "wasmtime-internal-component-util", + "wasmtime-internal-wit-bindgen", + "wit-parser 0.248.0", +] + +[[package]] +name = "wasmtime-internal-component-util" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d2eeb9b53222859e6f5dc73d2ccfb33254d672469cac11b693a71912e2f3817" + +[[package]] +name = "wasmtime-internal-core" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1deaf6bc3430abd7497b00c64f06ca2b97ca0fe41af87836446ca30949965c" +dependencies = [ + "anyhow", + "hashbrown 0.17.1", + "libm", + "serde", +] + +[[package]] +name = "wasmtime-internal-cranelift" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b845f83b5b04b11bc48329b53eb4fa8cf9f28a43c71ed8e1203f68ffa9806d1b" +dependencies = [ + "cfg-if", + "cranelift-codegen", + "cranelift-control", + "cranelift-entity", + "cranelift-frontend", + "cranelift-native", + "gimli 0.33.0", + "itertools 0.14.0", + "log", + "object 0.39.1", + "pulley-interpreter", + "smallvec", + "target-lexicon", + "thiserror 2.0.18", + "wasmparser 0.248.0", + "wasmtime-environ", + "wasmtime-internal-core", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", +] + +[[package]] +name = "wasmtime-internal-fiber" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10c8466f72965ae85c250f90aaa7992c089a2f8502009bd0d2c9e7d6409174a" +dependencies = [ + "cc", + "cfg-if", + "libc", + "rustix", + "wasmtime-environ", + "wasmtime-internal-versioned-export-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-internal-jit-debug" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d3adfecf5621b14d8f8871f4cb4ed9f844197b1ddefc702ef4c859552cd9551" +dependencies = [ + "cc", + "object 0.39.1", + "rustix", + "wasmtime-internal-versioned-export-macros", +] + +[[package]] +name = "wasmtime-internal-jit-icache-coherence" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d3c1e9fb618ec45c9b3477ea683cd37bee427273d7b13bba5c66a1caaf1dd6" +dependencies = [ + "cfg-if", + "libc", + "wasmtime-internal-core", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-internal-unwinder" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aa91132b81f1e172ec7e7c3c114ac34209ee6b3524b3a8d6943af99803f66c5" +dependencies = [ + "cfg-if", + "cranelift-codegen", + "log", + "object 0.39.1", + "wasmtime-environ", +] + +[[package]] +name = "wasmtime-internal-versioned-export-macros" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea811ffe23f597cc7708327ea25d9eb018dcf760ffe15ccb7d0b27ad635de61" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "wasmtime-internal-winch" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "828b66175c54a0d00b4c1c1c76658d8aa73aeb9fa3553575c5eee56d40f2eb18" +dependencies = [ + "cranelift-codegen", + "gimli 0.33.0", + "log", + "object 0.39.1", + "target-lexicon", + "wasmparser 0.248.0", + "wasmtime-environ", + "wasmtime-internal-cranelift", + "winch-codegen", +] + +[[package]] +name = "wasmtime-internal-wit-bindgen" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae00896ad9bef1b3ca6401ae9a841daa6f357dd91541b6baf87082946d1bde1" +dependencies = [ + "anyhow", + "bitflags 2.13.0", + "heck", + "indexmap", + "wit-parser 0.248.0", +] + +[[package]] +name = "wasmtime-wasi" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6032ceffcb74cf30a2cdac298a4d6ce219058f08479ce1ab38434aadc044000b" +dependencies = [ + "async-trait", + "bitflags 2.13.0", + "bytes", + "cap-fs-ext", + "cap-net-ext", + "cap-std", + "cap-time-ext", + "cfg-if", + "fs-set-times", + "futures", + "io-extras", + "io-lifetimes", + "rand 0.10.1", + "rustix", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "wasmtime", + "wasmtime-wasi-io", + "wiggle", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-wasi-io" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b6e6868e5b93e1e10983a17afb631b39c236d8b6b4abe9faffe78f1ee0c6e7" +dependencies = [ + "async-trait", + "bytes", + "futures", + "tracing", + "wasmtime", +] + +[[package]] +name = "wast" +version = "35.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ef140f1b49946586078353a453a1d28ba90adfc54dde75710bc1931de204d68" +dependencies = [ + "leb128", +] + +[[package]] +name = "wast" +version = "252.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942a3449d6a593fccc111a6241c8df52bda168af30e40bf9580d4394d7374c65" +dependencies = [ + "bumpalo", + "leb128fmt", + "memchr", + "unicode-width 0.2.2", + "wasm-encoder 0.252.0", +] + +[[package]] +name = "wat" +version = "1.252.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c72a4ba7088f7bac94cf516e49882bdf97068904a563768cf249efc839ec42cb" +dependencies = [ + "wast 252.0.0", +] + [[package]] name = "wax" version = "0.7.0" @@ -6664,6 +7540,46 @@ dependencies = [ "libc", ] +[[package]] +name = "wiggle" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "176527a028d6a426a514e3ca650c251a60541cde26df421f781339f27553ff9f" +dependencies = [ + "bitflags 2.13.0", + "thiserror 2.0.18", + "tracing", + "wasmtime", + "wasmtime-environ", + "wiggle-macro", +] + +[[package]] +name = "wiggle-generate" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604976b16d40f15606ae47ca22473c7574b317a6445ea2e3986f834a2ca0f449" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", + "wasmtime-environ", + "witx", +] + +[[package]] +name = "wiggle-macro" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7252f1689c33cf77cfac6115047c6a8b53f188c25c644f7856ad66c881c4077" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "wiggle-generate", +] + [[package]] name = "winapi" version = "0.3.9" @@ -6695,6 +7611,25 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winch-codegen" +version = "45.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89c09acfdfa281b3340e1e94ef3cf6618d69eab975280f881e154c29f49419c1" +dependencies = [ + "cranelift-assembler-x64", + "cranelift-codegen", + "gimli 0.33.0", + "regalloc2", + "smallvec", + "target-lexicon", + "thiserror 2.0.18", + "wasmparser 0.248.0", + "wasmtime-environ", + "wasmtime-internal-core", + "wasmtime-internal-cranelift", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -7000,6 +7935,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winx" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" +dependencies = [ + "bitflags 2.13.0", + "windows-sys 0.59.0", +] + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -7023,7 +7968,7 @@ checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ "anyhow", "heck", - "wit-parser", + "wit-parser 0.244.0", ] [[package]] @@ -7070,10 +8015,10 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "wasm-encoder", + "wasm-encoder 0.244.0", "wasm-metadata", - "wasmparser", - "wit-parser", + "wasmparser 0.244.0", + "wit-parser 0.244.0", ] [[package]] @@ -7091,7 +8036,38 @@ dependencies = [ "serde_derive", "serde_json", "unicode-xid", - "wasmparser", + "wasmparser 0.244.0", +] + +[[package]] +name = "wit-parser" +version = "0.248.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "247ad505da2915a082fe13204c5ba8788425aea1de54f43b284818cf82637856" +dependencies = [ + "anyhow", + "hashbrown 0.17.1", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.248.0", +] + +[[package]] +name = "witx" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e366f27a5cabcddb2706a78296a40b8fcc451e1a6aba2fc1d94b4a01bdaaef4b" +dependencies = [ + "anyhow", + "log", + "thiserror 1.0.69", + "wast 35.0.2", ] [[package]] @@ -7224,3 +8200,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 70a8f33e..63604437 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,9 @@ edition = "2024" [workspace] members = ["gen/proto", "crates/e2e", "crates/testkit", "crates/plugingo-e2e", "crates/htspec-derive", "crates/core", "crates/walk", "crates/proc", "crates/model", "crates/sandboxfuse", "crates/plugin", "crates/plugin-abi", "crates/plugin-sdk", "crates/plugin-remote", "crates/plugin-echo", "crates/builtins", "crates/plugin-buildfile", "crates/driver-support", "crates/plugin-exec", "crates/plugin-nix", "crates/plugin-query", "crates/plugin-go", "crates/telemetry", "crates/tui", "crates/lock", "crates/engine"] +# wasm guest components are built with cargo-component for wasm32-wasip2, not by +# the host `cargo build --workspace`; keep them out of the workspace. +exclude = ["crates/wasm-guests"] [profile.profiling] inherits = "release" diff --git a/crates/plugin-remote/Cargo.toml b/crates/plugin-remote/Cargo.toml index 6ef9f7c1..ff272e31 100644 --- a/crates/plugin-remote/Cargo.toml +++ b/crates/plugin-remote/Cargo.toml @@ -20,11 +20,13 @@ prost = "0.14" tokio = { version = "1.52", features = ["rt", "sync", "io-util", "macros", "net"] } tracing = "0.1" libc = "0.2" +wasmtime = { version = "45", optional = true } +wasmtime-wasi = { version = "45", optional = true } [features] # heavy transports, off by default; proto (UDS) is always available shm = ["plugin-abi/shm"] -wasm = [] +wasm = ["dep:wasmtime", "dep:wasmtime-wasi"] [dev-dependencies] plugin-sdk = { path = "../plugin-sdk" } diff --git a/crates/plugin-remote/src/host.rs b/crates/plugin-remote/src/host.rs index 6f5f8157..089271f7 100644 --- a/crates/plugin-remote/src/host.rs +++ b/crates/plugin-remote/src/host.rs @@ -101,7 +101,10 @@ impl InboundHandler for HostCallbackHandler { impl HostCallbackHandler { async fn handle_result(&self, id: u64, req: pb::ResultRequest, mux: &Arc) { let Some(scope) = self.inner.scope(&req.request_id) else { - mux.send_body(id, err_frame(format!("unknown request scope {}", req.request_id))); + mux.send_body( + id, + err_frame(format!("unknown request scope {}", req.request_id)), + ); return; }; let addr = convert::addr_from_pb(req.addr.unwrap_or_default()); @@ -140,7 +143,10 @@ impl HostCallbackHandler { async fn handle_note_dep(&self, id: u64, req: pb::NoteDepRequest, mux: &Arc) { let Some(scope) = self.inner.scope(&req.request_id) else { - mux.send_body(id, err_frame(format!("unknown request scope {}", req.request_id))); + mux.send_body( + id, + err_frame(format!("unknown request scope {}", req.request_id)), + ); return; }; let addr = convert::addr_from_pb(req.addr.unwrap_or_default()); @@ -162,7 +168,10 @@ impl HostCallbackHandler { async fn handle_query(&self, id: u64, req: pb::QueryRequest, mux: &Arc) { let Some(scope) = self.inner.scope(&req.request_id) else { - mux.send_body(id, err_frame(format!("unknown request scope {}", req.request_id))); + mux.send_body( + id, + err_frame(format!("unknown request scope {}", req.request_id)), + ); return; }; let matcher = convert::matcher_from_pb(req.matcher.unwrap_or_default()); @@ -180,7 +189,13 @@ impl HostCallbackHandler { async fn handle_open_artifact(&self, id: u64, req: pb::OpenArtifactRequest, mux: &Arc) { let idx: usize = req.handle_id.parse().unwrap_or(usize::MAX); let Some(content) = self.inner.leases.get(&req.lease_id, idx) else { - mux.send_body(id, err_frame(format!("unknown artifact {}#{}", req.lease_id, req.handle_id))); + mux.send_body( + id, + err_frame(format!( + "unknown artifact {}#{}", + req.lease_id, req.handle_id + )), + ); return; }; // Content::reader is sync; read on a blocking thread. M1 reads the whole diff --git a/crates/plugin-remote/src/lib.rs b/crates/plugin-remote/src/lib.rs index e610d093..aab55d02 100644 --- a/crates/plugin-remote/src/lib.rs +++ b/crates/plugin-remote/src/lib.rs @@ -21,7 +21,7 @@ pub use driver::RemoteDriver; pub use managed::RemoteManagedDriver; pub use provider::RemoteProvider; #[cfg(unix)] -pub use spawn::{spawn_plugin, spawn_streams, PLUGIN_FD}; +pub use spawn::{PLUGIN_FD, spawn_plugin, spawn_streams}; #[cfg(feature = "shm")] pub mod shm; diff --git a/crates/plugin-remote/src/provider.rs b/crates/plugin-remote/src/provider.rs index e218dbd6..626373c2 100644 --- a/crates/plugin-remote/src/provider.rs +++ b/crates/plugin-remote/src/provider.rs @@ -120,7 +120,9 @@ impl Provider for RemoteProvider { } } Ok(Box::new(out.into_iter()) - as Box> + Send>) + as Box< + dyn Iterator> + Send, + >) }) } @@ -158,9 +160,9 @@ impl Provider for RemoteProvider { to: req.addr.clone(), }, ))), - pb::get_error::Kind::Cancelled => { - Err(GetError::Other(anyhow::Error::new(hplugin::error::CancelledError))) - } + pb::get_error::Kind::Cancelled => Err(GetError::Other(anyhow::Error::new( + hplugin::error::CancelledError, + ))), _ => Err(GetError::Other(anyhow::anyhow!("{}", ge.message))), } } diff --git a/crates/plugin-remote/src/wasm.rs b/crates/plugin-remote/src/wasm.rs index ae9fc4b4..33d7f2e3 100644 --- a/crates/plugin-remote/src/wasm.rs +++ b/crates/plugin-remote/src/wasm.rs @@ -1,6 +1,70 @@ //! wasm transport (in-process wasmtime component) — milestone M4. //! -//! Will instantiate a plugin `.wasm` component via wasmtime, bind the WIT -//! interface (`wit/heph-plugin.wit`), wire the `AbiHost` callbacks as host -//! imports, and grant capabilities (WASI preopens) per the `launch` policy. -//! Scaffolded here so the feature wiring exists from M1; implemented in M4. +//! Instantiates a plugin `.wasm` component via wasmtime, binds the WIT +//! interface, and (eventually) wires the `AbiHost` callbacks as host imports +//! with capability-scoped WASI preopens per the `launch` policy. +//! +//! This module currently carries the de-risk vertical slice: load a component +//! that exports `greet`, link WASI, instantiate, and call it. It proves the +//! cargo-component guest ↔ wasmtime host contract end-to-end before the full +//! provider/driver WIT is brought up. + +use anyhow::{Context, Result}; +use wasmtime::component::{Component, Linker, ResourceTable}; +use wasmtime::{Config, Engine, Store}; +use wasmtime_wasi::{WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView}; + +mod echo_bindings { + wasmtime::component::bindgen!({ + inline: "package component:echo;\nworld echo {\n export greet: func(name: string) -> string;\n}", + world: "echo", + }); +} + +/// Per-instance host state: the WASI context plus the resource table wasmtime +/// uses to hand out handles to host-owned resources. +struct HostState { + table: ResourceTable, + wasi: WasiCtx, +} + +impl WasiView for HostState { + fn ctx(&mut self) -> WasiCtxView<'_> { + WasiCtxView { + ctx: &mut self.wasi, + table: &mut self.table, + } + } +} + +/// De-risk slice: instantiate `wasm` (a component exporting `greet`) and call +/// `greet(name)`. Synchronous — `greet` is pure compute, no host imports beyond +/// the WASI shims the component links against. +pub fn instantiate_and_greet(wasm: &[u8], name: &str) -> Result { + let mut config = Config::new(); + config.wasm_component_model(true); + let engine = Engine::new(&config) + .map_err(anyhow::Error::from) + .context("building wasmtime engine")?; + let component = Component::from_binary(&engine, wasm) + .map_err(anyhow::Error::from) + .context("loading wasm component")?; + + let mut linker = Linker::::new(&engine); + wasmtime_wasi::p2::add_to_linker_sync(&mut linker) + .map_err(anyhow::Error::from) + .context("linking WASI imports")?; + + let state = HostState { + table: ResourceTable::new(), + wasi: WasiCtxBuilder::new().build(), + }; + let mut store = Store::new(&engine, state); + + let echo = echo_bindings::Echo::instantiate(&mut store, &component, &linker) + .map_err(anyhow::Error::from) + .context("instantiating echo component")?; + echo.call_greet(&mut store, name) + .map_err(anyhow::Error::from) + .context("calling greet export") +} diff --git a/crates/plugin-remote/tests/proto_e2e.rs b/crates/plugin-remote/tests/proto_e2e.rs index ae7fb012..fac01b7c 100644 --- a/crates/plugin-remote/tests/proto_e2e.rs +++ b/crates/plugin-remote/tests/proto_e2e.rs @@ -7,15 +7,15 @@ use futures::future::BoxFuture; use hcore::hartifactcontent::{Content, WalkEntry, WalkEntryKind}; use hcore::hasync::{Cancellable, StdCancellationToken}; +use hmodel::htaddr::Addr; +use hmodel::htmatcher::Matcher; +use hmodel::htpkg::PkgBuf; use hplugin::eresult::{ArtifactMeta, EResult}; use hplugin::provider::{ ConfigRequest, ConfigResponse, GetError, GetRequest, GetResponse, ListPackageResponse, ListPackagesRequest, ListRequest, ListResponse, ProbeRequest, ProbeResponse, Provider, ProviderExecutor, TargetSpec, }; -use hmodel::htaddr::Addr; -use hmodel::htmatcher::Matcher; -use hmodel::htpkg::PkgBuf; use plugin_remote::RemoteProvider; use std::collections::BTreeMap; use std::io::{Cursor, Read}; @@ -55,7 +55,9 @@ impl Provider for TestProvider { }), ]; Ok(Box::new(items.into_iter()) - as Box> + Send>) + as Box< + dyn Iterator> + Send, + >) }) } @@ -72,7 +74,9 @@ impl Provider for TestProvider { pkg: PkgBuf::from("//pkg"), })]; Ok(Box::new(items.into_iter()) - as Box> + Send>) + as Box< + dyn Iterator> + Send, + >) }) } @@ -84,11 +88,7 @@ impl Provider for TestProvider { Box::pin(async move { // Call back into the host to resolve a dependency. let dep = addr("//dep", "lib"); - let eres = req - .executor - .result(&dep) - .await - .map_err(GetError::Other)?; + let eres = req.executor.result(&dep).await.map_err(GetError::Other)?; // Verify the callback round-tripped the artifact metadata. let got = eres .artifacts_meta @@ -597,8 +597,14 @@ async fn multi_driver_routing_one_connection() { let (br, bw) = b.into_split(); let mut map: HashMap> = HashMap::new(); - map.insert("alpha".to_string(), Arc::new(NamedDriver("alpha".to_string()))); - map.insert("beta".to_string(), Arc::new(NamedDriver("beta".to_string()))); + map.insert( + "alpha".to_string(), + Arc::new(NamedDriver("alpha".to_string())), + ); + map.insert( + "beta".to_string(), + Arc::new(NamedDriver("beta".to_string())), + ); let _guest = plugin_sdk::serve_components(None, map, br, bw); // One connection, two driver handles selected by name. @@ -660,7 +666,9 @@ impl Provider for CycleTestProvider { { Box::pin(async move { Ok(Box::new(std::iter::empty()) - as Box> + Send>) + as Box< + dyn Iterator> + Send, + >) }) } fn list_packages<'a>( @@ -673,7 +681,9 @@ impl Provider for CycleTestProvider { > { Box::pin(async move { Ok(Box::new(std::iter::empty()) - as Box> + Send>) + as Box< + dyn Iterator> + Send, + >) }) } fn get<'a>( diff --git a/crates/plugin-remote/tests/shm_e2e.rs b/crates/plugin-remote/tests/shm_e2e.rs index 8da61156..2f8d4417 100644 --- a/crates/plugin-remote/tests/shm_e2e.rs +++ b/crates/plugin-remote/tests/shm_e2e.rs @@ -9,13 +9,13 @@ use futures::future::BoxFuture; use hcore::hasync::{Cancellable, StdCancellationToken}; use hmodel::htaddr::Addr; +use hmodel::htmatcher::Matcher; use hmodel::htpkg::PkgBuf; use hplugin::provider::{ ConfigRequest, ConfigResponse, GetError, GetRequest, GetResponse, ListPackageResponse, ListPackagesRequest, ListRequest, ListResponse, ProbeRequest, ProbeResponse, Provider, ProviderExecutor, TargetSpec, }; -use hmodel::htmatcher::Matcher; use std::collections::BTreeMap; use std::sync::Arc; @@ -42,7 +42,9 @@ impl Provider for TestProvider { addr: addr("pkg", "a"), })]; Ok(Box::new(items.into_iter()) - as Box> + Send>) + as Box< + dyn Iterator> + Send, + >) }) } fn list_packages<'a>( @@ -55,7 +57,9 @@ impl Provider for TestProvider { > { Box::pin(async move { Ok(Box::new(std::iter::empty()) - as Box> + Send>) + as Box< + dyn Iterator> + Send, + >) }) } fn get<'a>( @@ -111,7 +115,10 @@ async fn provider_over_shm() { let _guest = plugin_sdk::serve(Arc::new(TestProvider), gr, gw); let host = plugin_remote::RemoteProvider::connect(hr, hw, "shm-test"); - assert_eq!(host.config(ConfigRequest {}).expect("config").name, "shm-test"); + assert_eq!( + host.config(ConfigRequest {}).expect("config").name, + "shm-test" + ); let ctoken = StdCancellationToken::new(); let executor: Arc = Arc::new(NoopExec); diff --git a/crates/plugin-remote/tests/wasm_e2e.rs b/crates/plugin-remote/tests/wasm_e2e.rs new file mode 100644 index 00000000..8ef1e158 --- /dev/null +++ b/crates/plugin-remote/tests/wasm_e2e.rs @@ -0,0 +1,50 @@ +//! wasm transport vertical slice (M4 de-risk). +//! +//! Builds the `echo` guest component with cargo-component, loads it through the +//! wasmtime host in `plugin_remote::wasm`, and calls its `greet` export. Proves +//! the cargo-component guest ↔ wasmtime host contract end-to-end before the full +//! provider/driver WIT is brought up. +//! +//! Gated behind `--features wasm` (the wasm toolchain — cargo-component, +//! wasm32-wasip1 std — is only present in the devenv shell). Run with: +//! devenv shell -- cargo test -p plugin-remote --features wasm +#![cfg(feature = "wasm")] + +use std::path::PathBuf; +use std::process::Command; + +/// Build the echo guest component into `out_dir` and return the path to the +/// produced `.wasm` component. +fn build_echo_component(out_dir: &std::path::Path) -> PathBuf { + let guest_manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../wasm-guests/echo/Cargo.toml") + .canonicalize() + .expect("echo guest manifest exists"); + + let status = Command::new("cargo") + .args(["component", "build"]) + .arg("--manifest-path") + .arg(&guest_manifest) + .arg("--target") + .arg("wasm32-wasip1") + .arg("--target-dir") + .arg(out_dir) + .status() + .expect("cargo-component is on PATH (run inside `devenv shell`)"); + assert!(status.success(), "cargo component build failed"); + + let wasm = out_dir.join("wasm32-wasip1/debug/echo.wasm"); + assert!(wasm.exists(), "expected component at {}", wasm.display()); + wasm +} + +#[test] +fn echo_component_greets_over_wasmtime_host() { + let tmp = tempfile::tempdir().expect("tempdir"); + let wasm_path = build_echo_component(tmp.path()); + let wasm = std::fs::read(&wasm_path).expect("read built component"); + + let out = plugin_remote::wasm::instantiate_and_greet(&wasm, "heph") + .expect("instantiate + call greet"); + assert_eq!(out, "hello heph"); +} diff --git a/crates/wasm-guests/echo/.gitignore b/crates/wasm-guests/echo/.gitignore new file mode 100644 index 00000000..ea8c4bf7 --- /dev/null +++ b/crates/wasm-guests/echo/.gitignore @@ -0,0 +1 @@ +/target diff --git a/crates/wasm-guests/echo/Cargo.lock b/crates/wasm-guests/echo/Cargo.lock new file mode 100644 index 00000000..93d0c723 --- /dev/null +++ b/crates/wasm-guests/echo/Cargo.lock @@ -0,0 +1,25 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "echo" +version = "0.1.0" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "653c85dd7aee6fe6f4bded0d242406deadae9819029ce6f7d258c920c384358a" +dependencies = [ + "bitflags", +] diff --git a/crates/wasm-guests/echo/Cargo.toml b/crates/wasm-guests/echo/Cargo.toml new file mode 100644 index 00000000..23e80e6d --- /dev/null +++ b/crates/wasm-guests/echo/Cargo.toml @@ -0,0 +1,20 @@ +# Standalone workspace root: this guest is built with cargo-component for +# wasm32-wasip2, kept out of the host workspace. The empty table stops cargo +# from walking up into the parent repo's workspace. +[workspace] + +[package] +name = "echo" +version = "0.1.0" +edition = "2024" + +[dependencies] +wit-bindgen-rt = { version = "0.44.0", features = ["bitflags"] } + +[lib] +crate-type = ["cdylib"] + +[package.metadata.component] +package = "component:echo" + +[package.metadata.component.dependencies] diff --git a/crates/wasm-guests/echo/src/bindings.rs b/crates/wasm-guests/echo/src/bindings.rs new file mode 100644 index 00000000..7f8409ad --- /dev/null +++ b/crates/wasm-guests/echo/src/bindings.rs @@ -0,0 +1,119 @@ +// Generated by `wit-bindgen` 0.41.0. DO NOT EDIT! +// Options used: +// * runtime_path: "wit_bindgen_rt" +#[doc(hidden)] +#[allow(non_snake_case)] +pub unsafe fn _export_greet_cabi(arg0: *mut u8, arg1: usize) -> *mut u8 { + #[cfg(target_arch = "wasm32")] _rt::run_ctors_once(); + let len0 = arg1; + let bytes0 = _rt::Vec::from_raw_parts(arg0.cast(), len0, len0); + let result1 = T::greet(_rt::string_lift(bytes0)); + let ptr2 = (&raw mut _RET_AREA.0).cast::(); + let vec3 = (result1.into_bytes()).into_boxed_slice(); + let ptr3 = vec3.as_ptr().cast::(); + let len3 = vec3.len(); + ::core::mem::forget(vec3); + *ptr2.add(::core::mem::size_of::<*const u8>()).cast::() = len3; + *ptr2.add(0).cast::<*mut u8>() = ptr3.cast_mut(); + ptr2 +} +#[doc(hidden)] +#[allow(non_snake_case)] +pub unsafe fn __post_return_greet(arg0: *mut u8) { + let l0 = *arg0.add(0).cast::<*mut u8>(); + let l1 = *arg0.add(::core::mem::size_of::<*const u8>()).cast::(); + _rt::cabi_dealloc(l0, l1, 1); +} +pub trait Guest { + fn greet(name: _rt::String) -> _rt::String; +} +#[doc(hidden)] +macro_rules! __export_world_echo_cabi { + ($ty:ident with_types_in $($path_to_types:tt)*) => { + const _ : () = { #[unsafe (export_name = "greet")] unsafe extern "C" fn + export_greet(arg0 : * mut u8, arg1 : usize,) -> * mut u8 { unsafe { + $($path_to_types)*:: _export_greet_cabi::<$ty > (arg0, arg1) } } #[unsafe + (export_name = "cabi_post_greet")] unsafe extern "C" fn _post_return_greet(arg0 : + * mut u8,) { unsafe { $($path_to_types)*:: __post_return_greet::<$ty > (arg0) } } + }; + }; +} +#[doc(hidden)] +pub(crate) use __export_world_echo_cabi; +#[cfg_attr(target_pointer_width = "64", repr(align(8)))] +#[cfg_attr(target_pointer_width = "32", repr(align(4)))] +struct _RetArea([::core::mem::MaybeUninit; 2 * ::core::mem::size_of::<*const u8>()]); +static mut _RET_AREA: _RetArea = _RetArea( + [::core::mem::MaybeUninit::uninit(); 2 * ::core::mem::size_of::<*const u8>()], +); +#[rustfmt::skip] +mod _rt { + #![allow(dead_code, clippy::all)] + #[cfg(target_arch = "wasm32")] + pub fn run_ctors_once() { + wit_bindgen_rt::run_ctors_once(); + } + pub use alloc_crate::vec::Vec; + pub unsafe fn string_lift(bytes: Vec) -> String { + if cfg!(debug_assertions) { + String::from_utf8(bytes).unwrap() + } else { + String::from_utf8_unchecked(bytes) + } + } + pub unsafe fn cabi_dealloc(ptr: *mut u8, size: usize, align: usize) { + if size == 0 { + return; + } + let layout = alloc::Layout::from_size_align_unchecked(size, align); + alloc::dealloc(ptr, layout); + } + pub use alloc_crate::string::String; + extern crate alloc as alloc_crate; + pub use alloc_crate::alloc; +} +/// Generates `#[unsafe(no_mangle)]` functions to export the specified type as +/// the root implementation of all generated traits. +/// +/// For more information see the documentation of `wit_bindgen::generate!`. +/// +/// ```rust +/// # macro_rules! export{ ($($t:tt)*) => (); } +/// # trait Guest {} +/// struct MyType; +/// +/// impl Guest for MyType { +/// // ... +/// } +/// +/// export!(MyType); +/// ``` +#[allow(unused_macros)] +#[doc(hidden)] +macro_rules! __export_echo_impl { + ($ty:ident) => { + self::export!($ty with_types_in self); + }; + ($ty:ident with_types_in $($path_to_types_root:tt)*) => { + $($path_to_types_root)*:: __export_world_echo_cabi!($ty with_types_in + $($path_to_types_root)*); + }; +} +#[doc(inline)] +pub(crate) use __export_echo_impl as export; +#[cfg(target_arch = "wasm32")] +#[unsafe( + link_section = "component-type:wit-bindgen:0.41.0:component:echo:echo:encoded world" +)] +#[doc(hidden)] +#[allow(clippy::octal_escapes)] +pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; 173] = *b"\ +\0asm\x0d\0\x01\0\0\x19\x16wit-component-encoding\x04\0\x073\x01A\x02\x01A\x02\x01\ +@\x01\x04names\0s\x04\0\x05greet\x01\0\x04\0\x13component:echo/echo\x04\0\x0b\x0a\ +\x01\0\x04echo\x03\0\0\0G\x09producers\x01\x0cprocessed-by\x02\x0dwit-component\x07\ +0.227.1\x10wit-bindgen-rust\x060.41.0"; +#[inline(never)] +#[doc(hidden)] +pub fn __link_custom_section_describing_imports() { + wit_bindgen_rt::maybe_link_cabi_realloc(); +} diff --git a/crates/wasm-guests/echo/src/lib.rs b/crates/wasm-guests/echo/src/lib.rs new file mode 100644 index 00000000..39c259a4 --- /dev/null +++ b/crates/wasm-guests/echo/src/lib.rs @@ -0,0 +1,14 @@ +#[allow(warnings)] +mod bindings; + +use bindings::Guest; + +struct Component; + +impl Guest for Component { + fn greet(name: String) -> String { + format!("hello {name}") + } +} + +bindings::export!(Component with_types_in bindings); diff --git a/crates/wasm-guests/echo/wit/world.wit b/crates/wasm-guests/echo/wit/world.wit new file mode 100644 index 00000000..567a828d --- /dev/null +++ b/crates/wasm-guests/echo/wit/world.wit @@ -0,0 +1,7 @@ +package component:echo; + +/// Minimal de-risk world: a single greet export to validate the +/// cargo-component guest + wasmtime host vertical slice. +world echo { + export greet: func(name: string) -> string; +} diff --git a/devenv.nix b/devenv.nix index ef435e63..62b83d55 100644 --- a/devenv.nix +++ b/devenv.nix @@ -45,7 +45,9 @@ in enable = true; channel = "stable"; components = [ "rustc" "cargo" "clippy" "rustfmt" "rust-analyzer" ]; - targets = [ "x86_64-apple-darwin" "aarch64-apple-darwin" "wasm32-wasip2" ] + # wasip1 is required by cargo-component's default build+adapt path even + # when the final component targets wasip2; keep both. + targets = [ "x86_64-apple-darwin" "aarch64-apple-darwin" "wasm32-wasip1" "wasm32-wasip2" ] ++ lib.optionals pkgs.stdenv.isLinux [ "x86_64-unknown-linux-gnu" "aarch64-unknown-linux-gnu" ]; }; From e271955d863c6a60e9e2e14da39cb6cf9553f36e Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 23:08:38 +0200 Subject: [PATCH 17/44] =?UTF-8?q?feat(plugin-remote):=20prove=20bidirectio?= =?UTF-8?q?nal=20wasm=20callbacks=20(guest=E2=86=92host=20import)=20[M4]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the echo de-risk slice to exercise BOTH call directions over the wasm transport, which is the crux of the plugin design: the guest provider calls back into the engine's executor mid-request. - echo world gains `import host-lookup: func(key) -> string`; guest `greet` invokes it and folds the host's answer into its reply. - host implements EchoImports for HostState and links it via Echo::add_to_linker (HasSelf marker). - inline WIT in the host bindgen kept in sync with the guest's world.wit. - e2e asserts the round-trip: greet("heph") == "hello heph (host:heph)". Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-remote/src/wasm.rs | 18 +++++++- crates/plugin-remote/tests/wasm_e2e.rs | 4 +- crates/wasm-guests/echo/src/bindings.rs | 57 ++++++++++++++++++++----- crates/wasm-guests/echo/src/lib.rs | 5 ++- crates/wasm-guests/echo/wit/world.wit | 12 +++++- 5 files changed, 81 insertions(+), 15 deletions(-) diff --git a/crates/plugin-remote/src/wasm.rs b/crates/plugin-remote/src/wasm.rs index 33d7f2e3..1a46d181 100644 --- a/crates/plugin-remote/src/wasm.rs +++ b/crates/plugin-remote/src/wasm.rs @@ -16,7 +16,8 @@ use wasmtime_wasi::{WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView}; mod echo_bindings { wasmtime::component::bindgen!({ - inline: "package component:echo;\nworld echo {\n export greet: func(name: string) -> string;\n}", + // Must stay in sync with crates/wasm-guests/echo/wit/world.wit. + inline: "package component:echo;\nworld echo {\n import host-lookup: func(key: string) -> string;\n export greet: func(name: string) -> string;\n}", world: "echo", }); } @@ -37,6 +38,15 @@ impl WasiView for HostState { } } +// Host-side implementation of the world's `host-lookup` import — the guest +// calls this back during `greet`. Stand-in for the real AbiHost executor +// surface (result/note_dep/query); proves the guest→host callback path. +impl echo_bindings::EchoImports for HostState { + fn host_lookup(&mut self, key: String) -> String { + format!("host:{key}") + } +} + /// De-risk slice: instantiate `wasm` (a component exporting `greet`) and call /// `greet(name)`. Synchronous — `greet` is pure compute, no host imports beyond /// the WASI shims the component links against. @@ -54,6 +64,12 @@ pub fn instantiate_and_greet(wasm: &[u8], name: &str) -> Result { wasmtime_wasi::p2::add_to_linker_sync(&mut linker) .map_err(anyhow::Error::from) .context("linking WASI imports")?; + echo_bindings::Echo::add_to_linker::<_, wasmtime::component::HasSelf>( + &mut linker, + |state| state, + ) + .map_err(anyhow::Error::from) + .context("linking host imports")?; let state = HostState { table: ResourceTable::new(), diff --git a/crates/plugin-remote/tests/wasm_e2e.rs b/crates/plugin-remote/tests/wasm_e2e.rs index 8ef1e158..781a2627 100644 --- a/crates/plugin-remote/tests/wasm_e2e.rs +++ b/crates/plugin-remote/tests/wasm_e2e.rs @@ -44,7 +44,9 @@ fn echo_component_greets_over_wasmtime_host() { let wasm_path = build_echo_component(tmp.path()); let wasm = std::fs::read(&wasm_path).expect("read built component"); + // greet() calls back into the host's `host-lookup` import (-> "host:heph") + // and folds it into the reply — exercising both call directions over wasm. let out = plugin_remote::wasm::instantiate_and_greet(&wasm, "heph") .expect("instantiate + call greet"); - assert_eq!(out, "hello heph"); + assert_eq!(out, "hello heph (host:heph)"); } diff --git a/crates/wasm-guests/echo/src/bindings.rs b/crates/wasm-guests/echo/src/bindings.rs index 7f8409ad..1cb3c792 100644 --- a/crates/wasm-guests/echo/src/bindings.rs +++ b/crates/wasm-guests/echo/src/bindings.rs @@ -1,6 +1,42 @@ // Generated by `wit-bindgen` 0.41.0. DO NOT EDIT! // Options used: // * runtime_path: "wit_bindgen_rt" +#[allow(unused_unsafe, clippy::all)] +/// Host-provided callback the guest invokes (stand-in for the AbiHost +/// executor surface: result/note_dep/query/...). +pub fn host_lookup(key: &str) -> _rt::String { + unsafe { + #[cfg_attr(target_pointer_width = "64", repr(align(8)))] + #[cfg_attr(target_pointer_width = "32", repr(align(4)))] + struct RetArea( + [::core::mem::MaybeUninit; 2 * ::core::mem::size_of::<*const u8>()], + ); + let mut ret_area = RetArea( + [::core::mem::MaybeUninit::uninit(); 2 * ::core::mem::size_of::<*const u8>()], + ); + let vec0 = key; + let ptr0 = vec0.as_ptr().cast::(); + let len0 = vec0.len(); + let ptr1 = ret_area.0.as_mut_ptr().cast::(); + #[cfg(target_arch = "wasm32")] + #[link(wasm_import_module = "$root")] + unsafe extern "C" { + #[link_name = "host-lookup"] + fn wit_import2(_: *mut u8, _: usize, _: *mut u8); + } + #[cfg(not(target_arch = "wasm32"))] + unsafe extern "C" fn wit_import2(_: *mut u8, _: usize, _: *mut u8) { + unreachable!() + } + unsafe { wit_import2(ptr0.cast_mut(), len0, ptr1) }; + let l3 = *ptr1.add(0).cast::<*mut u8>(); + let l4 = *ptr1.add(::core::mem::size_of::<*const u8>()).cast::(); + let len5 = l4; + let bytes5 = _rt::Vec::from_raw_parts(l3.cast(), len5, len5); + let result6 = _rt::string_lift(bytes5); + result6 + } +} #[doc(hidden)] #[allow(non_snake_case)] pub unsafe fn _export_greet_cabi(arg0: *mut u8, arg1: usize) -> *mut u8 { @@ -49,10 +85,7 @@ static mut _RET_AREA: _RetArea = _RetArea( #[rustfmt::skip] mod _rt { #![allow(dead_code, clippy::all)] - #[cfg(target_arch = "wasm32")] - pub fn run_ctors_once() { - wit_bindgen_rt::run_ctors_once(); - } + pub use alloc_crate::string::String; pub use alloc_crate::vec::Vec; pub unsafe fn string_lift(bytes: Vec) -> String { if cfg!(debug_assertions) { @@ -61,6 +94,10 @@ mod _rt { String::from_utf8_unchecked(bytes) } } + #[cfg(target_arch = "wasm32")] + pub fn run_ctors_once() { + wit_bindgen_rt::run_ctors_once(); + } pub unsafe fn cabi_dealloc(ptr: *mut u8, size: usize, align: usize) { if size == 0 { return; @@ -68,7 +105,6 @@ mod _rt { let layout = alloc::Layout::from_size_align_unchecked(size, align); alloc::dealloc(ptr, layout); } - pub use alloc_crate::string::String; extern crate alloc as alloc_crate; pub use alloc_crate::alloc; } @@ -107,11 +143,12 @@ pub(crate) use __export_echo_impl as export; )] #[doc(hidden)] #[allow(clippy::octal_escapes)] -pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; 173] = *b"\ -\0asm\x0d\0\x01\0\0\x19\x16wit-component-encoding\x04\0\x073\x01A\x02\x01A\x02\x01\ -@\x01\x04names\0s\x04\0\x05greet\x01\0\x04\0\x13component:echo/echo\x04\0\x0b\x0a\ -\x01\0\x04echo\x03\0\0\0G\x09producers\x01\x0cprocessed-by\x02\x0dwit-component\x07\ -0.227.1\x10wit-bindgen-rust\x060.41.0"; +pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; 199] = *b"\ +\0asm\x0d\0\x01\0\0\x19\x16wit-component-encoding\x04\0\x07M\x01A\x02\x01A\x04\x01\ +@\x01\x03keys\0s\x03\0\x0bhost-lookup\x01\0\x01@\x01\x04names\0s\x04\0\x05greet\x01\ +\x01\x04\0\x13component:echo/echo\x04\0\x0b\x0a\x01\0\x04echo\x03\0\0\0G\x09prod\ +ucers\x01\x0cprocessed-by\x02\x0dwit-component\x070.227.1\x10wit-bindgen-rust\x06\ +0.41.0"; #[inline(never)] #[doc(hidden)] pub fn __link_custom_section_describing_imports() { diff --git a/crates/wasm-guests/echo/src/lib.rs b/crates/wasm-guests/echo/src/lib.rs index 39c259a4..7e6475be 100644 --- a/crates/wasm-guests/echo/src/lib.rs +++ b/crates/wasm-guests/echo/src/lib.rs @@ -7,7 +7,10 @@ struct Component; impl Guest for Component { fn greet(name: String) -> String { - format!("hello {name}") + // Call back into the host, then fold the host's answer into our reply. + // Proves the guest→host callback path over wasm. + let from_host = bindings::host_lookup(&name); + format!("hello {name} ({from_host})") } } diff --git a/crates/wasm-guests/echo/wit/world.wit b/crates/wasm-guests/echo/wit/world.wit index 567a828d..8b87114e 100644 --- a/crates/wasm-guests/echo/wit/world.wit +++ b/crates/wasm-guests/echo/wit/world.wit @@ -1,7 +1,15 @@ package component:echo; -/// Minimal de-risk world: a single greet export to validate the -/// cargo-component guest + wasmtime host vertical slice. +/// Minimal de-risk world for the wasm transport slice. +/// +/// `greet` (export) is called by the host; inside it the guest calls back into +/// the host through `host-lookup` (import). This exercises BOTH directions — +/// host→guest call and guest→host callback — which is the crux of the plugin +/// design (the guest provider/driver calls back into the engine's executor). world echo { + /// Host-provided callback the guest invokes (stand-in for the AbiHost + /// executor surface: result/note_dep/query/...). + import host-lookup: func(key: string) -> string; + export greet: func(name: string) -> string; } From db86663565abfbee4f38d43d8c2e94d9e1bd7220 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Mon, 15 Jun 2026 23:30:54 +0200 Subject: [PATCH 18/44] feat(plugin-remote): wasm provider+driver ABI + guest SDK + hello-world [M4] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brings the wasm tier from de-risk slice to a working provider/driver plugin over wasmtime, with the host-callback (AbiHost) path wired. ABI (WIT, crates/plugin-abi/wit/heph-plugin.wit): - Mirrors the *service* surface — provider (config/get) + driver (config/parse/run) exports, host (resolve/note-dep/query) imports. Payloads are protobuf-encoded heph.plugin.v1 bytes (single schema source = proto; no structural WIT triplication). Typed error via an `error-kind` enum so neither side matches on message strings. Guest SDK (crates/wasm-guests/sdk, wasm-only standalone workspace): - pb-typed Provider/Driver/Host traits + PluginError + encode/decode helpers. - Depends on plugin-abi WITHOUT convert/transport (those pull hplugin -> hsandboxfuse/hwalk which don't build for wasm); guest works on prost `pb`. Host (plugin-remote/src/wasm.rs, feature `wasm`): - WasmPlugin loads a component (async wasmtime 45 + wasmtime-wasi p2); WasmProvider / WasmDriver implement the in-process hplugin traits, instantiating a fresh store/instance per call so the per-request executor is isolated. - host imports serve result/note_dep/query against the real ProviderExecutor; cycle/cancelled classified by downcast (never message), mapped to the WIT error kind. Hello-world plugin (crates/wasm-guests/helloworld, cargo-component): - provider.get folds a guest->host `query` callback count into a label; driver parse carries the greeting in the opaque raw_def; run returns one inline output artifact. e2e (--features wasm): builds the guest, drives get (callback observed) + parse (raw_def round-trip via def_de) + run (artifact) — all green. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-abi/wit/heph-plugin.wit | 68 ++ crates/plugin-remote/src/wasm.rs | 509 +++++++++- crates/plugin-remote/tests/wasm_plugin_e2e.rs | 161 ++++ crates/wasm-guests/helloworld/Cargo.lock | 473 ++++++++++ crates/wasm-guests/helloworld/Cargo.toml | 19 + crates/wasm-guests/helloworld/src/bindings.rs | 885 ++++++++++++++++++ crates/wasm-guests/helloworld/src/lib.rs | 214 +++++ crates/wasm-guests/helloworld/wit/world.wit | 68 ++ crates/wasm-guests/sdk/Cargo.lock | 450 +++++++++ crates/wasm-guests/sdk/Cargo.toml | 19 + crates/wasm-guests/sdk/src/lib.rs | 101 ++ 11 files changed, 2940 insertions(+), 27 deletions(-) create mode 100644 crates/plugin-abi/wit/heph-plugin.wit create mode 100644 crates/plugin-remote/tests/wasm_plugin_e2e.rs create mode 100644 crates/wasm-guests/helloworld/Cargo.lock create mode 100644 crates/wasm-guests/helloworld/Cargo.toml create mode 100644 crates/wasm-guests/helloworld/src/bindings.rs create mode 100644 crates/wasm-guests/helloworld/src/lib.rs create mode 100644 crates/wasm-guests/helloworld/wit/world.wit create mode 100644 crates/wasm-guests/sdk/Cargo.lock create mode 100644 crates/wasm-guests/sdk/Cargo.toml create mode 100644 crates/wasm-guests/sdk/src/lib.rs diff --git a/crates/plugin-abi/wit/heph-plugin.wit b/crates/plugin-abi/wit/heph-plugin.wit new file mode 100644 index 00000000..bd3d65ea --- /dev/null +++ b/crates/plugin-abi/wit/heph-plugin.wit @@ -0,0 +1,68 @@ +package heph:plugin; + +/// Shared types for the wasm tier of the heph external-plugin ABI. +/// +/// Payloads are deliberately `list` carrying protobuf-encoded +/// `heph.plugin.v1` messages (proto/plugin/v1/*.proto) — the SAME wire schema +/// the proto (UDS) and shm (iceoryx2) transports use. Keeping one schema source +/// (proto) avoids re-encoding every message as a WIT record and the three-way +/// drift that would create; the WIT here mirrors the *service* surface, not the +/// message structure. +interface types { + /// Typed error category so neither guest nor host ever matches on the + /// message string. Mirrors the wire `Error.Kind` / `GetError.Kind`. + enum error-kind { + other, + not-found, + cycle, + cancelled, + unimplemented, + } + record plugin-error { + kind: error-kind, + message: string, + } +} + +/// Callbacks the guest invokes back into the host while serving a request — the +/// AbiHost surface. This is the bidirectional crux: a guest provider calls these +/// to resolve dependencies through the engine's executor mid-`get`. +interface host { + use types.{plugin-error}; + + /// ResultRequest -> ResultResponse. Registers the parent->addr DepDag edge + /// host-side (cycle check) and returns artifact handles + a lease. + /// (Named `resolve`, not `result`: `result` is a WIT reserved keyword.) + resolve: func(req: list) -> result, plugin-error>; + + /// NoteDepRequest -> NoteDepResponse. Cache-hit fast path: register the dep + /// edge only. Returns the cycle flag when the edge closes a cycle. + note-dep: func(req: list) -> result, plugin-error>; + + /// QueryRequest -> QueryResponse. + query: func(req: list) -> result, plugin-error>; +} + +/// Methods the guest exports — the provider and driver surface. A single guest +/// component may serve both. Mirrors hplugin::Provider / hplugin::Driver. +/// (Named `api`, not `plugin`: a world and an interface can't share a name.) +interface api { + use types.{plugin-error}; + + /// -> ConfigResponse (provider name). + provider-config: func() -> result, plugin-error>; + /// GetRequest -> GetResponse. + provider-get: func(req: list) -> result, plugin-error>; + + /// -> ConfigResponse (driver name). + driver-config: func() -> result, plugin-error>; + /// ParseRequest -> ParseResponse. + driver-parse: func(req: list) -> result, plugin-error>; + /// RunRequest -> RunResponse. + driver-run: func(req: list) -> result, plugin-error>; +} + +world plugin { + import host; + export api; +} diff --git a/crates/plugin-remote/src/wasm.rs b/crates/plugin-remote/src/wasm.rs index 1a46d181..38a422b0 100644 --- a/crates/plugin-remote/src/wasm.rs +++ b/crates/plugin-remote/src/wasm.rs @@ -1,19 +1,34 @@ //! wasm transport (in-process wasmtime component) — milestone M4. //! -//! Instantiates a plugin `.wasm` component via wasmtime, binds the WIT -//! interface, and (eventually) wires the `AbiHost` callbacks as host imports -//! with capability-scoped WASI preopens per the `launch` policy. +//! Runs a plugin compiled to a wasm component in-process via wasmtime. The +//! guest exports the provider/driver surface and imports the host callback +//! surface (the AbiHost: result/note_dep/query); both directions carry +//! protobuf-encoded `heph.plugin.v1` messages — the same wire schema the proto +//! and shm transports use (WIT in `crates/plugin-abi/wit/heph-plugin.wit`). //! -//! This module currently carries the de-risk vertical slice: load a component -//! that exports `greet`, link WASI, instantiate, and call it. It proves the -//! cargo-component guest ↔ wasmtime host contract end-to-end before the full -//! provider/driver WIT is brought up. +//! [`WasmPlugin`] loads a component and exposes [`WasmPlugin::provider`] / +//! [`WasmPlugin::driver`] which implement the in-process `hplugin` traits, so +//! the engine registers them through its normal factory hooks and stays unaware +//! the plugin is wasm. Each call instantiates a fresh store/instance so the +//! per-request executor is isolated and concurrent calls don't share state. use anyhow::{Context, Result}; -use wasmtime::component::{Component, Linker, ResourceTable}; +use async_trait::async_trait; +use futures::future::BoxFuture; +use hcore::hartifactcontent::Content; +use hcore::hasync::Cancellable; +use hplugin::provider::ProviderExecutor; +use plugin_abi::{convert, pb}; +use prost::Message; +use std::sync::Arc; +use wasmtime::component::{Component, HasSelf, Linker, ResourceTable}; use wasmtime::{Config, Engine, Store}; use wasmtime_wasi::{WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView}; +// =========================================================================== +// echo de-risk slice — kept; exercised by the `echo_*` wasm_e2e test. Proves +// the bare cargo-component guest ↔ wasmtime host contract (both directions). +// =========================================================================== mod echo_bindings { wasmtime::component::bindgen!({ // Must stay in sync with crates/wasm-guests/echo/wit/world.wit. @@ -22,14 +37,13 @@ mod echo_bindings { }); } -/// Per-instance host state: the WASI context plus the resource table wasmtime -/// uses to hand out handles to host-owned resources. -struct HostState { +/// Per-instance host state for the echo slice. +struct EchoState { table: ResourceTable, wasi: WasiCtx, } -impl WasiView for HostState { +impl WasiView for EchoState { fn ctx(&mut self) -> WasiCtxView<'_> { WasiCtxView { ctx: &mut self.wasi, @@ -38,18 +52,14 @@ impl WasiView for HostState { } } -// Host-side implementation of the world's `host-lookup` import — the guest -// calls this back during `greet`. Stand-in for the real AbiHost executor -// surface (result/note_dep/query); proves the guest→host callback path. -impl echo_bindings::EchoImports for HostState { +impl echo_bindings::EchoImports for EchoState { fn host_lookup(&mut self, key: String) -> String { format!("host:{key}") } } /// De-risk slice: instantiate `wasm` (a component exporting `greet`) and call -/// `greet(name)`. Synchronous — `greet` is pure compute, no host imports beyond -/// the WASI shims the component links against. +/// `greet(name)`. `greet` calls back into the host's `host-lookup` import. pub fn instantiate_and_greet(wasm: &[u8], name: &str) -> Result { let mut config = Config::new(); config.wasm_component_model(true); @@ -60,18 +70,15 @@ pub fn instantiate_and_greet(wasm: &[u8], name: &str) -> Result { .map_err(anyhow::Error::from) .context("loading wasm component")?; - let mut linker = Linker::::new(&engine); + let mut linker = Linker::::new(&engine); wasmtime_wasi::p2::add_to_linker_sync(&mut linker) .map_err(anyhow::Error::from) .context("linking WASI imports")?; - echo_bindings::Echo::add_to_linker::<_, wasmtime::component::HasSelf>( - &mut linker, - |state| state, - ) - .map_err(anyhow::Error::from) - .context("linking host imports")?; - - let state = HostState { + echo_bindings::Echo::add_to_linker::<_, HasSelf>(&mut linker, |state| state) + .map_err(anyhow::Error::from) + .context("linking host imports")?; + + let state = EchoState { table: ResourceTable::new(), wasi: WasiCtxBuilder::new().build(), }; @@ -84,3 +91,451 @@ pub fn instantiate_and_greet(wasm: &[u8], name: &str) -> Result { .map_err(anyhow::Error::from) .context("calling greet export") } + +// =========================================================================== +// plugin transport — provider + driver over wasm, with host callbacks. +// =========================================================================== +mod plugin_bindings { + wasmtime::component::bindgen!({ + path: "../plugin-abi/wit/heph-plugin.wit", + world: "plugin", + imports: { default: async }, + exports: { default: async }, + }); +} + +use plugin_bindings::heph::plugin::types::{ErrorKind as WErrorKind, PluginError as WPluginError}; + +/// Per-call host state: the current request's executor plus the lease table +/// holding result-artifact read-guards alive for the duration of the call. +struct PluginStore { + wasi: WasiCtx, + table: ResourceTable, + executor: Option>, + leases: crate::lease::LeaseTable, +} + +impl WasiView for PluginStore { + fn ctx(&mut self) -> WasiCtxView<'_> { + WasiCtxView { + ctx: &mut self.wasi, + table: &mut self.table, + } + } +} + +/// True if `e`'s chain contains a dependency-cycle error. +fn is_cycle(e: &anyhow::Error) -> bool { + hcore::hmemoizer::downcast_chain_ref::(e).is_some() +} + +fn werr(kind: WErrorKind, message: String) -> WPluginError { + WPluginError { kind, message } +} + +/// Classify an engine error into a typed wire error kind (downcast, never +/// message-matched) so the guest can reconstruct a typed error. +fn werr_from(e: &anyhow::Error) -> WPluginError { + let kind = if is_cycle(e) { + WErrorKind::Cycle + } else if hplugin::error::is_cancelled(e) { + WErrorKind::Cancelled + } else { + WErrorKind::Other + }; + werr(kind, e.to_string()) +} + +// Host callback surface (the AbiHost): the guest calls these back while serving +// get()/parse(). Implemented against the per-call executor in the store. +impl plugin_bindings::heph::plugin::host::Host for PluginStore { + async fn resolve(&mut self, req: Vec) -> Result, WPluginError> { + let Some(executor) = self.executor.clone() else { + return Err(werr(WErrorKind::Other, "no executor bound".into())); + }; + let req = pb::ResultRequest::decode(&req[..]) + .map_err(|e| werr(WErrorKind::Other, format!("decode ResultRequest: {e}")))?; + let addr = convert::addr_from_pb(req.addr.unwrap_or_default()); + match executor.result(&addr).await { + Ok(eres) => { + let artifacts: Vec> = eres.artifacts.clone(); + let mut handles = Vec::with_capacity(artifacts.len()); + for (idx, art) in artifacts.iter().enumerate() { + let hashout = eres + .artifacts_meta + .get(idx) + .map(|m| m.hashout.clone()) + .or_else(|| art.hashout().ok()) + .unwrap_or_default(); + handles.push(pb::ArtifactHandle { + handle_id: idx.to_string(), + group: String::new(), + name: String::new(), + hashout, + byte_size: art.byte_size().unwrap_or(0), + support: false, + }); + } + let lease_id = self.leases.insert(artifacts); + Ok(pb::ResultResponse { + lease_id, + artifacts: handles, + } + .encode_to_vec()) + } + Err(e) => Err(werr_from(&e)), + } + } + + async fn note_dep(&mut self, req: Vec) -> Result, WPluginError> { + let Some(executor) = self.executor.clone() else { + return Err(werr(WErrorKind::Other, "no executor bound".into())); + }; + let req = pb::NoteDepRequest::decode(&req[..]) + .map_err(|e| werr(WErrorKind::Other, format!("decode NoteDepRequest: {e}")))?; + let addr = convert::addr_from_pb(req.addr.unwrap_or_default()); + let resp = match executor.note_dep(&addr).await { + Ok(()) => pb::NoteDepResponse { + ok: true, + cycle: false, + message: String::new(), + }, + Err(e) => pb::NoteDepResponse { + ok: false, + cycle: is_cycle(&e), + message: e.to_string(), + }, + }; + Ok(resp.encode_to_vec()) + } + + async fn query(&mut self, req: Vec) -> Result, WPluginError> { + let Some(executor) = self.executor.clone() else { + return Err(werr(WErrorKind::Other, "no executor bound".into())); + }; + let req = pb::QueryRequest::decode(&req[..]) + .map_err(|e| werr(WErrorKind::Other, format!("decode QueryRequest: {e}")))?; + let matcher = convert::matcher_from_pb(req.matcher.unwrap_or_default()); + match executor.query(&matcher, &req.extra_skip).await { + Ok(addrs) => Ok(pb::QueryResponse { + addrs: addrs.iter().map(convert::addr_to_pb).collect(), + } + .encode_to_vec()), + Err(e) => Err(werr_from(&e)), + } + } +} + +/// A loaded wasm plugin component. Holds the engine + linked component; each +/// provider/driver call instantiates a fresh store/instance. +pub struct WasmPlugin { + engine: Engine, + component: Component, + linker: Linker, + provider_name: String, + driver_name: String, +} + +impl WasmPlugin { + /// Load a plugin component from raw bytes. `provider_name`/`driver_name` are + /// the registry names the engine will route to this plugin (the guest also + /// reports them via `*_config`, used as the source of truth where queried). + pub fn load( + wasm: &[u8], + provider_name: impl Into, + driver_name: impl Into, + ) -> Result> { + // wasmtime 45 always supports async; component-model must be enabled. + let mut config = Config::new(); + config.wasm_component_model(true); + let engine = Engine::new(&config) + .map_err(anyhow::Error::from) + .context("building wasmtime engine")?; + let component = Component::from_binary(&engine, wasm) + .map_err(anyhow::Error::from) + .context("loading plugin component")?; + + let mut linker = Linker::::new(&engine); + wasmtime_wasi::p2::add_to_linker_async(&mut linker) + .map_err(anyhow::Error::from) + .context("linking WASI imports")?; + plugin_bindings::heph::plugin::host::add_to_linker::<_, HasSelf>( + &mut linker, + |s| s, + ) + .map_err(anyhow::Error::from) + .context("linking host callback imports")?; + + Ok(Arc::new(Self { + engine, + component, + linker, + provider_name: provider_name.into(), + driver_name: driver_name.into(), + })) + } + + fn new_store(&self, executor: Option>) -> Store { + Store::new( + &self.engine, + PluginStore { + wasi: WasiCtxBuilder::new().build(), + table: ResourceTable::new(), + executor, + leases: crate::lease::LeaseTable::default(), + }, + ) + } + + async fn instantiate(&self, store: &mut Store) -> Result { + plugin_bindings::Plugin::instantiate_async(store, &self.component, &self.linker) + .await + .map_err(anyhow::Error::from) + .context("instantiating plugin component") + } + + /// Provider handle implementing `hplugin::Provider`. + pub fn provider(self: &Arc) -> WasmProvider { + WasmProvider { + plugin: Arc::clone(self), + } + } + + /// Driver handle implementing `hplugin::Driver`. + pub fn driver(self: &Arc) -> WasmDriver { + WasmDriver { + plugin: Arc::clone(self), + } + } + + pub fn provider_name(&self) -> &str { + &self.provider_name + } + pub fn driver_name(&self) -> &str { + &self.driver_name + } +} + +/// A typed WIT error surfaced from a guest export. +fn wit_to_anyhow(e: &WPluginError) -> anyhow::Error { + anyhow::anyhow!("plugin error ({:?}): {}", e.kind, e.message) +} + +pub struct WasmProvider { + plugin: Arc, +} + +impl hplugin::provider::Provider for WasmProvider { + fn config( + &self, + _req: hplugin::provider::ConfigRequest, + ) -> Result { + Ok(hplugin::provider::ConfigResponse { + name: self.plugin.provider_name.clone(), + }) + } + + fn list<'a>( + &'a self, + _req: hplugin::provider::ListRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture< + 'a, + Result> + Send>>, + > { + // list() is not part of the hello-world data path; an empty listing is + // correct for a provider addressed only via get(). + Box::pin(async move { + Ok(Box::new(std::iter::empty()) + as Box< + dyn Iterator> + Send, + >) + }) + } + + fn list_packages<'a>( + &'a self, + _req: hplugin::provider::ListPackagesRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture< + 'a, + Result> + Send>>, + > { + Box::pin(async move { + Ok(Box::new(std::iter::empty()) + as Box< + dyn Iterator> + Send, + >) + }) + } + + fn get<'a>( + &'a self, + req: hplugin::provider::GetRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture< + 'a, + std::result::Result, + > { + Box::pin(async move { + use hplugin::provider::GetError; + let pb_req = pb::GetRequest { + request_id: req.request_id, + addr: Some(convert::addr_to_pb(&req.addr)), + states: req.states.iter().map(convert::state_to_pb).collect(), + }; + let mut store = self.plugin.new_store(Some(Arc::clone(&req.executor))); + let inst = self + .plugin + .instantiate(&mut store) + .await + .map_err(GetError::Other)?; + let res = inst + .heph_plugin_api() + .call_provider_get(&mut store, &pb_req.encode_to_vec()) + .await + .map_err(anyhow::Error::from) + .map_err(GetError::Other)?; + match res { + Ok(bytes) => { + let resp = pb::GetResponse::decode(&bytes[..]) + .map_err(|e| GetError::Other(anyhow::anyhow!("decode GetResponse: {e}")))?; + Ok(hplugin::provider::GetResponse { + target_spec: convert::target_spec_from_pb( + resp.target_spec.unwrap_or_default(), + ), + }) + } + Err(e) => match e.kind { + WErrorKind::NotFound => Err(GetError::NotFound), + WErrorKind::Cycle => Err(GetError::Other(anyhow::Error::new( + hplugin::error::CycleError { + from: req.addr.clone(), + to: req.addr.clone(), + }, + ))), + WErrorKind::Cancelled => Err(GetError::Other(anyhow::Error::new( + hplugin::error::CancelledError, + ))), + _ => Err(GetError::Other(wit_to_anyhow(&e))), + }, + } + }) + } + + fn probe<'a>( + &'a self, + _req: hplugin::provider::ProbeRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, Result> { + Box::pin(async move { Ok(hplugin::provider::ProbeResponse { states: vec![] }) }) + } +} + +pub struct WasmDriver { + plugin: Arc, +} + +#[async_trait] +impl hplugin::driver::Driver for WasmDriver { + fn config( + &self, + _req: hplugin::driver::ConfigRequest, + ) -> Result { + Ok(hplugin::driver::ConfigResponse { + name: self.plugin.driver_name.clone(), + }) + } + + fn schema(&self) -> hplugin::driver::DriverSchema { + hplugin::driver::DriverSchema::default() + } + + async fn parse( + &self, + req: hplugin::driver::ParseRequest, + _ctoken: &(dyn Cancellable + Send + Sync), + ) -> Result { + let pb_req = pb::ParseRequest { + request_id: req.request_id, + target_spec: Some(convert::target_spec_to_pb(req.target_spec.as_ref())), + driver: self.plugin.driver_name.clone(), + }; + let mut store = self.plugin.new_store(None); + let inst = self.plugin.instantiate(&mut store).await?; + let res = inst + .heph_plugin_api() + .call_driver_parse(&mut store, &pb_req.encode_to_vec()) + .await + .map_err(anyhow::Error::from)?; + let bytes = res.map_err(|e| wit_to_anyhow(&e))?; + let resp = pb::ParseResponse::decode(&bytes[..]).context("decode ParseResponse")?; + Ok(hplugin::driver::ParseResponse { + target_def: convert::target_def_from_pb(resp.target_def.unwrap_or_default())?, + }) + } + + async fn apply_transitive( + &self, + _req: hplugin::driver::ApplyTransitiveRequest, + _ctoken: &(dyn Cancellable + Send + Sync), + ) -> Result { + anyhow::bail!("wasm driver apply_transitive() not implemented") + } + + async fn run<'a, 'io>( + &self, + req: hplugin::driver::RunRequest<'a, 'io>, + _ctoken: &(dyn Cancellable + Send + Sync), + ) -> Result { + self.run_inner(req, false).await + } + + async fn run_shell<'a, 'io>( + &self, + req: hplugin::driver::RunRequest<'a, 'io>, + _ctoken: &(dyn Cancellable + Send + Sync), + ) -> Result { + self.run_inner(req, true).await + } +} + +impl WasmDriver { + async fn run_inner<'a, 'io>( + &self, + req: hplugin::driver::RunRequest<'a, 'io>, + shell: bool, + ) -> Result { + // stdio crosses as inherited fds elsewhere; inputs for the hello-world + // driver are not materialized over wasm (no abstract-artifact pull yet). + let pb_req = pb::RunRequest { + request_id: req.request_id.clone(), + target: Some(convert::target_def_to_pb(req.target)?), + tree_root_path: req.tree_root_path.to_string_lossy().into_owned(), + inputs: vec![], + hashin: req.hashin.to_string(), + sandbox_dir: req.sandbox_dir.to_string_lossy().into_owned(), + has_stdin: req.stdin.is_some(), + has_stdout: req.stdout.is_some(), + has_stderr: req.stderr.is_some(), + shell, + }; + let mut store = self.plugin.new_store(None); + let inst = self.plugin.instantiate(&mut store).await?; + let res = inst + .heph_plugin_api() + .call_driver_run(&mut store, &pb_req.encode_to_vec()) + .await + .map_err(anyhow::Error::from)?; + let bytes = res.map_err(|e| wit_to_anyhow(&e))?; + let resp = pb::RunResponse::decode(&bytes[..]).context("decode RunResponse")?; + Ok(hplugin::driver::RunResponse { + artifacts: resp + .artifacts + .into_iter() + .map(convert::output_artifact_from_pb) + .collect(), + ..Default::default() + }) + } +} diff --git a/crates/plugin-remote/tests/wasm_plugin_e2e.rs b/crates/plugin-remote/tests/wasm_plugin_e2e.rs new file mode 100644 index 00000000..d61bd670 --- /dev/null +++ b/crates/plugin-remote/tests/wasm_plugin_e2e.rs @@ -0,0 +1,161 @@ +//! End-to-end: the `helloworld` wasm plugin (provider + driver) served through +//! the wasmtime host transport, exercising the full data path — +//! provider.get (with a guest->host `query` callback), +//! driver.parse (TargetDef round-trip incl. opaque raw_def), +//! driver.run (output artifact crossing back as inline bytes). +//! +//! Gated behind `--features wasm` (needs the devenv wasm toolchain). Run with: +//! devenv shell -- cargo test -p plugin-remote --features wasm +#![cfg(feature = "wasm")] + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; + +use futures::future::BoxFuture; +use hcore::hasync::StdCancellationToken; +use hcore::htvalue::Value; +use hmodel::htaddr::Addr; +use hmodel::htmatcher::Matcher; +use hmodel::htpkg::PkgBuf; +use hplugin::driver::outputartifact::Content as OaContent; +use hplugin::driver::{Driver, ParseRequest, RunRequest}; +use hplugin::eresult::EResult; +use hplugin::provider::{GetRequest, Provider, ProviderExecutor}; +use plugin_remote::wasm::WasmPlugin; + +/// Build the helloworld guest component into `out_dir`, return the `.wasm` path. +fn build_component(out_dir: &Path) -> PathBuf { + let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../wasm-guests/helloworld/Cargo.toml") + .canonicalize() + .expect("helloworld manifest exists"); + let status = Command::new("cargo") + .args(["component", "build"]) + .arg("--manifest-path") + .arg(&manifest) + .arg("--target") + .arg("wasm32-wasip1") + .arg("--target-dir") + .arg(out_dir) + .status() + .expect("cargo-component on PATH (run inside `devenv shell`)"); + assert!(status.success(), "cargo component build failed"); + let wasm = out_dir.join("wasm32-wasip1/debug/helloworld.wasm"); + assert!(wasm.exists(), "component at {}", wasm.display()); + wasm +} + +/// Stub executor: `query` returns a fixed set of addrs so the guest's callback +/// has an observable effect; `result` is unused on this path. +struct StubExecutor { + query_addrs: Vec, +} + +impl ProviderExecutor for StubExecutor { + fn result<'a>(&'a self, _addr: &'a Addr) -> BoxFuture<'a, anyhow::Result>> { + Box::pin(async move { anyhow::bail!("result() not used in this test") }) + } + fn query<'a>( + &'a self, + _m: &'a Matcher, + _extra: &'a [String], + ) -> BoxFuture<'a, anyhow::Result>> { + Box::pin(async move { Ok(self.query_addrs.clone()) }) + } +} + +#[derive(serde::Deserialize, PartialEq, Debug)] +struct HelloDef { + greeting: String, +} + +fn addr(pkg: &str, name: &str) -> Addr { + Addr::new(PkgBuf::from(pkg), name.to_string(), BTreeMap::new()) +} + +#[tokio::test] +async fn helloworld_plugin_over_wasm() { + let tmp = tempfile::tempdir().expect("tempdir"); + let wasm = std::fs::read(build_component(tmp.path())).expect("read component"); + + let plugin = WasmPlugin::load(&wasm, "hello", "hello").expect("load plugin"); + let provider = plugin.provider(); + let driver = plugin.driver(); + let ctoken = StdCancellationToken::new(); + + let target = addr("//hello", "world"); + // Two addrs => the guest's query callback should observe a count of 2. + let executor = Arc::new(StubExecutor { + query_addrs: vec![addr("//hello", "world"), addr("//hello", "other")], + }); + + // --- provider.get (+ guest->host query callback) --- + let resp = provider + .get( + GetRequest { + request_id: "r1".to_string(), + addr: target.clone(), + states: vec![], + executor: executor.clone(), + }, + &ctoken, + ) + .await + .expect("get"); + assert_eq!(resp.target_spec.driver, "hello"); + assert!( + resp.target_spec.labels.contains(&"queried:2".to_string()), + "callback count folded into label, got {:?}", + resp.target_spec.labels + ); + assert_eq!( + resp.target_spec.config.get("greeting"), + Some(&Value::String("hello world".to_string())) + ); + + // --- driver.parse (TargetDef + opaque raw_def round-trip) --- + let parsed = driver + .parse( + ParseRequest { + request_id: "r2".to_string(), + target_spec: Arc::new(resp.target_spec), + }, + &ctoken, + ) + .await + .expect("parse"); + let def = &parsed.target_def; + assert_eq!( + def.def_de::(), + &HelloDef { + greeting: "hello world".to_string() + } + ); + + // --- driver.run (output artifact crosses back as inline bytes) --- + let rid = "r3".to_string(); + let run = driver + .run( + RunRequest { + request_id: &rid, + target: def, + tree_root_path: tmp.path().to_path_buf(), + inputs: vec![], + hashin: "hash", + stdin: None, + stdout: None, + stderr: None, + sandbox_dir: tmp.path().to_path_buf(), + }, + &ctoken, + ) + .await + .expect("run"); + assert_eq!(run.artifacts.len(), 1); + match &run.artifacts[0].content { + OaContent::Raw(raw) => assert_eq!(raw.data, b"hello world"), + _ => panic!("expected Raw output artifact"), + } +} diff --git a/crates/wasm-guests/helloworld/Cargo.lock b/crates/wasm-guests/helloworld/Cargo.lock new file mode 100644 index 00000000..141939b2 --- /dev/null +++ b/crates/wasm-guests/helloworld/Cargo.lock @@ -0,0 +1,473 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytecheck" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "rancor", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "helloworld" +version = "0.1.0" +dependencies = [ + "heph-plugin-sdk-wasm", + "wit-bindgen-rt", +] + +[[package]] +name = "heph-plugin-sdk-wasm" +version = "0.1.0" +dependencies = [ + "plugin-abi", + "prost", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pbjson" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e6349fa080353f4a597daffd05cb81572a9c031a6d4fff7e504947496fcc68" +dependencies = [ + "base64", + "serde", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plugin-abi" +version = "0.1.0" +dependencies = [ + "anyhow", + "proto-gen", + "rkyv", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "proto-gen" +version = "0.1.0" +dependencies = [ + "pbjson", + "prost", + "prost-types", + "serde", +] + +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rancor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +dependencies = [ + "ptr_meta", +] + +[[package]] +name = "rend" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "rkyv" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +dependencies = [ + "bytecheck", + "bytes", + "hashbrown", + "indexmap", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "653c85dd7aee6fe6f4bded0d242406deadae9819029ce6f7d258c920c384358a" +dependencies = [ + "bitflags", +] diff --git a/crates/wasm-guests/helloworld/Cargo.toml b/crates/wasm-guests/helloworld/Cargo.toml new file mode 100644 index 00000000..d4ec66e0 --- /dev/null +++ b/crates/wasm-guests/helloworld/Cargo.toml @@ -0,0 +1,19 @@ +# Hello-world heph plugin compiled to a wasm component (provider + driver). +# Standalone workspace, built with cargo-component for wasm32-wasip2; kept out +# of the host `cargo build --workspace`. +[workspace] + +[package] +name = "helloworld" +version = "0.1.0" +edition = "2024" + +[dependencies] +heph-plugin-sdk-wasm = { path = "../sdk" } +wit-bindgen-rt = { version = "0.44.0", features = ["bitflags"] } + +[lib] +crate-type = ["cdylib"] + +[package.metadata.component] +package = "heph:plugin" diff --git a/crates/wasm-guests/helloworld/src/bindings.rs b/crates/wasm-guests/helloworld/src/bindings.rs new file mode 100644 index 00000000..82993ae7 --- /dev/null +++ b/crates/wasm-guests/helloworld/src/bindings.rs @@ -0,0 +1,885 @@ +// Generated by `wit-bindgen` 0.41.0. DO NOT EDIT! +// Options used: +// * runtime_path: "wit_bindgen_rt" +#[rustfmt::skip] +#[allow(dead_code, clippy::all)] +pub mod heph { + pub mod plugin { + /// Shared types for the wasm tier of the heph external-plugin ABI. + /// + /// Payloads are deliberately `list` carrying protobuf-encoded + /// `heph.plugin.v1` messages (proto/plugin/v1/*.proto) — the SAME wire schema + /// the proto (UDS) and shm (iceoryx2) transports use. Keeping one schema source + /// (proto) avoids re-encoding every message as a WIT record and the three-way + /// drift that would create; the WIT here mirrors the *service* surface, not the + /// message structure. + #[allow(dead_code, async_fn_in_trait, unused_imports, clippy::all)] + pub mod types { + #[used] + #[doc(hidden)] + static __FORCE_SECTION_REF: fn() = super::super::super::__link_custom_section_describing_imports; + use super::super::super::_rt; + /// Typed error category so neither guest nor host ever matches on the + /// message string. Mirrors the wire `Error.Kind` / `GetError.Kind`. + #[repr(u8)] + #[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] + pub enum ErrorKind { + Other, + NotFound, + Cycle, + Cancelled, + Unimplemented, + } + impl ::core::fmt::Debug for ErrorKind { + fn fmt( + &self, + f: &mut ::core::fmt::Formatter<'_>, + ) -> ::core::fmt::Result { + match self { + ErrorKind::Other => f.debug_tuple("ErrorKind::Other").finish(), + ErrorKind::NotFound => { + f.debug_tuple("ErrorKind::NotFound").finish() + } + ErrorKind::Cycle => f.debug_tuple("ErrorKind::Cycle").finish(), + ErrorKind::Cancelled => { + f.debug_tuple("ErrorKind::Cancelled").finish() + } + ErrorKind::Unimplemented => { + f.debug_tuple("ErrorKind::Unimplemented").finish() + } + } + } + } + impl ErrorKind { + #[doc(hidden)] + pub unsafe fn _lift(val: u8) -> ErrorKind { + if !cfg!(debug_assertions) { + return ::core::mem::transmute(val); + } + match val { + 0 => ErrorKind::Other, + 1 => ErrorKind::NotFound, + 2 => ErrorKind::Cycle, + 3 => ErrorKind::Cancelled, + 4 => ErrorKind::Unimplemented, + _ => panic!("invalid enum discriminant"), + } + } + } + #[derive(Clone)] + pub struct PluginError { + pub kind: ErrorKind, + pub message: _rt::String, + } + impl ::core::fmt::Debug for PluginError { + fn fmt( + &self, + f: &mut ::core::fmt::Formatter<'_>, + ) -> ::core::fmt::Result { + f.debug_struct("PluginError") + .field("kind", &self.kind) + .field("message", &self.message) + .finish() + } + } + impl ::core::fmt::Display for PluginError { + fn fmt( + &self, + f: &mut ::core::fmt::Formatter<'_>, + ) -> ::core::fmt::Result { + write!(f, "{:?}", self) + } + } + impl std::error::Error for PluginError {} + } + /// Callbacks the guest invokes back into the host while serving a request — the + /// AbiHost surface. This is the bidirectional crux: a guest provider calls these + /// to resolve dependencies through the engine's executor mid-`get`. + #[allow(dead_code, async_fn_in_trait, unused_imports, clippy::all)] + pub mod host { + #[used] + #[doc(hidden)] + static __FORCE_SECTION_REF: fn() = super::super::super::__link_custom_section_describing_imports; + use super::super::super::_rt; + pub type PluginError = super::super::super::heph::plugin::types::PluginError; + #[allow(unused_unsafe, clippy::all)] + /// ResultRequest -> ResultResponse. Registers the parent->addr DepDag edge + /// host-side (cycle check) and returns artifact handles + a lease. + /// (Named `resolve`, not `result`: `result` is a WIT reserved keyword.) + pub fn resolve(req: &[u8]) -> Result<_rt::Vec, PluginError> { + unsafe { + #[cfg_attr(target_pointer_width = "64", repr(align(8)))] + #[cfg_attr(target_pointer_width = "32", repr(align(4)))] + struct RetArea( + [::core::mem::MaybeUninit< + u8, + >; 4 * ::core::mem::size_of::<*const u8>()], + ); + let mut ret_area = RetArea( + [::core::mem::MaybeUninit::uninit(); 4 + * ::core::mem::size_of::<*const u8>()], + ); + let vec0 = req; + let ptr0 = vec0.as_ptr().cast::(); + let len0 = vec0.len(); + let ptr1 = ret_area.0.as_mut_ptr().cast::(); + #[cfg(target_arch = "wasm32")] + #[link(wasm_import_module = "heph:plugin/host")] + unsafe extern "C" { + #[link_name = "resolve"] + fn wit_import2(_: *mut u8, _: usize, _: *mut u8); + } + #[cfg(not(target_arch = "wasm32"))] + unsafe extern "C" fn wit_import2(_: *mut u8, _: usize, _: *mut u8) { + unreachable!() + } + unsafe { wit_import2(ptr0.cast_mut(), len0, ptr1) }; + let l3 = i32::from(*ptr1.add(0).cast::()); + let result11 = match l3 { + 0 => { + let e = { + let l4 = *ptr1 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l5 = *ptr1 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::(); + let len6 = l5; + _rt::Vec::from_raw_parts(l4.cast(), len6, len6) + }; + Ok(e) + } + 1 => { + let e = { + let l7 = i32::from( + *ptr1.add(::core::mem::size_of::<*const u8>()).cast::(), + ); + let l8 = *ptr1 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l9 = *ptr1 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::(); + let len10 = l9; + let bytes10 = _rt::Vec::from_raw_parts( + l8.cast(), + len10, + len10, + ); + super::super::super::heph::plugin::types::PluginError { + kind: super::super::super::heph::plugin::types::ErrorKind::_lift( + l7 as u8, + ), + message: _rt::string_lift(bytes10), + } + }; + Err(e) + } + _ => _rt::invalid_enum_discriminant(), + }; + result11 + } + } + #[allow(unused_unsafe, clippy::all)] + /// NoteDepRequest -> NoteDepResponse. Cache-hit fast path: register the dep + /// edge only. Returns the cycle flag when the edge closes a cycle. + pub fn note_dep(req: &[u8]) -> Result<_rt::Vec, PluginError> { + unsafe { + #[cfg_attr(target_pointer_width = "64", repr(align(8)))] + #[cfg_attr(target_pointer_width = "32", repr(align(4)))] + struct RetArea( + [::core::mem::MaybeUninit< + u8, + >; 4 * ::core::mem::size_of::<*const u8>()], + ); + let mut ret_area = RetArea( + [::core::mem::MaybeUninit::uninit(); 4 + * ::core::mem::size_of::<*const u8>()], + ); + let vec0 = req; + let ptr0 = vec0.as_ptr().cast::(); + let len0 = vec0.len(); + let ptr1 = ret_area.0.as_mut_ptr().cast::(); + #[cfg(target_arch = "wasm32")] + #[link(wasm_import_module = "heph:plugin/host")] + unsafe extern "C" { + #[link_name = "note-dep"] + fn wit_import2(_: *mut u8, _: usize, _: *mut u8); + } + #[cfg(not(target_arch = "wasm32"))] + unsafe extern "C" fn wit_import2(_: *mut u8, _: usize, _: *mut u8) { + unreachable!() + } + unsafe { wit_import2(ptr0.cast_mut(), len0, ptr1) }; + let l3 = i32::from(*ptr1.add(0).cast::()); + let result11 = match l3 { + 0 => { + let e = { + let l4 = *ptr1 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l5 = *ptr1 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::(); + let len6 = l5; + _rt::Vec::from_raw_parts(l4.cast(), len6, len6) + }; + Ok(e) + } + 1 => { + let e = { + let l7 = i32::from( + *ptr1.add(::core::mem::size_of::<*const u8>()).cast::(), + ); + let l8 = *ptr1 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l9 = *ptr1 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::(); + let len10 = l9; + let bytes10 = _rt::Vec::from_raw_parts( + l8.cast(), + len10, + len10, + ); + super::super::super::heph::plugin::types::PluginError { + kind: super::super::super::heph::plugin::types::ErrorKind::_lift( + l7 as u8, + ), + message: _rt::string_lift(bytes10), + } + }; + Err(e) + } + _ => _rt::invalid_enum_discriminant(), + }; + result11 + } + } + #[allow(unused_unsafe, clippy::all)] + /// QueryRequest -> QueryResponse. + pub fn query(req: &[u8]) -> Result<_rt::Vec, PluginError> { + unsafe { + #[cfg_attr(target_pointer_width = "64", repr(align(8)))] + #[cfg_attr(target_pointer_width = "32", repr(align(4)))] + struct RetArea( + [::core::mem::MaybeUninit< + u8, + >; 4 * ::core::mem::size_of::<*const u8>()], + ); + let mut ret_area = RetArea( + [::core::mem::MaybeUninit::uninit(); 4 + * ::core::mem::size_of::<*const u8>()], + ); + let vec0 = req; + let ptr0 = vec0.as_ptr().cast::(); + let len0 = vec0.len(); + let ptr1 = ret_area.0.as_mut_ptr().cast::(); + #[cfg(target_arch = "wasm32")] + #[link(wasm_import_module = "heph:plugin/host")] + unsafe extern "C" { + #[link_name = "query"] + fn wit_import2(_: *mut u8, _: usize, _: *mut u8); + } + #[cfg(not(target_arch = "wasm32"))] + unsafe extern "C" fn wit_import2(_: *mut u8, _: usize, _: *mut u8) { + unreachable!() + } + unsafe { wit_import2(ptr0.cast_mut(), len0, ptr1) }; + let l3 = i32::from(*ptr1.add(0).cast::()); + let result11 = match l3 { + 0 => { + let e = { + let l4 = *ptr1 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l5 = *ptr1 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::(); + let len6 = l5; + _rt::Vec::from_raw_parts(l4.cast(), len6, len6) + }; + Ok(e) + } + 1 => { + let e = { + let l7 = i32::from( + *ptr1.add(::core::mem::size_of::<*const u8>()).cast::(), + ); + let l8 = *ptr1 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l9 = *ptr1 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::(); + let len10 = l9; + let bytes10 = _rt::Vec::from_raw_parts( + l8.cast(), + len10, + len10, + ); + super::super::super::heph::plugin::types::PluginError { + kind: super::super::super::heph::plugin::types::ErrorKind::_lift( + l7 as u8, + ), + message: _rt::string_lift(bytes10), + } + }; + Err(e) + } + _ => _rt::invalid_enum_discriminant(), + }; + result11 + } + } + } + } +} +#[rustfmt::skip] +#[allow(dead_code, clippy::all)] +pub mod exports { + pub mod heph { + pub mod plugin { + /// Methods the guest exports — the provider and driver surface. A single guest + /// component may serve both. Mirrors hplugin::Provider / hplugin::Driver. + /// (Named `api`, not `plugin`: a world and an interface can't share a name.) + #[allow(dead_code, async_fn_in_trait, unused_imports, clippy::all)] + pub mod api { + #[used] + #[doc(hidden)] + static __FORCE_SECTION_REF: fn() = super::super::super::super::__link_custom_section_describing_imports; + use super::super::super::super::_rt; + pub type PluginError = super::super::super::super::heph::plugin::types::PluginError; + #[doc(hidden)] + #[allow(non_snake_case)] + pub unsafe fn _export_provider_config_cabi() -> *mut u8 { + #[cfg(target_arch = "wasm32")] _rt::run_ctors_once(); + let result0 = T::provider_config(); + let ptr1 = (&raw mut _RET_AREA.0).cast::(); + match result0 { + Ok(e) => { + *ptr1.add(0).cast::() = (0i32) as u8; + let vec2 = (e).into_boxed_slice(); + let ptr2 = vec2.as_ptr().cast::(); + let len2 = vec2.len(); + ::core::mem::forget(vec2); + *ptr1 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::() = len2; + *ptr1 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>() = ptr2.cast_mut(); + } + Err(e) => { + *ptr1.add(0).cast::() = (1i32) as u8; + let super::super::super::super::heph::plugin::types::PluginError { + kind: kind3, + message: message3, + } = e; + *ptr1 + .add(::core::mem::size_of::<*const u8>()) + .cast::() = (kind3.clone() as i32) as u8; + let vec4 = (message3.into_bytes()).into_boxed_slice(); + let ptr4 = vec4.as_ptr().cast::(); + let len4 = vec4.len(); + ::core::mem::forget(vec4); + *ptr1 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::() = len4; + *ptr1 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>() = ptr4.cast_mut(); + } + }; + ptr1 + } + #[doc(hidden)] + #[allow(non_snake_case)] + pub unsafe fn __post_return_provider_config(arg0: *mut u8) { + let l0 = i32::from(*arg0.add(0).cast::()); + match l0 { + 0 => { + let l1 = *arg0 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l2 = *arg0 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::(); + let base3 = l1; + let len3 = l2; + _rt::cabi_dealloc(base3, len3 * 1, 1); + } + _ => { + let l4 = *arg0 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l5 = *arg0 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::(); + _rt::cabi_dealloc(l4, l5, 1); + } + } + } + #[doc(hidden)] + #[allow(non_snake_case)] + pub unsafe fn _export_provider_get_cabi( + arg0: *mut u8, + arg1: usize, + ) -> *mut u8 { + #[cfg(target_arch = "wasm32")] _rt::run_ctors_once(); + let len0 = arg1; + let result1 = T::provider_get( + _rt::Vec::from_raw_parts(arg0.cast(), len0, len0), + ); + let ptr2 = (&raw mut _RET_AREA.0).cast::(); + match result1 { + Ok(e) => { + *ptr2.add(0).cast::() = (0i32) as u8; + let vec3 = (e).into_boxed_slice(); + let ptr3 = vec3.as_ptr().cast::(); + let len3 = vec3.len(); + ::core::mem::forget(vec3); + *ptr2 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::() = len3; + *ptr2 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>() = ptr3.cast_mut(); + } + Err(e) => { + *ptr2.add(0).cast::() = (1i32) as u8; + let super::super::super::super::heph::plugin::types::PluginError { + kind: kind4, + message: message4, + } = e; + *ptr2 + .add(::core::mem::size_of::<*const u8>()) + .cast::() = (kind4.clone() as i32) as u8; + let vec5 = (message4.into_bytes()).into_boxed_slice(); + let ptr5 = vec5.as_ptr().cast::(); + let len5 = vec5.len(); + ::core::mem::forget(vec5); + *ptr2 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::() = len5; + *ptr2 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>() = ptr5.cast_mut(); + } + }; + ptr2 + } + #[doc(hidden)] + #[allow(non_snake_case)] + pub unsafe fn __post_return_provider_get(arg0: *mut u8) { + let l0 = i32::from(*arg0.add(0).cast::()); + match l0 { + 0 => { + let l1 = *arg0 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l2 = *arg0 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::(); + let base3 = l1; + let len3 = l2; + _rt::cabi_dealloc(base3, len3 * 1, 1); + } + _ => { + let l4 = *arg0 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l5 = *arg0 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::(); + _rt::cabi_dealloc(l4, l5, 1); + } + } + } + #[doc(hidden)] + #[allow(non_snake_case)] + pub unsafe fn _export_driver_config_cabi() -> *mut u8 { + #[cfg(target_arch = "wasm32")] _rt::run_ctors_once(); + let result0 = T::driver_config(); + let ptr1 = (&raw mut _RET_AREA.0).cast::(); + match result0 { + Ok(e) => { + *ptr1.add(0).cast::() = (0i32) as u8; + let vec2 = (e).into_boxed_slice(); + let ptr2 = vec2.as_ptr().cast::(); + let len2 = vec2.len(); + ::core::mem::forget(vec2); + *ptr1 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::() = len2; + *ptr1 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>() = ptr2.cast_mut(); + } + Err(e) => { + *ptr1.add(0).cast::() = (1i32) as u8; + let super::super::super::super::heph::plugin::types::PluginError { + kind: kind3, + message: message3, + } = e; + *ptr1 + .add(::core::mem::size_of::<*const u8>()) + .cast::() = (kind3.clone() as i32) as u8; + let vec4 = (message3.into_bytes()).into_boxed_slice(); + let ptr4 = vec4.as_ptr().cast::(); + let len4 = vec4.len(); + ::core::mem::forget(vec4); + *ptr1 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::() = len4; + *ptr1 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>() = ptr4.cast_mut(); + } + }; + ptr1 + } + #[doc(hidden)] + #[allow(non_snake_case)] + pub unsafe fn __post_return_driver_config(arg0: *mut u8) { + let l0 = i32::from(*arg0.add(0).cast::()); + match l0 { + 0 => { + let l1 = *arg0 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l2 = *arg0 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::(); + let base3 = l1; + let len3 = l2; + _rt::cabi_dealloc(base3, len3 * 1, 1); + } + _ => { + let l4 = *arg0 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l5 = *arg0 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::(); + _rt::cabi_dealloc(l4, l5, 1); + } + } + } + #[doc(hidden)] + #[allow(non_snake_case)] + pub unsafe fn _export_driver_parse_cabi( + arg0: *mut u8, + arg1: usize, + ) -> *mut u8 { + #[cfg(target_arch = "wasm32")] _rt::run_ctors_once(); + let len0 = arg1; + let result1 = T::driver_parse( + _rt::Vec::from_raw_parts(arg0.cast(), len0, len0), + ); + let ptr2 = (&raw mut _RET_AREA.0).cast::(); + match result1 { + Ok(e) => { + *ptr2.add(0).cast::() = (0i32) as u8; + let vec3 = (e).into_boxed_slice(); + let ptr3 = vec3.as_ptr().cast::(); + let len3 = vec3.len(); + ::core::mem::forget(vec3); + *ptr2 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::() = len3; + *ptr2 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>() = ptr3.cast_mut(); + } + Err(e) => { + *ptr2.add(0).cast::() = (1i32) as u8; + let super::super::super::super::heph::plugin::types::PluginError { + kind: kind4, + message: message4, + } = e; + *ptr2 + .add(::core::mem::size_of::<*const u8>()) + .cast::() = (kind4.clone() as i32) as u8; + let vec5 = (message4.into_bytes()).into_boxed_slice(); + let ptr5 = vec5.as_ptr().cast::(); + let len5 = vec5.len(); + ::core::mem::forget(vec5); + *ptr2 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::() = len5; + *ptr2 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>() = ptr5.cast_mut(); + } + }; + ptr2 + } + #[doc(hidden)] + #[allow(non_snake_case)] + pub unsafe fn __post_return_driver_parse(arg0: *mut u8) { + let l0 = i32::from(*arg0.add(0).cast::()); + match l0 { + 0 => { + let l1 = *arg0 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l2 = *arg0 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::(); + let base3 = l1; + let len3 = l2; + _rt::cabi_dealloc(base3, len3 * 1, 1); + } + _ => { + let l4 = *arg0 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l5 = *arg0 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::(); + _rt::cabi_dealloc(l4, l5, 1); + } + } + } + #[doc(hidden)] + #[allow(non_snake_case)] + pub unsafe fn _export_driver_run_cabi( + arg0: *mut u8, + arg1: usize, + ) -> *mut u8 { + #[cfg(target_arch = "wasm32")] _rt::run_ctors_once(); + let len0 = arg1; + let result1 = T::driver_run( + _rt::Vec::from_raw_parts(arg0.cast(), len0, len0), + ); + let ptr2 = (&raw mut _RET_AREA.0).cast::(); + match result1 { + Ok(e) => { + *ptr2.add(0).cast::() = (0i32) as u8; + let vec3 = (e).into_boxed_slice(); + let ptr3 = vec3.as_ptr().cast::(); + let len3 = vec3.len(); + ::core::mem::forget(vec3); + *ptr2 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::() = len3; + *ptr2 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>() = ptr3.cast_mut(); + } + Err(e) => { + *ptr2.add(0).cast::() = (1i32) as u8; + let super::super::super::super::heph::plugin::types::PluginError { + kind: kind4, + message: message4, + } = e; + *ptr2 + .add(::core::mem::size_of::<*const u8>()) + .cast::() = (kind4.clone() as i32) as u8; + let vec5 = (message4.into_bytes()).into_boxed_slice(); + let ptr5 = vec5.as_ptr().cast::(); + let len5 = vec5.len(); + ::core::mem::forget(vec5); + *ptr2 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::() = len5; + *ptr2 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>() = ptr5.cast_mut(); + } + }; + ptr2 + } + #[doc(hidden)] + #[allow(non_snake_case)] + pub unsafe fn __post_return_driver_run(arg0: *mut u8) { + let l0 = i32::from(*arg0.add(0).cast::()); + match l0 { + 0 => { + let l1 = *arg0 + .add(::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l2 = *arg0 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::(); + let base3 = l1; + let len3 = l2; + _rt::cabi_dealloc(base3, len3 * 1, 1); + } + _ => { + let l4 = *arg0 + .add(2 * ::core::mem::size_of::<*const u8>()) + .cast::<*mut u8>(); + let l5 = *arg0 + .add(3 * ::core::mem::size_of::<*const u8>()) + .cast::(); + _rt::cabi_dealloc(l4, l5, 1); + } + } + } + pub trait Guest { + /// -> ConfigResponse (provider name). + fn provider_config() -> Result<_rt::Vec, PluginError>; + /// GetRequest -> GetResponse. + fn provider_get( + req: _rt::Vec, + ) -> Result<_rt::Vec, PluginError>; + /// -> ConfigResponse (driver name). + fn driver_config() -> Result<_rt::Vec, PluginError>; + /// ParseRequest -> ParseResponse. + fn driver_parse( + req: _rt::Vec, + ) -> Result<_rt::Vec, PluginError>; + /// RunRequest -> RunResponse. + fn driver_run( + req: _rt::Vec, + ) -> Result<_rt::Vec, PluginError>; + } + #[doc(hidden)] + macro_rules! __export_heph_plugin_api_cabi { + ($ty:ident with_types_in $($path_to_types:tt)*) => { + const _ : () = { #[unsafe (export_name = + "heph:plugin/api#provider-config")] unsafe extern "C" fn + export_provider_config() -> * mut u8 { unsafe { + $($path_to_types)*:: _export_provider_config_cabi::<$ty > () } } + #[unsafe (export_name = + "cabi_post_heph:plugin/api#provider-config")] unsafe extern "C" + fn _post_return_provider_config(arg0 : * mut u8,) { unsafe { + $($path_to_types)*:: __post_return_provider_config::<$ty > (arg0) + } } #[unsafe (export_name = "heph:plugin/api#provider-get")] + unsafe extern "C" fn export_provider_get(arg0 : * mut u8, arg1 : + usize,) -> * mut u8 { unsafe { $($path_to_types)*:: + _export_provider_get_cabi::<$ty > (arg0, arg1) } } #[unsafe + (export_name = "cabi_post_heph:plugin/api#provider-get")] unsafe + extern "C" fn _post_return_provider_get(arg0 : * mut u8,) { + unsafe { $($path_to_types)*:: __post_return_provider_get::<$ty > + (arg0) } } #[unsafe (export_name = + "heph:plugin/api#driver-config")] unsafe extern "C" fn + export_driver_config() -> * mut u8 { unsafe { + $($path_to_types)*:: _export_driver_config_cabi::<$ty > () } } + #[unsafe (export_name = + "cabi_post_heph:plugin/api#driver-config")] unsafe extern "C" fn + _post_return_driver_config(arg0 : * mut u8,) { unsafe { + $($path_to_types)*:: __post_return_driver_config::<$ty > (arg0) } + } #[unsafe (export_name = "heph:plugin/api#driver-parse")] unsafe + extern "C" fn export_driver_parse(arg0 : * mut u8, arg1 : usize,) + -> * mut u8 { unsafe { $($path_to_types)*:: + _export_driver_parse_cabi::<$ty > (arg0, arg1) } } #[unsafe + (export_name = "cabi_post_heph:plugin/api#driver-parse")] unsafe + extern "C" fn _post_return_driver_parse(arg0 : * mut u8,) { + unsafe { $($path_to_types)*:: __post_return_driver_parse::<$ty > + (arg0) } } #[unsafe (export_name = "heph:plugin/api#driver-run")] + unsafe extern "C" fn export_driver_run(arg0 : * mut u8, arg1 : + usize,) -> * mut u8 { unsafe { $($path_to_types)*:: + _export_driver_run_cabi::<$ty > (arg0, arg1) } } #[unsafe + (export_name = "cabi_post_heph:plugin/api#driver-run")] unsafe + extern "C" fn _post_return_driver_run(arg0 : * mut u8,) { unsafe + { $($path_to_types)*:: __post_return_driver_run::<$ty > (arg0) } + } }; + }; + } + #[doc(hidden)] + pub(crate) use __export_heph_plugin_api_cabi; + #[cfg_attr(target_pointer_width = "64", repr(align(8)))] + #[cfg_attr(target_pointer_width = "32", repr(align(4)))] + struct _RetArea( + [::core::mem::MaybeUninit< + u8, + >; 4 * ::core::mem::size_of::<*const u8>()], + ); + static mut _RET_AREA: _RetArea = _RetArea( + [::core::mem::MaybeUninit::uninit(); 4 + * ::core::mem::size_of::<*const u8>()], + ); + } + } + } +} +#[rustfmt::skip] +mod _rt { + #![allow(dead_code, clippy::all)] + pub use alloc_crate::string::String; + pub use alloc_crate::vec::Vec; + pub unsafe fn string_lift(bytes: Vec) -> String { + if cfg!(debug_assertions) { + String::from_utf8(bytes).unwrap() + } else { + String::from_utf8_unchecked(bytes) + } + } + pub unsafe fn invalid_enum_discriminant() -> T { + if cfg!(debug_assertions) { + panic!("invalid enum discriminant") + } else { + unsafe { core::hint::unreachable_unchecked() } + } + } + #[cfg(target_arch = "wasm32")] + pub fn run_ctors_once() { + wit_bindgen_rt::run_ctors_once(); + } + pub unsafe fn cabi_dealloc(ptr: *mut u8, size: usize, align: usize) { + if size == 0 { + return; + } + let layout = alloc::Layout::from_size_align_unchecked(size, align); + alloc::dealloc(ptr, layout); + } + extern crate alloc as alloc_crate; + pub use alloc_crate::alloc; +} +/// Generates `#[unsafe(no_mangle)]` functions to export the specified type as +/// the root implementation of all generated traits. +/// +/// For more information see the documentation of `wit_bindgen::generate!`. +/// +/// ```rust +/// # macro_rules! export{ ($($t:tt)*) => (); } +/// # trait Guest {} +/// struct MyType; +/// +/// impl Guest for MyType { +/// // ... +/// } +/// +/// export!(MyType); +/// ``` +#[allow(unused_macros)] +#[doc(hidden)] +macro_rules! __export_plugin_impl { + ($ty:ident) => { + self::export!($ty with_types_in self); + }; + ($ty:ident with_types_in $($path_to_types_root:tt)*) => { + $($path_to_types_root)*:: + exports::heph::plugin::api::__export_heph_plugin_api_cabi!($ty with_types_in + $($path_to_types_root)*:: exports::heph::plugin::api); + }; +} +#[doc(inline)] +pub(crate) use __export_plugin_impl as export; +#[cfg(target_arch = "wasm32")] +#[unsafe(link_section = "component-type:wit-bindgen:0.41.0:heph:plugin:plugin:encoded world")] +#[doc(hidden)] +#[allow(clippy::octal_escapes)] +pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; 555] = *b"\ +\0asm\x0d\0\x01\0\0\x19\x16wit-component-encoding\x04\0\x07\xae\x03\x01A\x02\x01\ +A\x07\x01B\x04\x01m\x05\x05other\x09not-found\x05cycle\x09cancelled\x0dunimpleme\ +nted\x04\0\x0aerror-kind\x03\0\0\x01r\x02\x04kind\x01\x07messages\x04\0\x0cplugi\ +n-error\x03\0\x02\x03\0\x11heph:plugin/types\x05\0\x02\x03\0\0\x0cplugin-error\x01\ +B\x08\x02\x03\x02\x01\x01\x04\0\x0cplugin-error\x03\0\0\x01p}\x01j\x01\x02\x01\x01\ +\x01@\x01\x03req\x02\0\x03\x04\0\x07resolve\x01\x04\x04\0\x08note-dep\x01\x04\x04\ +\0\x05query\x01\x04\x03\0\x10heph:plugin/host\x05\x02\x01B\x0b\x02\x03\x02\x01\x01\ +\x04\0\x0cplugin-error\x03\0\0\x01p}\x01j\x01\x02\x01\x01\x01@\0\0\x03\x04\0\x0f\ +provider-config\x01\x04\x01@\x01\x03req\x02\0\x03\x04\0\x0cprovider-get\x01\x05\x04\ +\0\x0ddriver-config\x01\x04\x04\0\x0cdriver-parse\x01\x05\x04\0\x0adriver-run\x01\ +\x05\x04\0\x0fheph:plugin/api\x05\x03\x04\0\x12heph:plugin/plugin\x04\0\x0b\x0c\x01\ +\0\x06plugin\x03\0\0\0G\x09producers\x01\x0cprocessed-by\x02\x0dwit-component\x07\ +0.227.1\x10wit-bindgen-rust\x060.41.0"; +#[inline(never)] +#[doc(hidden)] +pub fn __link_custom_section_describing_imports() { + wit_bindgen_rt::maybe_link_cabi_realloc(); +} diff --git a/crates/wasm-guests/helloworld/src/lib.rs b/crates/wasm-guests/helloworld/src/lib.rs new file mode 100644 index 00000000..e1c7c85a --- /dev/null +++ b/crates/wasm-guests/helloworld/src/lib.rs @@ -0,0 +1,214 @@ +//! Hello-world heph plugin as a wasm component: a provider and a driver. +//! +//! - provider `get` resolves any requested addr to a `hello`-driver target, and +//! calls back into the host (`query`) to prove the bidirectional callback path +//! — the count is stitched into a label. +//! - driver `parse` produces a `TargetDef` carrying the greeting in its opaque +//! `raw_def`; `run` returns one inline output artifact ("hello world"). +//! +//! The author code is written against the pb-typed SDK traits; the glue below +//! (decode in / dispatch / encode out / map errors to the WIT `plugin-error`) +//! is the only thing that touches the generated bindings. + +#[allow(warnings)] +mod bindings; + +use bindings::exports::heph::plugin::api::Guest; +use bindings::heph::plugin::host; +use bindings::heph::plugin::types::{ErrorKind as WErrorKind, PluginError as WErr}; + +use heph_plugin_sdk_wasm as sdk; +use sdk::pb; +use sdk::{Driver, ErrorKind, Host, PluginError, Provider}; + +// ---- error mapping between the SDK's typed error and the WIT record ---- + +fn kind_to_wit(k: ErrorKind) -> WErrorKind { + match k { + ErrorKind::Other => WErrorKind::Other, + ErrorKind::NotFound => WErrorKind::NotFound, + ErrorKind::Cycle => WErrorKind::Cycle, + ErrorKind::Cancelled => WErrorKind::Cancelled, + ErrorKind::Unimplemented => WErrorKind::Unimplemented, + } +} + +fn kind_from_wit(k: WErrorKind) -> ErrorKind { + match k { + WErrorKind::Other => ErrorKind::Other, + WErrorKind::NotFound => ErrorKind::NotFound, + WErrorKind::Cycle => ErrorKind::Cycle, + WErrorKind::Cancelled => ErrorKind::Cancelled, + WErrorKind::Unimplemented => ErrorKind::Unimplemented, + } +} + +fn to_wit(e: PluginError) -> WErr { + WErr { + kind: kind_to_wit(e.kind), + message: e.message, + } +} + +fn from_wit(e: WErr) -> PluginError { + PluginError { + kind: kind_from_wit(e.kind), + message: e.message, + } +} + +// ---- host bridge: the SDK callback surface over the generated WIT imports ---- + +struct HostBridge; + +impl Host for HostBridge { + fn result(&self, req: pb::ResultRequest) -> Result { + let bytes = host::resolve(&sdk::encode(&req)).map_err(from_wit)?; + sdk::decode(&bytes) + } + fn note_dep(&self, req: pb::NoteDepRequest) -> Result { + let bytes = host::note_dep(&sdk::encode(&req)).map_err(from_wit)?; + sdk::decode(&bytes) + } + fn query(&self, req: pb::QueryRequest) -> Result { + let bytes = host::query(&sdk::encode(&req)).map_err(from_wit)?; + sdk::decode(&bytes) + } +} + +// ---- the hello-world provider + driver (pb-typed author code) ---- + +struct HelloProvider; + +impl Provider for HelloProvider { + fn config(&self) -> Result { + Ok(sdk::config_response("hello")) + } + + fn get(&self, req: pb::GetRequest, host: &dyn Host) -> Result { + let addr = req.addr.clone().unwrap_or_default(); + // Prove the guest->host callback path: query the engine for everything in + // this package and fold the count into a label so the host can observe it. + let q = host.query(pb::QueryRequest { + request_id: req.request_id.clone(), + matcher: Some(pb::Matcher { + kind: Some(pb::matcher::Kind::Package(addr.package.clone())), + }), + extra_skip: vec![], + })?; + let mut config = std::collections::HashMap::new(); + config.insert( + "greeting".to_string(), + pb::Value { + kind: Some(pb::value::Kind::StringVal("hello world".to_string())), + }, + ); + Ok(pb::GetResponse { + target_spec: Some(pb::TargetSpec { + addr: Some(addr), + driver: "hello".to_string(), + config, + labels: vec![format!("queried:{}", q.addrs.len())], + transitive: None, + }), + }) + } +} + +struct HelloDriver; + +impl Driver for HelloDriver { + fn config(&self) -> Result { + Ok(sdk::config_response("hello")) + } + + fn parse(&self, req: pb::ParseRequest) -> Result { + let spec = req.target_spec.unwrap_or_default(); + let raw_def = pb::RawDefBlob { + driver: "hello".to_string(), + format: pb::raw_def_blob::Format::Json as i32, + data: br#"{"greeting":"hello world"}"#.to_vec().into(), + }; + Ok(pb::ParseResponse { + target_def: Some(pb::TargetDef { + addr: spec.addr, + labels: spec.labels, + raw_def: Some(raw_def), + inputs: vec![], + outputs: vec![pb::Output { + group: String::new(), + paths: vec![], + }], + support_files: vec![], + cache: None, + pty: false, + hash: vec![].into(), + transparent: false, + }), + }) + } + + fn run(&self, _req: pb::RunRequest) -> Result { + // Outputs are fresh bytes the driver "produced"; cross inline as Raw. + let artifact = pb::OutputArtifactRef { + group: String::new(), + name: "out".to_string(), + r#type: pb::ArtifactType::Output as i32, + content: Some(pb::output_artifact_ref::Content::Raw(pb::ContentRaw { + data: b"hello world".to_vec().into(), + path: "out.txt".to_string(), + x: false, + })), + hashout: String::new(), + }; + Ok(pb::RunResponse { + artifacts: vec![artifact], + }) + } +} + +// ---- generated-export glue ---- + +struct Component; + +impl Guest for Component { + fn provider_config() -> Result, WErr> { + HelloProvider + .config() + .map(|r| sdk::encode(&r)) + .map_err(to_wit) + } + + fn provider_get(req: Vec) -> Result, WErr> { + let req: pb::GetRequest = sdk::decode(&req).map_err(to_wit)?; + HelloProvider + .get(req, &HostBridge) + .map(|r| sdk::encode(&r)) + .map_err(to_wit) + } + + fn driver_config() -> Result, WErr> { + HelloDriver + .config() + .map(|r| sdk::encode(&r)) + .map_err(to_wit) + } + + fn driver_parse(req: Vec) -> Result, WErr> { + let req: pb::ParseRequest = sdk::decode(&req).map_err(to_wit)?; + HelloDriver + .parse(req) + .map(|r| sdk::encode(&r)) + .map_err(to_wit) + } + + fn driver_run(req: Vec) -> Result, WErr> { + let req: pb::RunRequest = sdk::decode(&req).map_err(to_wit)?; + HelloDriver + .run(req) + .map(|r| sdk::encode(&r)) + .map_err(to_wit) + } +} + +bindings::export!(Component with_types_in bindings); diff --git a/crates/wasm-guests/helloworld/wit/world.wit b/crates/wasm-guests/helloworld/wit/world.wit new file mode 100644 index 00000000..bd3d65ea --- /dev/null +++ b/crates/wasm-guests/helloworld/wit/world.wit @@ -0,0 +1,68 @@ +package heph:plugin; + +/// Shared types for the wasm tier of the heph external-plugin ABI. +/// +/// Payloads are deliberately `list` carrying protobuf-encoded +/// `heph.plugin.v1` messages (proto/plugin/v1/*.proto) — the SAME wire schema +/// the proto (UDS) and shm (iceoryx2) transports use. Keeping one schema source +/// (proto) avoids re-encoding every message as a WIT record and the three-way +/// drift that would create; the WIT here mirrors the *service* surface, not the +/// message structure. +interface types { + /// Typed error category so neither guest nor host ever matches on the + /// message string. Mirrors the wire `Error.Kind` / `GetError.Kind`. + enum error-kind { + other, + not-found, + cycle, + cancelled, + unimplemented, + } + record plugin-error { + kind: error-kind, + message: string, + } +} + +/// Callbacks the guest invokes back into the host while serving a request — the +/// AbiHost surface. This is the bidirectional crux: a guest provider calls these +/// to resolve dependencies through the engine's executor mid-`get`. +interface host { + use types.{plugin-error}; + + /// ResultRequest -> ResultResponse. Registers the parent->addr DepDag edge + /// host-side (cycle check) and returns artifact handles + a lease. + /// (Named `resolve`, not `result`: `result` is a WIT reserved keyword.) + resolve: func(req: list) -> result, plugin-error>; + + /// NoteDepRequest -> NoteDepResponse. Cache-hit fast path: register the dep + /// edge only. Returns the cycle flag when the edge closes a cycle. + note-dep: func(req: list) -> result, plugin-error>; + + /// QueryRequest -> QueryResponse. + query: func(req: list) -> result, plugin-error>; +} + +/// Methods the guest exports — the provider and driver surface. A single guest +/// component may serve both. Mirrors hplugin::Provider / hplugin::Driver. +/// (Named `api`, not `plugin`: a world and an interface can't share a name.) +interface api { + use types.{plugin-error}; + + /// -> ConfigResponse (provider name). + provider-config: func() -> result, plugin-error>; + /// GetRequest -> GetResponse. + provider-get: func(req: list) -> result, plugin-error>; + + /// -> ConfigResponse (driver name). + driver-config: func() -> result, plugin-error>; + /// ParseRequest -> ParseResponse. + driver-parse: func(req: list) -> result, plugin-error>; + /// RunRequest -> RunResponse. + driver-run: func(req: list) -> result, plugin-error>; +} + +world plugin { + import host; + export api; +} diff --git a/crates/wasm-guests/sdk/Cargo.lock b/crates/wasm-guests/sdk/Cargo.lock new file mode 100644 index 00000000..ad7acc97 --- /dev/null +++ b/crates/wasm-guests/sdk/Cargo.lock @@ -0,0 +1,450 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytecheck" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "rancor", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heph-plugin-sdk-wasm" +version = "0.1.0" +dependencies = [ + "plugin-abi", + "prost", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pbjson" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e6349fa080353f4a597daffd05cb81572a9c031a6d4fff7e504947496fcc68" +dependencies = [ + "base64", + "serde", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plugin-abi" +version = "0.1.0" +dependencies = [ + "anyhow", + "proto-gen", + "rkyv", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "proto-gen" +version = "0.1.0" +dependencies = [ + "pbjson", + "prost", + "prost-types", + "serde", +] + +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rancor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +dependencies = [ + "ptr_meta", +] + +[[package]] +name = "rend" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "rkyv" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +dependencies = [ + "bytecheck", + "bytes", + "hashbrown", + "indexmap", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] diff --git a/crates/wasm-guests/sdk/Cargo.toml b/crates/wasm-guests/sdk/Cargo.toml new file mode 100644 index 00000000..aee6a24b --- /dev/null +++ b/crates/wasm-guests/sdk/Cargo.toml @@ -0,0 +1,19 @@ +# Guest-side SDK for heph wasm plugins. Standalone workspace: compiled only for +# wasm32-wasip2 by the guest, kept out of the host `cargo build --workspace`. +# +# It depends on plugin-abi WITHOUT the `convert`/`transport` features — those +# pull in hplugin (-> hsandboxfuse FUSE, hwalk) which do not build for wasm. The +# guest works against the prost `pb` types directly; the host keeps `convert`. +[workspace] + +[package] +name = "heph-plugin-sdk-wasm" +version = "0.1.0" +edition = "2024" + +[dependencies] +plugin-abi = { path = "../../plugin-abi" } +prost = "0.14" + +[lib] +crate-type = ["rlib"] diff --git a/crates/wasm-guests/sdk/src/lib.rs b/crates/wasm-guests/sdk/src/lib.rs new file mode 100644 index 00000000..7a8385e2 --- /dev/null +++ b/crates/wasm-guests/sdk/src/lib.rs @@ -0,0 +1,101 @@ +//! Guest-side SDK for heph wasm plugins. +//! +//! A plugin author implements [`Provider`] and/or [`Driver`] against the prost +//! `pb` types, and calls the engine back through the [`Host`] callback surface. +//! The thin generated-bindings glue (decode pb in, dispatch, encode pb out, map +//! errors to the WIT `plugin-error`) lives in the guest crate; this SDK supplies +//! the pb-typed traits, the [`PluginError`] type, and the encode/decode helpers +//! so that glue stays mechanical. +//! +//! Payloads are protobuf-encoded `heph.plugin.v1` messages — the same wire +//! schema the proto and shm transports use (see `crates/plugin-abi/wit`). + +use prost::Message; + +/// Re-export of the prost wire types so guests need not depend on plugin-abi. +pub use plugin_abi::pb; + +/// Typed error category. Mirrors the WIT `error-kind` / wire `*.Kind` enums so +/// errors are classified structurally — never by matching the message string. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ErrorKind { + Other, + NotFound, + Cycle, + Cancelled, + Unimplemented, +} + +/// An error crossing the wasm boundary: a typed kind plus a human message. +#[derive(Debug, Clone)] +pub struct PluginError { + pub kind: ErrorKind, + pub message: String, +} + +impl PluginError { + pub fn new(kind: ErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + } + } + pub fn other(message: impl Into) -> Self { + Self::new(ErrorKind::Other, message) + } + pub fn not_found(message: impl Into) -> Self { + Self::new(ErrorKind::NotFound, message) + } + pub fn unimplemented(what: &str) -> Self { + Self::new(ErrorKind::Unimplemented, format!("{what} not implemented")) + } +} + +/// Decode a protobuf request payload, mapping a malformed payload to a typed +/// error rather than a panic. +pub fn decode(bytes: &[u8]) -> Result { + M::decode(bytes).map_err(|e| PluginError::other(format!("decoding request: {e}"))) +} + +/// Encode a protobuf response payload. +pub fn encode(msg: &M) -> Vec { + msg.encode_to_vec() +} + +/// The host callback surface (AbiHost), pb-typed. The guest glue implements this +/// by calling the generated WIT `host` imports; provider/driver code receives it +/// as `&dyn Host` and uses it to resolve dependencies through the engine. +pub trait Host { + fn result(&self, req: pb::ResultRequest) -> Result; + fn note_dep(&self, req: pb::NoteDepRequest) -> Result; + fn query(&self, req: pb::QueryRequest) -> Result; +} + +/// A provider implemented in wasm. Mirrors the in-process `hplugin::Provider` +/// surface that is relevant to the remote data path. +pub trait Provider { + /// Provider name. + fn config(&self) -> Result; + + /// Resolve one target. May call back into `host` to resolve dependencies. + fn get(&self, req: pb::GetRequest, host: &dyn Host) -> Result; +} + +/// A driver implemented in wasm. Mirrors `hplugin::Driver`. +pub trait Driver { + /// Driver name. + fn config(&self) -> Result; + + /// Parse a `TargetSpec` into a `TargetDef`. + fn parse(&self, req: pb::ParseRequest) -> Result; + + /// Execute the target, returning output artifacts. + fn run(&self, req: pb::RunRequest) -> Result; +} + +/// Convenience for guests that serve a config name. +pub fn config_response(name: &str) -> pb::ConfigResponse { + pb::ConfigResponse { + name: name.to_string(), + } +} From e0084ea0efcc34544576d1cedadab30574582b21 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 10:22:50 +0200 Subject: [PATCH 19/44] feat(config): out-of-process plugins via `bin:` entry; drop HEPH_REMOTE_* env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Providers/drivers are matched by name: no `bin:` => in-process built-in (unchanged); a `bin:` entry routes the name to a spawned plugin over the proto transport. Three launch forms (exactly one): - bin: { path: } spawn directly - bin: { exec: [argv...] } argv[0] PATH-resolved + args (e.g. cargo run) - bin: { url: } download, cache under /plugins/-/, chmod +x, then spawn bootstrap: - in-process built-in factories (buildfile/go/go_*/exec/bash/sh) are skipped for any name a `bin:` entry overrides. - register_bin_plugins groups entries by resolved argv so one process serves all names sharing a command (the `go` provider + its `go_*` drivers); providers register as provider factories, drivers as managed-driver factories. - spawned plugins get HEPH_PLUGIN_ROOT injected; heph-plugin-go reads it (HEPH_PLUGIN_GO_ROOT still overrides). - removes the HEPH_REMOTE_GO / HEPH_REMOTE_EXEC env opt-ins entirely — the go/exec plugins are now launched via a `bin:` entry (cargo run … or a local binary). `url` downloads use reqwest blocking on a dedicated thread (safe inside the async runtime). Adds BinConfig/BinSource to config_yaml with parse + validation tests, plus argv-resolution / os-arch-substitution / spawn-grouping unit tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + Cargo.toml | 4 + crates/engine/src/engine/config_yaml.rs | 148 ++++++++ crates/plugin-go/src/bin/heph-plugin-go.rs | 4 +- src/commands/bootstrap.rs | 415 ++++++++++++++++----- 5 files changed, 483 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc16af6a..e7beb872 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2344,6 +2344,7 @@ dependencies = [ "r2d2_sqlite", "ratatui", "rayon", + "reqwest 0.13.4", "rusqlite", "rustc-hash", "sandboxfuse", diff --git a/Cargo.toml b/Cargo.toml index 63604437..3e9519ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -166,6 +166,10 @@ uuid = { version = "1.23.3", features = ["v4"] } object_store = { version = "0.13.2", features = ["aws", "gcp", "azure", "http", "tokio"] } url = "2.5.8" flate2 = "1.1.9" +# Blocking HTTP client, used only to download `bin: { url }` plugins. rustls-tls +# avoids a system OpenSSL link; blocking runs on its own thread so it is safe to +# call from inside the async runtime (see bootstrap::download_plugin). +reqwest = { version = "0.13", default-features = false, features = ["blocking", "rustls"] } [features] # FUSE sandbox is on by default. The FUSE machinery (and its `fuser`/libfuse diff --git a/crates/engine/src/engine/config_yaml.rs b/crates/engine/src/engine/config_yaml.rs index 7fb7f05e..961408a6 100644 --- a/crates/engine/src/engine/config_yaml.rs +++ b/crates/engine/src/engine/config_yaml.rs @@ -336,6 +336,61 @@ pub struct PluginEntry { pub name: String, #[serde(default)] pub options: Options, + /// External-plugin launch spec. When present, this provider/driver runs + /// out-of-process and is matched by name to a spawned plugin; when absent, + /// the name resolves to an in-process built-in factory. + #[serde(default)] + pub bin: Option, +} + +/// How to launch an out-of-process plugin. Exactly one of `path`/`exec`/`url` +/// must be set: +/// - `path`: a binary on disk, spawned directly with no extra args. +/// - `exec`: an argv (`exec[0]` is the program, resolved via `PATH`; the rest +/// are args) — e.g. `["cargo", "run", "-p", "plugin-go"]`. +/// - `url`: a URL with `{os}`/`{arch}` placeholders; the binary is downloaded, +/// cached, made executable, then spawned. +#[derive(Debug, Deserialize, Clone, PartialEq, Eq)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct BinConfig { + #[serde(default)] + pub path: Option, + #[serde(default)] + pub exec: Option>, + #[serde(default)] + pub url: Option, +} + +/// The resolved, mutually-exclusive launch source of a [`BinConfig`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BinSource { + /// Spawn this binary directly (no args). + Path(String), + /// Spawn `argv[0]` (PATH-resolved) with `argv[1..]` as args. + Exec(Vec), + /// Download (after `{os}`/`{arch}` substitution), cache, then spawn. + Url(String), +} + +impl BinConfig { + /// Validate that exactly one of `path`/`exec`/`url` is set and return it. + /// `ctx` names the entry for error messages. + pub fn resolve(&self, ctx: &str) -> anyhow::Result { + match (&self.path, &self.exec, &self.url) { + (Some(p), None, None) => Ok(BinSource::Path(p.clone())), + (None, Some(e), None) => { + if e.is_empty() { + anyhow::bail!("plugin `{ctx}` bin.exec must not be empty"); + } + Ok(BinSource::Exec(e.clone())) + } + (None, None, Some(u)) => Ok(BinSource::Url(u.clone())), + (None, None, None) => { + anyhow::bail!("plugin `{ctx}` bin requires one of path/exec/url") + } + _ => anyhow::bail!("plugin `{ctx}` bin: set exactly one of path/exec/url"), + } + } } /// Canonical filename of the per-workspace config, located at the repo root. @@ -959,6 +1014,99 @@ caches: assert!(!r.write); } + #[test] + fn parses_bin_path_exec_url() { + let yaml = r#" +providers: + - name: go + bin: + exec: [cargo, run, -p, plugin-go] +drivers: + - name: exec + bin: + path: /usr/local/bin/heph-plugin-exec + - name: rust + bin: + url: https://example.com/rust-{os}-{arch} +"#; + let cfg: ConfigYaml = serde_yaml::from_str(yaml).expect("parse"); + assert_eq!( + cfg.providers[0] + .bin + .as_ref() + .unwrap() + .resolve("go") + .unwrap(), + BinSource::Exec(vec![ + "cargo".into(), + "run".into(), + "-p".into(), + "plugin-go".into() + ]) + ); + assert_eq!( + cfg.drivers[0] + .bin + .as_ref() + .unwrap() + .resolve("exec") + .unwrap(), + BinSource::Path("/usr/local/bin/heph-plugin-exec".into()) + ); + assert_eq!( + cfg.drivers[1] + .bin + .as_ref() + .unwrap() + .resolve("rust") + .unwrap(), + BinSource::Url("https://example.com/rust-{os}-{arch}".into()) + ); + } + + #[test] + fn bin_absent_is_in_process() { + let cfg: ConfigYaml = + serde_yaml::from_str("providers:\n - name: buildfile\n").expect("parse"); + assert!(cfg.providers[0].bin.is_none()); + } + + #[test] + fn bin_rejects_multiple_sources() { + let cfg: ConfigYaml = serde_yaml::from_str( + "drivers:\n - name: x\n bin:\n path: /a\n url: http://b\n", + ) + .expect("parse"); + let err = cfg.drivers[0] + .bin + .as_ref() + .unwrap() + .resolve("x") + .expect_err("must reject"); + assert!(err.to_string().contains("exactly one"), "{err}"); + } + + #[test] + fn bin_rejects_empty_exec() { + let cfg: ConfigYaml = + serde_yaml::from_str("drivers:\n - name: x\n bin:\n exec: []\n") + .expect("parse"); + let err = cfg.drivers[0] + .bin + .as_ref() + .unwrap() + .resolve("x") + .expect_err("must reject"); + assert!(err.to_string().contains("must not be empty"), "{err}"); + } + + #[test] + fn bin_rejects_unknown_field() { + let yaml = "drivers:\n - name: x\n bin:\n bogus: 1\n"; + let err = serde_yaml::from_str::(yaml).expect_err("must reject"); + assert!(err.to_string().contains("bogus"), "{err}"); + } + #[test] fn resolve_errors_on_cache_missing_uri() { let yaml: ConfigYaml = diff --git a/crates/plugin-go/src/bin/heph-plugin-go.rs b/crates/plugin-go/src/bin/heph-plugin-go.rs index 6754722e..faedbfc6 100644 --- a/crates/plugin-go/src/bin/heph-plugin-go.rs +++ b/crates/plugin-go/src/bin/heph-plugin-go.rs @@ -7,7 +7,8 @@ //! exactly as before. //! //! Config is passed by the host at spawn via env vars: -//! - `HEPH_PLUGIN_GO_ROOT` — workspace root (default: cwd) +//! - `HEPH_PLUGIN_ROOT` — workspace root (injected by the host for every +//! `bin:` plugin); `HEPH_PLUGIN_GO_ROOT` overrides it; default cwd. //! - `HEPH_PLUGIN_GO_BIN` — addr of the go toolchain (default `//@heph/bin:go`) //! - `HEPH_PLUGIN_GO_WALK_DB` — fs-walk cache db path (default: under root) @@ -20,6 +21,7 @@ use std::sync::Arc; #[tokio::main(flavor = "current_thread")] async fn main() -> anyhow::Result<()> { let root = std::env::var_os("HEPH_PLUGIN_GO_ROOT") + .or_else(|| std::env::var_os("HEPH_PLUGIN_ROOT")) .map(PathBuf::from) .unwrap_or(std::env::current_dir()?); let go_bin = diff --git a/src/commands/bootstrap.rs b/src/commands/bootstrap.rs index dedff1fc..38179970 100644 --- a/src/commands/bootstrap.rs +++ b/src/commands/bootstrap.rs @@ -97,26 +97,33 @@ pub fn new_engine() -> anyhow::Result<(Arc, ShutdownTrigger)> { e.register_driver(|_| Box::new(plugintextfile::Driver))?; e.register_managed_driver(|_| Box::new(pluginnix::Driver::new(home_dir.join("nix-driver"))))?; - // Opt-in factories — instantiated by `apply_config` if listed in the YAML. - e.register_provider_factory("buildfile", |init, opts| { - Ok(Box::new( - pluginbuildfile::Provider::from_options( - init.root.to_path_buf(), - &init.skip_dirs, - &init.skip_globs, - opts, - )? - .with_walker(init.walker.clone()), - )) - })?; - // The go plugin can run out-of-process (opt-in via HEPH_REMOTE_GO): one - // heph-plugin-go process serves the `go` provider + golist/embed/testmain - // drivers over the plugin transport. Default stays in-process. - let remote_go = std::env::var_os("HEPH_REMOTE_GO").is_some() - && tokio::runtime::Handle::try_current().is_ok(); - if remote_go { - register_remote_go(&mut e, &root)?; - } else { + // Names a `bin:` config entry routes to an out-of-process plugin. For those, + // skip the in-process built-in factory below and register a remote handle + // (spawned by `register_bin_plugins`) under the same name instead. + let bin_names: std::collections::HashSet<&str> = file + .providers + .iter() + .chain(file.drivers.iter()) + .filter(|en| en.bin.is_some()) + .map(|en| en.name.as_str()) + .collect(); + + // Opt-in built-in factories — instantiated by `apply_config` if listed in + // the YAML and not overridden by a `bin:` entry. + if !bin_names.contains("buildfile") { + e.register_provider_factory("buildfile", |init, opts| { + Ok(Box::new( + pluginbuildfile::Provider::from_options( + init.root.to_path_buf(), + &init.skip_dirs, + &init.skip_globs, + opts, + )? + .with_walker(init.walker.clone()), + )) + })?; + } + if !bin_names.contains("go") { e.register_provider_factory("go", |init, opts| { Ok(Box::new(plugingo::Provider::from_options( init.root.to_path_buf(), @@ -126,36 +133,46 @@ pub fn new_engine() -> anyhow::Result<(Arc, ShutdownTrigger)> { init.walker.clone(), )?)) })?; + } + if !bin_names.contains("go_golist") { e.register_managed_driver_factory("go_golist", |_init, opts| { config_yaml::deny_unknown("go_golist driver", opts, &[])?; Ok(Box::new(plugingo::GoGolistDriver::new("//@heph/bin:go"))) })?; + } + if !bin_names.contains("go_embed") { e.register_managed_driver_factory("go_embed", |_init, opts| { config_yaml::deny_unknown("go_embed driver", opts, &[])?; Ok(Box::new(plugingo::GoEmbedDriver)) })?; + } + if !bin_names.contains("go_testmain") { e.register_managed_driver_factory("go_testmain", |_init, opts| { config_yaml::deny_unknown("go_testmain driver", opts, &[])?; Ok(Box::new(plugingo::GoTestmainDriver)) })?; } - - // exec/bash/sh can likewise run out-of-process (opt-in via HEPH_REMOTE_EXEC). - let remote_exec = std::env::var_os("HEPH_REMOTE_EXEC").is_some() - && tokio::runtime::Handle::try_current().is_ok(); - if remote_exec { - register_remote_exec(&mut e)?; - } else { + if !bin_names.contains("exec") { e.register_managed_driver_factory("exec", |_init, opts| { Ok(Box::new(pluginexec::Driver::from_options_exec(opts)?)) })?; + } + if !bin_names.contains("bash") { e.register_managed_driver_factory("bash", |_init, opts| { Ok(Box::new(pluginexec::Driver::from_options_bash(opts)?)) })?; + } + if !bin_names.contains("sh") { e.register_managed_driver_factory("sh", |_init, opts| { Ok(Box::new(pluginexec::Driver::from_options_sh(opts)?)) })?; } + drop(bin_names); + + // Spawn + register every out-of-process plugin declared via `bin:`. One + // process per distinct launch command; all the names that share it (e.g. the + // `go` provider + its `go_*` drivers) register against that one connection. + register_bin_plugins(&mut e, &root, &home_dir, &file)?; e.apply_config(&file.providers, &file.drivers)?; @@ -413,79 +430,301 @@ drivers: "trigger after resume must cancel" ); } + + #[test] + fn substitute_os_arch_replaces_placeholders() { + let out = super::substitute_os_arch("https://x/heph-{os}-{arch}.bin"); + assert!(out.contains(std::env::consts::OS), "{out}"); + assert!(out.contains(std::env::consts::ARCH), "{out}"); + assert!(!out.contains("{os}") && !out.contains("{arch}"), "{out}"); + } + + #[cfg(unix)] + #[test] + fn resolve_bin_argv_path_and_exec() { + use crate::engine::config_yaml::BinConfig; + let home = std::path::Path::new("/tmp/unused"); + let path = BinConfig { + path: Some("/usr/local/bin/heph-plugin-go".into()), + exec: None, + url: None, + }; + assert_eq!( + super::resolve_bin_argv(&path, "go", home).unwrap(), + vec!["/usr/local/bin/heph-plugin-go".to_string()] + ); + let exec = BinConfig { + path: None, + exec: Some(vec![ + "cargo".into(), + "run".into(), + "-p".into(), + "plugin-go".into(), + ]), + url: None, + }; + assert_eq!( + super::resolve_bin_argv(&exec, "go", home).unwrap(), + vec![ + "cargo".to_string(), + "run".to_string(), + "-p".to_string(), + "plugin-go".to_string() + ] + ); + } + + #[cfg(unix)] + #[test] + fn plan_bin_groups_collapses_shared_command() { + // A provider + two drivers all pointing at the same binary collapse into + // one spawn group; a driver with a different binary is its own group. + let yaml = r#" +providers: + - name: go + bin: + path: /opt/heph-plugin-go +drivers: + - name: go_golist + bin: + path: /opt/heph-plugin-go + - name: go_embed + bin: + path: /opt/heph-plugin-go + - name: exec + bin: + path: /opt/heph-plugin-exec +"#; + let file: config_yaml::ConfigYaml = serde_yaml::from_str(yaml).expect("parse"); + let home = std::path::Path::new("/tmp/unused"); + let groups = super::plan_bin_groups(&file, home).expect("plan"); + assert_eq!(groups.len(), 2, "two distinct binaries => two groups"); + + let go = groups + .get(&vec!["/opt/heph-plugin-go".to_string()]) + .expect("go group"); + assert_eq!( + go, + &vec![ + ("go".to_string(), super::BinKind::Provider), + ("go_golist".to_string(), super::BinKind::Driver), + ("go_embed".to_string(), super::BinKind::Driver), + ] + ); + let exec = groups + .get(&vec!["/opt/heph-plugin-exec".to_string()]) + .expect("exec group"); + assert_eq!(exec, &vec![("exec".to_string(), super::BinKind::Driver)]); + } + + #[cfg(unix)] + #[test] + fn plan_bin_groups_empty_without_bin() { + let file: config_yaml::ConfigYaml = + serde_yaml::from_str("providers:\n - name: buildfile\n").expect("parse"); + let groups = super::plan_bin_groups(&file, std::path::Path::new("/tmp")).expect("plan"); + assert!(groups.is_empty()); + } +} + +/// Whether a `bin` entry contributes a provider or a (managed) driver. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum BinKind { + Provider, + Driver, } -/// Spawn `heph-plugin-go` (next to the heph executable) and register its `go` -/// provider + golist/embed/testmain managed drivers as remote handles sharing -/// one connection. Opt-in via `HEPH_REMOTE_GO`. plugin-go stays Rust — this just -/// runs it out-of-process. +/// Spawn plan: launch argv → the named provider/driver entries served by that +/// one process. #[cfg(unix)] -fn register_remote_go(e: &mut engine::Engine, root: &std::path::Path) -> anyhow::Result<()> { - let exe = std::env::current_exe().context("locate heph executable")?; - let bin = exe - .parent() - .map(|p| p.join("heph-plugin-go")) - .context("heph-plugin-go directory")?; - let env = vec![ - ( - "HEPH_PLUGIN_GO_ROOT".to_string(), - root.to_string_lossy().into_owned(), - ), - ("HEPH_PLUGIN_GO_BIN".to_string(), "//@heph/bin:go".to_string()), - ]; - let ((r, w), child) = hplugin_remote::spawn_streams(&bin, &[], &env) - .with_context(|| format!("spawn {}", bin.display()))?; - // The plugin self-exits when our end of the socket closes (engine drop), so - // we don't reap the child eagerly. - // Dropping the handle does not kill the child (std detaches); the plugin - // self-exits when our socket end closes at engine drop. - drop(child); - let plugin = hplugin_remote::RemotePlugin::connect(r, w); - // Register as factories (same opt-in semantics as in-process): only - // activated when the config lists `go` / `go_*`. - { - let p = plugin.clone(); - e.register_provider_factory("go", move |_init, _opts| Ok(Box::new(p.provider("go"))))?; +type BinGroups = std::collections::BTreeMap, Vec<(String, BinKind)>>; + +/// Plan which out-of-process plugins to spawn: resolve every `bin:` entry to its +/// launch argv and group entries that share the identical command, so one +/// process serves all of them (e.g. the `go` provider + its `go_*` drivers). +/// `url` sources are downloaded here. Pure of engine state so it is unit-tested. +#[cfg(unix)] +fn plan_bin_groups( + file: &config_yaml::ConfigYaml, + home_dir: &std::path::Path, +) -> anyhow::Result { + let mut groups: BinGroups = std::collections::BTreeMap::new(); + for (entries, kind) in [ + (&file.providers, BinKind::Provider), + (&file.drivers, BinKind::Driver), + ] { + for entry in entries { + if let Some(bin) = &entry.bin { + let argv = resolve_bin_argv(bin, &entry.name, home_dir)?; + groups + .entry(argv) + .or_default() + .push((entry.name.clone(), kind)); + } + } } - for name in ["go_golist", "go_embed", "go_testmain"] { - let p = plugin.clone(); - e.register_managed_driver_factory(name, move |_init, _opts| { - Ok(Box::new(p.managed_driver(name))) - })?; + Ok(groups) +} + +/// Spawn every out-of-process plugin declared via a `bin:` config entry and +/// register a remote handle under each entry's name. Entries that resolve to the +/// identical launch command share one spawned process + connection (so the `go` +/// provider and its `go_*` drivers, all pointing at the same binary, run in one +/// process). plugin-go/exec stay Rust binaries — `bin.exec: [cargo, run, ...]` +/// or `bin.path: /usr/local/bin/heph-plugin-go` is how they are now launched. +#[cfg(unix)] +fn register_bin_plugins( + e: &mut engine::Engine, + root: &std::path::Path, + home_dir: &std::path::Path, + file: &config_yaml::ConfigYaml, +) -> anyhow::Result<()> { + // Group entries by resolved argv so identical launch commands collapse to one + // process (deterministic order via BTreeMap). + let groups = plan_bin_groups(file, home_dir)?; + if groups.is_empty() { + return Ok(()); + } + + // Spawned plugins discover the workspace root through this env var. + let env = vec![( + "HEPH_PLUGIN_ROOT".to_string(), + root.to_string_lossy().into_owned(), + )]; + + for (argv, members) in groups { + let (program, args) = argv + .split_first() + .context("internal: empty plugin argv after resolution")?; + let ((r, w), child) = + hplugin_remote::spawn_streams(std::path::Path::new(program), args, &env) + .with_context(|| format!("spawn plugin `{program}`"))?; + // Dropping the handle does not kill the child (std detaches); the plugin + // self-exits when our socket end closes at engine drop. + drop(child); + let plugin = hplugin_remote::RemotePlugin::connect(r, w); + for (name, kind) in members { + match kind { + BinKind::Provider => { + let p = plugin.clone(); + let nm = name.clone(); + e.register_provider_factory(&name, move |_init, _opts| { + Ok(Box::new(p.provider(nm))) + })?; + } + BinKind::Driver => { + let p = plugin.clone(); + let nm = name.clone(); + e.register_managed_driver_factory(&name, move |_init, _opts| { + Ok(Box::new(p.managed_driver(nm))) + })?; + } + } + } } Ok(()) } #[cfg(not(unix))] -fn register_remote_go(_e: &mut engine::Engine, _root: &std::path::Path) -> anyhow::Result<()> { - anyhow::bail!("HEPH_REMOTE_GO is only supported on unix") +fn register_bin_plugins( + _e: &mut engine::Engine, + _root: &std::path::Path, + _home_dir: &std::path::Path, + file: &config_yaml::ConfigYaml, +) -> anyhow::Result<()> { + if file + .providers + .iter() + .chain(file.drivers.iter()) + .any(|en| en.bin.is_some()) + { + anyhow::bail!("out-of-process `bin:` plugins are only supported on unix"); + } + Ok(()) } -/// Spawn `heph-plugin-exec` and register its exec/bash/sh managed drivers as -/// remote handles sharing one connection. Opt-in via `HEPH_REMOTE_EXEC`. +/// Resolve a [`config_yaml::BinConfig`] to a spawnable argv (`argv[0]` is the +/// program). `url` sources are downloaded + cached first; `path`/`exec` are +/// used as-is. #[cfg(unix)] -fn register_remote_exec(e: &mut engine::Engine) -> anyhow::Result<()> { - let exe = std::env::current_exe().context("locate heph executable")?; - let bin = exe - .parent() - .map(|p| p.join("heph-plugin-exec")) - .context("heph-plugin-exec directory")?; - let ((r, w), child) = hplugin_remote::spawn_streams(&bin, &[], &[]) - .with_context(|| format!("spawn {}", bin.display()))?; - // Dropping the handle does not kill the child (std detaches); the plugin - // self-exits when our socket end closes at engine drop. - drop(child); - let plugin = hplugin_remote::RemotePlugin::connect(r, w); - for name in ["exec", "bash", "sh"] { - let p = plugin.clone(); - e.register_managed_driver_factory(name, move |_init, _opts| { - Ok(Box::new(p.managed_driver(name))) - })?; - } - Ok(()) +fn resolve_bin_argv( + bin: &config_yaml::BinConfig, + ctx: &str, + home_dir: &std::path::Path, +) -> anyhow::Result> { + Ok(match bin.resolve(ctx)? { + config_yaml::BinSource::Path(p) => vec![p], + config_yaml::BinSource::Exec(argv) => argv, + config_yaml::BinSource::Url(url) => { + vec![ + download_plugin(&url, home_dir)? + .to_string_lossy() + .into_owned(), + ] + } + }) } -#[cfg(not(unix))] -fn register_remote_exec(_e: &mut engine::Engine) -> anyhow::Result<()> { - anyhow::bail!("HEPH_REMOTE_EXEC is only supported on unix") +/// Substitute `{os}`/`{arch}` in a plugin download URL with this host's values +/// (`std::env::consts::OS` / `ARCH`, e.g. `linux`/`macos`, `x86_64`/`aarch64`). +fn substitute_os_arch(url: &str) -> String { + url.replace("{os}", std::env::consts::OS) + .replace("{arch}", std::env::consts::ARCH) +} + +/// Download a plugin binary from `url_tmpl` (after `{os}`/`{arch}` substitution), +/// cache it under `/plugins/-/`, make it executable, and return +/// its path. A previously-downloaded binary is reused (no re-fetch). +#[cfg(unix)] +fn download_plugin( + url_tmpl: &str, + home_dir: &std::path::Path, +) -> anyhow::Result { + use std::io::Write; + use std::os::unix::fs::PermissionsExt; + + let url = substitute_os_arch(url_tmpl); + let filename = url + .rsplit('/') + .next() + .filter(|s| !s.is_empty()) + .unwrap_or("plugin"); + let dir = home_dir.join("plugins").join(format!( + "{}-{}", + std::env::consts::OS, + std::env::consts::ARCH + )); + let dest = dir.join(filename); + if dest.exists() { + return Ok(dest); + } + std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?; + + // reqwest::blocking spins up its own runtime; run it on a dedicated std + // thread so it is safe to call from within the async runtime new_engine runs + // on (a nested block_on would otherwise panic). + let url_for_thread = url.clone(); + let bytes = std::thread::spawn(move || -> anyhow::Result> { + let resp = reqwest::blocking::get(&url_for_thread) + .with_context(|| format!("GET {url_for_thread}"))? + .error_for_status() + .with_context(|| format!("GET {url_for_thread}"))?; + Ok(resp.bytes()?.to_vec()) + }) + .join() + .map_err(|_e| anyhow::anyhow!("plugin download thread panicked"))??; + + // Write to a temp path then rename so a partial download is never seen as a + // usable binary by a concurrent run. + let tmp = dir.join(format!(".{filename}.download")); + { + let mut f = + std::fs::File::create(&tmp).with_context(|| format!("create {}", tmp.display()))?; + f.write_all(&bytes)?; + f.flush()?; + } + std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o755))?; + std::fs::rename(&tmp, &dest) + .with_context(|| format!("install plugin to {}", dest.display()))?; + Ok(dest) } From f7256094a41271afb81afa4ea7b9e2ce59620cd4 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 10:34:03 +0200 Subject: [PATCH 20/44] docs(example): run go plugin out-of-process via cargo run --release Demonstrates the new `bin:` config: the go provider and its go_* drivers point at the same `cargo run --release -p plugin-go --bin heph-plugin-go` command, so they share one spawned process. Co-Authored-By: Claude Opus 4.8 (1M context) --- example/.hephconfig2 | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/example/.hephconfig2 b/example/.hephconfig2 index f2a7bf6c..88ff4d21 100644 --- a/example/.hephconfig2 +++ b/example/.hephconfig2 @@ -3,14 +3,26 @@ providers: options: patterns: - BUILD + # The go plugin runs out-of-process. `bin.exec` launches it via cargo in + # release mode; the provider + its go_* drivers share one process because they + # resolve to the same launch command. Swap for `bin: { path: }` + # to use a prebuilt binary. - name: go + bin: + exec: [cargo, run, --release, -p, plugin-go, --bin, heph-plugin-go] drivers: - name: exec - name: bash - name: sh - name: go_golist + bin: + exec: [cargo, run, --release, -p, plugin-go, --bin, heph-plugin-go] - name: go_embed + bin: + exec: [cargo, run, --release, -p, plugin-go, --bin, heph-plugin-go] - name: go_testmain + bin: + exec: [cargo, run, --release, -p, plugin-go, --bin, heph-plugin-go] #fuse: # enabled: true # Remote (shared) caches. Each entry is keyed by a name and has a `uri` plus From b313245b988beb1ea32b81af7203a89e8f6e4dd0 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 10:39:56 +0200 Subject: [PATCH 21/44] ci: publish heph-plugin-go release artifacts per os/arch Builds heph-plugin-go alongside heph in the same cargo invocation across the existing matrix (linux amd64/arm64, darwin arm64), applies the macOS libiconv portability rewrite, and uploads it as a release asset (heph-plugin-go__) to hephbuild/heph-artifacts-v1. Lets a workspace launch the go plugin via `bin: { url: ... }` without building it. Release download pattern widened heph_* -> heph* to include the plugin (still excludes the `repo` source artifact). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/heph.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/heph.yml b/.github/workflows/heph.yml index dfe66875..a9d911d3 100644 --- a/.github/workflows/heph.yml +++ b/.github/workflows/heph.yml @@ -169,24 +169,32 @@ jobs: env: CARGO_TARGET_DIR: "./target" BIN_NAME: "heph_${{ matrix.os }}_${{ matrix.arch }}" + PLUGIN_GO_NAME: "heph-plugin-go_${{ matrix.os }}_${{ matrix.arch }}" HEPH_BUILD_VERSION: "${{needs.gen.outputs.version}}" run: | echo "bin_name=$BIN_NAME" >> $GITHUB_OUTPUT + echo "plugin_go_name=$PLUGIN_GO_NAME" >> $GITHUB_OUTPUT + # Build the CLI and the out-of-process go plugin together (one cargo + # invocation shares the compile). The go plugin is published so a + # workspace can launch it via `bin: { url: ... }` without building it. + OUT="$CARGO_TARGET_DIR/${{ matrix.target }}/release" # FUSE (default feature) on Linux is pure-Rust (no libfuse link), so # zigbuild cross-compiles cleanly. On macOS it links libfuse via the # devenv macfuse-stubs; zig mangles the nix `-L` path under its SDK # syslibroot and fails to find `-lfuse`, so the (native arm64) macOS # runner uses plain `cargo build` — no cross needed there. if [ "${{ matrix.os }}" = "darwin" ]; then - cargo build --release --locked --target ${{ matrix.target }} --bin heph + cargo build --release --locked --target ${{ matrix.target }} --bin heph --bin heph-plugin-go # The nix toolchain hard-links libiconv against its /nix/store path, # which is absent on user machines (dyld aborts at launch). Rewrite # those load commands to the OS /usr/lib copies. - bash scripts/macos-portable.sh $CARGO_TARGET_DIR/${{ matrix.target }}/release/heph + bash scripts/macos-portable.sh "$OUT/heph" + bash scripts/macos-portable.sh "$OUT/heph-plugin-go" else - cargo zigbuild --release --locked --target ${{ matrix.target }} --bin heph + cargo zigbuild --release --locked --target ${{ matrix.target }} --bin heph --bin heph-plugin-go fi - cp $CARGO_TARGET_DIR/${{ matrix.target }}/release/heph $BIN_NAME + cp "$OUT/heph" $BIN_NAME + cp "$OUT/heph-plugin-go" $PLUGIN_GO_NAME - name: sccache stats if: always() @@ -208,6 +216,12 @@ jobs: name: ${{steps.build.outputs.bin_name}} path: ${{steps.build.outputs.bin_name}} + - name: Upload go plugin artifact + uses: actions/upload-artifact@v6 + with: + name: ${{steps.build.outputs.plugin_go_name}} + path: ${{steps.build.outputs.plugin_go_name}} + upload_artifacts: name: Upload artifacts needs: [gen, build] @@ -220,7 +234,9 @@ jobs: - name: Download all artifacts uses: actions/download-artifact@v7 with: - pattern: "heph_*" + # Matches the CLI (`heph__`) and the go plugin + # (`heph-plugin-go__`); excludes the `repo` source artifact. + pattern: "heph*" path: dist merge-multiple: true From 07fc7eb94fcbfa7cec2511a993d823a63e9d40b3 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 10:42:04 +0200 Subject: [PATCH 22/44] fix(config): map bin url {os}/{arch} to published artifact spellings download_plugin now substitutes {os} -> linux/darwin and {arch} -> amd64/arm64 (from the rust consts macos / x86_64 / aarch64), matching the CI release asset names (heph-plugin-go__), so `bin: { url }` resolves correctly. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/commands/bootstrap.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/commands/bootstrap.rs b/src/commands/bootstrap.rs index 38179970..4f453518 100644 --- a/src/commands/bootstrap.rs +++ b/src/commands/bootstrap.rs @@ -434,9 +434,11 @@ drivers: #[test] fn substitute_os_arch_replaces_placeholders() { let out = super::substitute_os_arch("https://x/heph-{os}-{arch}.bin"); - assert!(out.contains(std::env::consts::OS), "{out}"); - assert!(out.contains(std::env::consts::ARCH), "{out}"); assert!(!out.contains("{os}") && !out.contains("{arch}"), "{out}"); + // arch uses the published spelling (amd64/arm64), never the rust consts. + assert!(!out.contains("x86_64") && !out.contains("aarch64"), "{out}"); + // os uses darwin, never the rust `macos` spelling. + assert!(!out.contains("macos"), "{out}"); } #[cfg(unix)] @@ -665,11 +667,21 @@ fn resolve_bin_argv( }) } -/// Substitute `{os}`/`{arch}` in a plugin download URL with this host's values -/// (`std::env::consts::OS` / `ARCH`, e.g. `linux`/`macos`, `x86_64`/`aarch64`). +/// Substitute `{os}`/`{arch}` in a plugin download URL with the values used in +/// published artifact names: os `linux`/`darwin` and arch `amd64`/`arm64` +/// (mapped from the rust `std::env::consts` spellings `macos` and +/// `x86_64`/`aarch64`). Unmapped hosts fall back to the raw consts value. fn substitute_os_arch(url: &str) -> String { - url.replace("{os}", std::env::consts::OS) - .replace("{arch}", std::env::consts::ARCH) + let os = match std::env::consts::OS { + "macos" => "darwin", + other => other, + }; + let arch = match std::env::consts::ARCH { + "x86_64" => "amd64", + "aarch64" => "arm64", + other => other, + }; + url.replace("{os}", os).replace("{arch}", arch) } /// Download a plugin binary from `url_tmpl` (after `{os}`/`{arch}` substitution), From ae42cd38ab2c597bed4dca939632764c9fcea2a3 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 11:17:25 +0200 Subject: [PATCH 23/44] feat(example): gen-example builds go plugin to .heph3; bin.path resolves vs root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - devenv.nix: add `gen-example` — runs gen-go-large, builds heph-plugin-go (release; macOS libiconv rewrite), installs it to example/.heph3/heph-go-plugin. - example/.hephconfig2: go provider + go_* drivers now use `bin: { path: .heph3/heph-go-plugin }` instead of `cargo run`. - bootstrap: a relative `bin.path` resolves against the workspace root (not the process cwd) so it works regardless of where heph is invoked. exec/url unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- devenv.nix | 18 +++++++++++++++++ example/.hephconfig2 | 16 +++++++-------- src/commands/bootstrap.rs | 42 ++++++++++++++++++++++++++++++--------- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/devenv.nix b/devenv.nix index 62b83d55..05116119 100644 --- a/devenv.nix +++ b/devenv.nix @@ -66,6 +66,24 @@ in go run . -seed 42 -out $DEVENV_ROOT/example/go/large -module example.com/large -pkgs 500 -max-depth 7 cd $DEVENV_ROOT/example/go/large && go mod tidy ''; + # Set up the example workspace end to end: regenerate the large go repo and + # build the out-of-process go plugin into example/.heph3/heph-go-plugin, which + # example/.hephconfig2 launches via `bin: { path: .heph3/heph-go-plugin }`. + scripts.gen-example.exec = '' + gen-go-large + cargo build --release --bin heph-plugin-go + bin="$CARGO_TARGET_DIR/release/heph-plugin-go" + if [ "$(uname -s)" = "Darwin" ]; then + # Rewrite the nix-store libiconv load command to /usr/lib so the spawned + # plugin keeps launching after the store path is GC'd (same as the CLI). + bash "$DEVENV_ROOT/scripts/macos-portable.sh" "$bin" + fi + dest="$DEVENV_ROOT/example/.heph3/heph-go-plugin" + mkdir -p "$(dirname "$dest")" + # Atomic replace (new inode) so a running macOS process keeps its signature. + cp "$bin" "$dest.new" + mv -f "$dest.new" "$dest" + ''; scripts.lint.exec = "echo '> clippy' && cargo clippy --all-targets --locked -- -D warnings && echo '> fmt' && cargo fmt --check ${qualityCrates}"; scripts.fix.exec = "cargo fix --allow-dirty && cargo fmt ${qualityCrates}"; scripts.tst.exec = "cargo test --locked --all"; diff --git a/example/.hephconfig2 b/example/.hephconfig2 index 88ff4d21..3a9136bc 100644 --- a/example/.hephconfig2 +++ b/example/.hephconfig2 @@ -3,26 +3,26 @@ providers: options: patterns: - BUILD - # The go plugin runs out-of-process. `bin.exec` launches it via cargo in - # release mode; the provider + its go_* drivers share one process because they - # resolve to the same launch command. Swap for `bin: { path: }` - # to use a prebuilt binary. + # The go plugin runs out-of-process. `bin.path` (relative to the workspace + # root) points at the binary built by the `gen-example` devenv script. The + # provider + its go_* drivers share one process because they resolve to the + # same launch command. - name: go bin: - exec: [cargo, run, --release, -p, plugin-go, --bin, heph-plugin-go] + path: .heph3/heph-go-plugin drivers: - name: exec - name: bash - name: sh - name: go_golist bin: - exec: [cargo, run, --release, -p, plugin-go, --bin, heph-plugin-go] + path: .heph3/heph-go-plugin - name: go_embed bin: - exec: [cargo, run, --release, -p, plugin-go, --bin, heph-plugin-go] + path: .heph3/heph-go-plugin - name: go_testmain bin: - exec: [cargo, run, --release, -p, plugin-go, --bin, heph-plugin-go] + path: .heph3/heph-go-plugin #fuse: # enabled: true # Remote (shared) caches. Each entry is keyed by a name and has a `uri` plus diff --git a/src/commands/bootstrap.rs b/src/commands/bootstrap.rs index 4f453518..38c790b5 100644 --- a/src/commands/bootstrap.rs +++ b/src/commands/bootstrap.rs @@ -445,6 +445,7 @@ drivers: #[test] fn resolve_bin_argv_path_and_exec() { use crate::engine::config_yaml::BinConfig; + let root = std::path::Path::new("/repo"); let home = std::path::Path::new("/tmp/unused"); let path = BinConfig { path: Some("/usr/local/bin/heph-plugin-go".into()), @@ -452,9 +453,19 @@ drivers: url: None, }; assert_eq!( - super::resolve_bin_argv(&path, "go", home).unwrap(), + super::resolve_bin_argv(&path, "go", root, home).unwrap(), vec!["/usr/local/bin/heph-plugin-go".to_string()] ); + // Relative path resolves against the workspace root, not the cwd. + let rel = BinConfig { + path: Some(".heph3/heph-go-plugin".into()), + exec: None, + url: None, + }; + assert_eq!( + super::resolve_bin_argv(&rel, "go", root, home).unwrap(), + vec!["/repo/.heph3/heph-go-plugin".to_string()] + ); let exec = BinConfig { path: None, exec: Some(vec![ @@ -466,7 +477,7 @@ drivers: url: None, }; assert_eq!( - super::resolve_bin_argv(&exec, "go", home).unwrap(), + super::resolve_bin_argv(&exec, "go", root, home).unwrap(), vec![ "cargo".to_string(), "run".to_string(), @@ -498,8 +509,9 @@ drivers: path: /opt/heph-plugin-exec "#; let file: config_yaml::ConfigYaml = serde_yaml::from_str(yaml).expect("parse"); + let root = std::path::Path::new("/repo"); let home = std::path::Path::new("/tmp/unused"); - let groups = super::plan_bin_groups(&file, home).expect("plan"); + let groups = super::plan_bin_groups(&file, root, home).expect("plan"); assert_eq!(groups.len(), 2, "two distinct binaries => two groups"); let go = groups @@ -524,7 +536,12 @@ drivers: fn plan_bin_groups_empty_without_bin() { let file: config_yaml::ConfigYaml = serde_yaml::from_str("providers:\n - name: buildfile\n").expect("parse"); - let groups = super::plan_bin_groups(&file, std::path::Path::new("/tmp")).expect("plan"); + let groups = super::plan_bin_groups( + &file, + std::path::Path::new("/repo"), + std::path::Path::new("/tmp"), + ) + .expect("plan"); assert!(groups.is_empty()); } } @@ -548,6 +565,7 @@ type BinGroups = std::collections::BTreeMap, Vec<(String, BinKind)>> #[cfg(unix)] fn plan_bin_groups( file: &config_yaml::ConfigYaml, + root: &std::path::Path, home_dir: &std::path::Path, ) -> anyhow::Result { let mut groups: BinGroups = std::collections::BTreeMap::new(); @@ -557,7 +575,7 @@ fn plan_bin_groups( ] { for entry in entries { if let Some(bin) = &entry.bin { - let argv = resolve_bin_argv(bin, &entry.name, home_dir)?; + let argv = resolve_bin_argv(bin, &entry.name, root, home_dir)?; groups .entry(argv) .or_default() @@ -583,7 +601,7 @@ fn register_bin_plugins( ) -> anyhow::Result<()> { // Group entries by resolved argv so identical launch commands collapse to one // process (deterministic order via BTreeMap). - let groups = plan_bin_groups(file, home_dir)?; + let groups = plan_bin_groups(file, root, home_dir)?; if groups.is_empty() { return Ok(()); } @@ -646,16 +664,22 @@ fn register_bin_plugins( } /// Resolve a [`config_yaml::BinConfig`] to a spawnable argv (`argv[0]` is the -/// program). `url` sources are downloaded + cached first; `path`/`exec` are -/// used as-is. +/// program). A relative `path` is resolved against the workspace `root` (so it +/// doesn't depend on the process cwd); `exec` is used as-is (its program is +/// PATH-resolved at spawn); `url` is downloaded + cached first. #[cfg(unix)] fn resolve_bin_argv( bin: &config_yaml::BinConfig, ctx: &str, + root: &std::path::Path, home_dir: &std::path::Path, ) -> anyhow::Result> { Ok(match bin.resolve(ctx)? { - config_yaml::BinSource::Path(p) => vec![p], + config_yaml::BinSource::Path(p) => { + let pb = std::path::PathBuf::from(&p); + let abs = if pb.is_absolute() { pb } else { root.join(pb) }; + vec![abs.to_string_lossy().into_owned()] + } config_yaml::BinSource::Exec(argv) => argv, config_yaml::BinSource::Url(url) => { vec![ From 57406c85de87a4ea605b6e9df60d4fb048aa203a Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 11:25:50 +0200 Subject: [PATCH 24/44] fix: select heph-plugin-go via -p plugin-go (it is not in the root package) `--bin heph-plugin-go` alone fails ("no bin target in default-run packages") because the bin lives in the plugin-go package, not the root heph package. gen-example and the CI build now pass `-p plugin-go` (CI: -p heph --bin heph -p plugin-go --bin heph-plugin-go). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/heph.yml | 4 ++-- devenv.nix | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/heph.yml b/.github/workflows/heph.yml index a9d911d3..6bfafef2 100644 --- a/.github/workflows/heph.yml +++ b/.github/workflows/heph.yml @@ -184,14 +184,14 @@ jobs: # syslibroot and fails to find `-lfuse`, so the (native arm64) macOS # runner uses plain `cargo build` — no cross needed there. if [ "${{ matrix.os }}" = "darwin" ]; then - cargo build --release --locked --target ${{ matrix.target }} --bin heph --bin heph-plugin-go + cargo build --release --locked --target ${{ matrix.target }} -p heph --bin heph -p plugin-go --bin heph-plugin-go # The nix toolchain hard-links libiconv against its /nix/store path, # which is absent on user machines (dyld aborts at launch). Rewrite # those load commands to the OS /usr/lib copies. bash scripts/macos-portable.sh "$OUT/heph" bash scripts/macos-portable.sh "$OUT/heph-plugin-go" else - cargo zigbuild --release --locked --target ${{ matrix.target }} --bin heph --bin heph-plugin-go + cargo zigbuild --release --locked --target ${{ matrix.target }} -p heph --bin heph -p plugin-go --bin heph-plugin-go fi cp "$OUT/heph" $BIN_NAME cp "$OUT/heph-plugin-go" $PLUGIN_GO_NAME diff --git a/devenv.nix b/devenv.nix index 05116119..2a631523 100644 --- a/devenv.nix +++ b/devenv.nix @@ -71,7 +71,7 @@ in # example/.hephconfig2 launches via `bin: { path: .heph3/heph-go-plugin }`. scripts.gen-example.exec = '' gen-go-large - cargo build --release --bin heph-plugin-go + cargo build --release -p plugin-go --bin heph-plugin-go bin="$CARGO_TARGET_DIR/release/heph-plugin-go" if [ "$(uname -s)" = "Darwin" ]; then # Rewrite the nix-store libiconv load command to /usr/lib so the spawned From f0fc35d028da3055b69cfea6146d2cff7f7a34df Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 11:54:49 +0200 Subject: [PATCH 25/44] fix(plugin-remote): per-call callback scope id (fixes "unknown request scope") The host keyed the callback scope by the engine request_id, but that id is shared across a whole request tree. plugingo's import-closure fan-out issues many concurrent RemoteProvider::get calls under one request_id; each get's scope teardown removed the shared key, so a sibling's still-in-flight result()/query() callback failed with "unknown request scope req-0". Concurrent gets also carry different executors (different cycle-detection parent), so a shared key was semantically wrong too. RemoteProvider::get now mints a unique scope id per call (HostInner.fresh_scope_id), registers the executor under it, and sends it as the wire request_id; the guest echoes it on every callback, routing to the correct per-call executor. Regression test: two gets sharing one engine request_id reach the guest as distinct wire scope ids. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-remote/src/host.rs | 28 ++++-- crates/plugin-remote/src/provider.rs | 14 +-- crates/plugin-remote/tests/proto_e2e.rs | 115 ++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 11 deletions(-) diff --git a/crates/plugin-remote/src/host.rs b/crates/plugin-remote/src/host.rs index 089271f7..9432644a 100644 --- a/crates/plugin-remote/src/host.rs +++ b/crates/plugin-remote/src/host.rs @@ -23,23 +23,39 @@ pub(crate) struct Scope { #[derive(Default)] pub(crate) struct HostInner { scopes: Mutex>>, + /// Mints unique callback-scope ids. The engine's request_id is shared across + /// a whole request tree, so concurrent provider `get`s within one request + /// (e.g. plugingo's import-closure fan-out) would collide on it — one get's + /// unregister would tear down a sibling's still-live scope, and they carry + /// different executors (different cycle-detection parent) anyway. So each + /// call gets its own scope id, which is what travels on the wire as the + /// request_id and comes back on every callback. + scope_seq: std::sync::atomic::AtomicU64, pub leases: LeaseTable, } impl HostInner { - pub fn register(&self, request_id: String, executor: Arc) { + /// A fresh, process-unique scope id for one host→guest call. + pub fn fresh_scope_id(&self) -> String { + let n = self + .scope_seq + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + format!("scope-{n}") + } + + pub fn register(&self, scope_id: String, executor: Arc) { self.scopes .lock() .expect("scopes") - .insert(request_id, Arc::new(Scope { executor })); + .insert(scope_id, Arc::new(Scope { executor })); } - pub fn unregister(&self, request_id: &str) { - self.scopes.lock().expect("scopes").remove(request_id); + pub fn unregister(&self, scope_id: &str) { + self.scopes.lock().expect("scopes").remove(scope_id); } - fn scope(&self, request_id: &str) -> Option> { - self.scopes.lock().expect("scopes").get(request_id).cloned() + fn scope(&self, scope_id: &str) -> Option> { + self.scopes.lock().expect("scopes").get(scope_id).cloned() } } diff --git a/crates/plugin-remote/src/provider.rs b/crates/plugin-remote/src/provider.rs index 626373c2..774831bf 100644 --- a/crates/plugin-remote/src/provider.rs +++ b/crates/plugin-remote/src/provider.rs @@ -132,18 +132,22 @@ impl Provider for RemoteProvider { ctoken: &'a (dyn Cancellable + Send + Sync), ) -> BoxFuture<'a, std::result::Result> { Box::pin(async move { - let request_id = req.request_id.clone(); + // Mint a unique scope id for this call: the engine's request_id is + // shared across the whole request tree, so concurrent gets within one + // request (plugingo's import-closure fan-out) would collide on it. + // This scope id is what travels on the wire and returns on callbacks. + let scope_id = self.inner.fresh_scope_id(); // Register the executor so the plugin's result()/query() callbacks - // for this request route back to it. + // for this call route back to it. self.inner - .register(request_id.clone(), Arc::clone(&req.executor)); + .register(scope_id.clone(), Arc::clone(&req.executor)); let body = Body::GetReq(pb::GetRequest { - request_id: req.request_id, + request_id: scope_id.clone(), addr: Some(convert::addr_to_pb(&req.addr)), states: req.states.iter().map(convert::state_to_pb).collect(), }); let res = self.mux.call_cancellable(body, ctoken.cancelled()).await; - self.inner.unregister(&request_id); + self.inner.unregister(&scope_id); match res { Ok(Body::GetResp(gr)) => Ok(GetResponse { target_spec: convert::target_spec_from_pb(gr.target_spec.unwrap_or_default()), diff --git a/crates/plugin-remote/tests/proto_e2e.rs b/crates/plugin-remote/tests/proto_e2e.rs index fac01b7c..7ebed56e 100644 --- a/crates/plugin-remote/tests/proto_e2e.rs +++ b/crates/plugin-remote/tests/proto_e2e.rs @@ -746,3 +746,118 @@ async fn cycle_error_propagates_typed() { ), } } + +// ---- regression: per-call scope ids ---- + +/// Records the wire request_id each `get` arrives with, so a test can assert the +/// host mints a distinct scope id per call (not the shared engine request_id). +struct RecordingProvider { + seen: Arc>>, +} + +impl Provider for RecordingProvider { + fn config(&self, _req: ConfigRequest) -> anyhow::Result { + Ok(ConfigResponse { + name: "rec".to_string(), + }) + } + fn list<'a>( + &'a self, + _req: ListRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> + Send>>> + { + Box::pin(async move { + Ok(Box::new(std::iter::empty()) + as Box< + dyn Iterator> + Send, + >) + }) + } + fn list_packages<'a>( + &'a self, + _req: ListPackagesRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture< + 'a, + anyhow::Result> + Send>>, + > { + Box::pin(async move { + Ok(Box::new(std::iter::empty()) + as Box< + dyn Iterator> + Send, + >) + }) + } + fn get<'a>( + &'a self, + req: GetRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, Result> { + self.seen.lock().expect("seen").push(req.request_id.clone()); + Box::pin(async move { + Ok(GetResponse { + target_spec: TargetSpec { + addr: req.addr, + driver: "exec".to_string(), + ..Default::default() + }, + }) + }) + } + fn probe<'a>( + &'a self, + _req: ProbeRequest, + _ctoken: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> { + Box::pin(async move { Ok(ProbeResponse { states: vec![] }) }) + } +} + +/// Two gets that share one engine request_id (as plugingo's import-closure +/// fan-out does within a single request) must reach the guest as DISTINCT wire +/// scope ids — otherwise one call's scope teardown would break the other's +/// callbacks ("unknown request scope"). +#[tokio::test] +async fn concurrent_gets_get_distinct_scope_ids() { + let (a, b) = UnixStream::pair().expect("socketpair"); + let (ar, aw) = a.into_split(); + let (br, bw) = b.into_split(); + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + let _guest = plugin_sdk::serve( + Arc::new(RecordingProvider { + seen: Arc::clone(&seen), + }), + br, + bw, + ); + let host = RemoteProvider::connect(ar, aw, "rec"); + let ctoken = StdCancellationToken::new(); + let executor: Arc = Arc::new(StubExec); + + for name in ["a", "b"] { + host.get( + GetRequest { + // Same engine request_id for both calls, on purpose. + request_id: "req-0".to_string(), + addr: addr("//pkg", name), + states: vec![], + executor: Arc::clone(&executor), + }, + &ctoken, + ) + .await + .expect("get ok"); + } + + let seen = seen.lock().expect("seen").clone(); + assert_eq!(seen.len(), 2, "{seen:?}"); + assert_ne!( + seen[0], seen[1], + "each get must mint a unique scope id: {seen:?}" + ); + assert!( + seen.iter().all(|s| s != "req-0"), + "wire id must be the scope id, not the engine request_id: {seen:?}" + ); +} From c620d019099fc287d0d8dd6ccba2c1948c16016a Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 11:55:49 +0200 Subject: [PATCH 26/44] chore(devenv): gen-example runs gen first (self-contained) Codegen (gen/) is regenerated per session; gen-example builds heph-plugin-go which depends on the generated proto bindings, so a fresh checkout failed with E0432 (hproto_gen::heph::plugin) without a prior `gen`. Run it up front. Co-Authored-By: Claude Opus 4.8 (1M context) --- devenv.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/devenv.nix b/devenv.nix index 2a631523..099bad5d 100644 --- a/devenv.nix +++ b/devenv.nix @@ -70,6 +70,7 @@ in # build the out-of-process go plugin into example/.heph3/heph-go-plugin, which # example/.hephconfig2 launches via `bin: { path: .heph3/heph-go-plugin }`. scripts.gen-example.exec = '' + gen gen-go-large cargo build --release -p plugin-go --bin heph-plugin-go bin="$CARGO_TARGET_DIR/release/heph-plugin-go" From c1edffcbebe7463c6f5013988e1e0ed4ba67674e Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 12:22:33 +0200 Subject: [PATCH 27/44] fix(plugin-go): read raw_def via def_de in driver run paths (remote-safe) The go_golist/go_embed/go_testmain run paths read raw_def with def::() (downcast), which panics out-of-process: the def round-trips parse(plugin)-> host->run(plugin) as serialized RawDefBytes, so the concrete downcast fails ("TargetDef raw_def type mismatch"). Switch to def_de::(), which downcasts in-process and deserializes the carried blob when remote. Adds serde::Deserialize to the def types + their enums (GoGolistDef/GoEmbedDef/ GoTestmainDef, EmbedVariant, TestmainMode) and to hplugin Input/InputMode (GoGolistDef carries Vec). TargetAddr already (de)serializes as a string, so no hmodel change is needed. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-go/src/plugingo/driver_embed.rs | 6 +++--- crates/plugin-go/src/plugingo/driver_golist.rs | 4 ++-- crates/plugin-go/src/plugingo/driver_testmain.rs | 6 +++--- crates/plugin/src/driver.rs | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/plugin-go/src/plugingo/driver_embed.rs b/crates/plugin-go/src/plugingo/driver_embed.rs index 597549a8..cf7e3680 100644 --- a/crates/plugin-go/src/plugingo/driver_embed.rs +++ b/crates/plugin-go/src/plugingo/driver_embed.rs @@ -33,7 +33,7 @@ struct GoEmbedSpec { out: HashMap>, } -#[derive(Clone, PartialEq, Debug, serde::Serialize, SpecEnum)] +#[derive(Clone, PartialEq, Debug, serde::Serialize, serde::Deserialize, SpecEnum)] enum EmbedVariant { Embed, /// embed_patterns ∪ test_embed_patterns / embed_files ∪ test_embed_files — @@ -57,7 +57,7 @@ impl Hash for EmbedVariant { /// (paths, file resolution semantics) changes. const GO_EMBED_FORMAT_VERSION: u32 = 5; -#[derive(Clone, serde::Serialize)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct GoEmbedDef { variant: EmbedVariant, golist_origin_id: String, @@ -191,7 +191,7 @@ impl ManagedDriver for GoEmbedDriver { req: ManagedRunRequest<'a, 'io>, _ctoken: &(dyn Cancellable + Send + Sync), ) -> anyhow::Result { - let def = req.request.target.def::(); + let def = req.request.target.def_de::(); // Find package.bin from the golist input let managed = req diff --git a/crates/plugin-go/src/plugingo/driver_golist.rs b/crates/plugin-go/src/plugingo/driver_golist.rs index 4c235d76..a764f119 100644 --- a/crates/plugin-go/src/plugingo/driver_golist.rs +++ b/crates/plugin-go/src/plugingo/driver_golist.rs @@ -65,7 +65,7 @@ impl GoGolistDriver { } } -#[derive(Clone, serde::Serialize)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct GoGolistDef { import_path: String, goos: String, @@ -238,7 +238,7 @@ impl ManagedDriver for GoGolistDriver { req: ManagedRunRequest<'a, 'io>, ctoken: &(dyn Cancellable + Send + Sync), ) -> anyhow::Result { - let def = req.request.target.def::(); + let def = req.request.target.def_de::(); // Find go binary from the go_bin tool input list let go_bin = { diff --git a/crates/plugin-go/src/plugingo/driver_testmain.rs b/crates/plugin-go/src/plugingo/driver_testmain.rs index 8e97b1ca..61cb821b 100644 --- a/crates/plugin-go/src/plugingo/driver_testmain.rs +++ b/crates/plugin-go/src/plugingo/driver_testmain.rs @@ -20,7 +20,7 @@ use xxhash_rust::xxh3::Xxh3Default; pub struct GoTestmainDriver; -#[derive(Clone, serde::Serialize, PartialEq, Eq, Default, SpecEnum)] +#[derive(Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default, SpecEnum)] enum TestmainMode { /// Internal tests only (pkg.test_go_files). testmain.go imports `_test "P"`. Internal, @@ -44,7 +44,7 @@ struct GoTestmainSpec { out: HashMap>, } -#[derive(Clone, serde::Serialize)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct GoTestmainDef { golist_origin_id: String, mode: TestmainMode, @@ -194,7 +194,7 @@ impl ManagedDriver for GoTestmainDriver { req: ManagedRunRequest<'a, 'io>, _ctoken: &(dyn Cancellable + Send + Sync), ) -> anyhow::Result { - let def = req.request.target.def::(); + let def = req.request.target.def_de::(); let managed = req .inputs diff --git a/crates/plugin/src/driver.rs b/crates/plugin/src/driver.rs index 7ab72be1..681098a8 100644 --- a/crates/plugin/src/driver.rs +++ b/crates/plugin/src/driver.rs @@ -643,7 +643,7 @@ pub mod targetdef { } } - #[derive(Clone, Hash, Serialize)] + #[derive(Clone, Hash, Serialize, serde::Deserialize)] pub struct Input { pub r#ref: TargetAddr, pub mode: InputMode, @@ -678,7 +678,7 @@ pub mod targetdef { } } - #[derive(Clone, Hash, PartialEq, Serialize)] + #[derive(Clone, Hash, PartialEq, Serialize, serde::Deserialize)] pub enum InputMode { Standard, Link, From fcda9d3a204c89e41f771ddcaa859136552fe372 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 12:44:33 +0200 Subject: [PATCH 28/44] fix(plugin-abi): fail pending mux calls on disconnect (no hang on plugin death) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a plugin process dies mid-request (panic/exit), the host's in-flight call_cancellable awaited a response that never came — hanging forever. mark_closed now clears the pending table (dropping senders -> RecvError -> "connection closed"), call_cancellable also races wait_closed, and it early-returns if the connection is already closed before issuing. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-abi/src/mux.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/plugin-abi/src/mux.rs b/crates/plugin-abi/src/mux.rs index ea8e0556..bb784b4c 100644 --- a/crates/plugin-abi/src/mux.rs +++ b/crates/plugin-abi/src/mux.rs @@ -123,6 +123,11 @@ impl Mux { fn mark_closed(&self) { self.closed.store(true, Ordering::Release); + // Drop every pending sender so in-flight unary calls observe + // RecvError (-> "connection closed") and streams end, instead of + // hanging forever when the peer dies mid-request (e.g. a plugin + // process that panicked/exited). + self.pending.lock().expect("mux pending").clear(); self.closed_notify.notify_waiters(); } @@ -147,6 +152,13 @@ impl Mux { where F: std::future::Future, { + // Closing-the-connection race: if the peer dies after we insert the + // pending entry but the read loop already passed mark_closed, the entry + // would never be answered. Racing `wait_closed` guarantees the call + // returns instead of hanging. + if self.is_closed() { + anyhow::bail!("plugin connection closed"); + } let id = self.alloc_id(); let (tx, rx) = oneshot::channel(); self.pending.lock().expect("mux pending").insert(id, Pending::Unary(tx)); @@ -166,6 +178,10 @@ impl Mux { self.send_body(id, Body::Cancel(pb::Cancel { request_id: id })); anyhow::bail!("cancelled") } + () = self.wait_closed() => { + self.pending.lock().expect("mux pending").remove(&id); + anyhow::bail!("plugin connection closed before response") + } } } From 109ada52520e97c0c4ad3789a965bbfb129c4849 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 12:44:33 +0200 Subject: [PATCH 29/44] fix(plugin-go,plugin-exec): run plugin on a multi-thread runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both plugin bins used #[tokio::main(flavor = "current_thread")]. plugingo fans out `go list` subprocesses and parks workers via block_in_place on every subprocess chunk read; on a current-thread runtime that path degrades to a 1ms poll loop ("unit tests only" per proc) and all work serializes onto one thread — crippling a `//...` walk. Use a tuned multi-thread runtime sized like the engine (worker_threads = cores, max_blocking_threads = 8N+64) for block_in_place headroom. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plugin-exec/src/bin/heph-plugin-exec.rs | 18 ++++++++++++++-- crates/plugin-go/src/bin/heph-plugin-go.rs | 21 +++++++++++++++++-- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/crates/plugin-exec/src/bin/heph-plugin-exec.rs b/crates/plugin-exec/src/bin/heph-plugin-exec.rs index 93f4f886..377b5ee2 100644 --- a/crates/plugin-exec/src/bin/heph-plugin-exec.rs +++ b/crates/plugin-exec/src/bin/heph-plugin-exec.rs @@ -12,8 +12,22 @@ use plugin_exec::pluginexec::Driver; use std::collections::HashMap; use std::sync::Arc; -#[tokio::main(flavor = "current_thread")] -async fn main() -> anyhow::Result<()> { +// Multi-thread runtime: the exec driver parks workers via `block_in_place` +// while reading the target's subprocess stdio, which requires a multi-thread +// runtime (current-thread falls back to a poll loop and serializes). +fn main() -> anyhow::Result<()> { + let n = std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(8); + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(n) + .max_blocking_threads(8 * n + 64) + .enable_all() + .build()?; + rt.block_on(run()) +} + +async fn run() -> anyhow::Result<()> { let opts = hplugin::config::Options::new(); let mut managed: HashMap> = HashMap::new(); managed.insert("exec".to_string(), Arc::new(Driver::from_options_exec(&opts)?)); diff --git a/crates/plugin-go/src/bin/heph-plugin-go.rs b/crates/plugin-go/src/bin/heph-plugin-go.rs index faedbfc6..88c2049b 100644 --- a/crates/plugin-go/src/bin/heph-plugin-go.rs +++ b/crates/plugin-go/src/bin/heph-plugin-go.rs @@ -18,8 +18,25 @@ use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; -#[tokio::main(flavor = "current_thread")] -async fn main() -> anyhow::Result<()> { +// A multi-thread runtime, sized like the engine's (crates/.../bootstrap.rs). +// plugingo fans out `go list` subprocesses and parks workers via +// `block_in_place` on every subprocess chunk read; on a current-thread runtime +// those fall back to a 1ms poll loop and all work serializes onto one thread — +// catastrophic for a wide `//...` walk. The generous blocking pool gives +// block_in_place headroom. +fn main() -> anyhow::Result<()> { + let n = std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(8); + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(n) + .max_blocking_threads(8 * n + 64) + .enable_all() + .build()?; + rt.block_on(run()) +} + +async fn run() -> anyhow::Result<()> { let root = std::env::var_os("HEPH_PLUGIN_GO_ROOT") .or_else(|| std::env::var_os("HEPH_PLUGIN_ROOT")) .map(PathBuf::from) From 62511dbe5727f37bb07d7b937c1102a06485e59b Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 13:10:30 +0200 Subject: [PATCH 30/44] chore(plugin-remote): TEMP transport instrumentation (frames + callback totals) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Logs on engine drop: Mux total frames in/out, and host callback counts (result/note_dep/query). Diagnostic only — to localize the remote resolve-path cost (transport round-trips vs fs walk). Revert after measuring. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-abi/src/mux.rs | 16 ++++++++++++++++ crates/plugin-remote/src/host.rs | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/crates/plugin-abi/src/mux.rs b/crates/plugin-abi/src/mux.rs index bb784b4c..38a2a6a5 100644 --- a/crates/plugin-abi/src/mux.rs +++ b/crates/plugin-abi/src/mux.rs @@ -69,6 +69,8 @@ pub struct Mux { pending: Mutex>, closed: AtomicBool, closed_notify: Notify, + frames_in: AtomicU64, + frames_out: AtomicU64, } impl Mux { @@ -87,6 +89,8 @@ impl Mux { pending: Mutex::new(HashMap::new()), closed: AtomicBool::new(false), closed_notify: Notify::new(), + frames_in: AtomicU64::new(0), + frames_out: AtomicU64::new(0), }); // writer task @@ -133,6 +137,7 @@ impl Mux { /// Send a frame with an explicit id (used by handlers to reply). pub fn send_body(&self, id: u64, body: Body) { + self.frames_out.fetch_add(1, Ordering::Relaxed); drop(self.out.send(pb::Frame { id, body: Some(body), @@ -235,6 +240,7 @@ async fn reader_loop( loop { match read_frame(&mut read).await { Ok(Some(frame)) => { + mux.frames_in.fetch_add(1, Ordering::Relaxed); let id = frame.id; let Some(body) = frame.body else { continue }; if is_response(&body) { @@ -256,3 +262,13 @@ async fn reader_loop( } mux.mark_closed(); } + +impl Drop for Mux { + fn drop(&mut self) { + tracing::info!( + frames_in = self.frames_in.load(Ordering::Relaxed), + frames_out = self.frames_out.load(Ordering::Relaxed), + "mux total frames" + ); + } +} diff --git a/crates/plugin-remote/src/host.rs b/crates/plugin-remote/src/host.rs index 9432644a..58e89499 100644 --- a/crates/plugin-remote/src/host.rs +++ b/crates/plugin-remote/src/host.rs @@ -32,6 +32,22 @@ pub(crate) struct HostInner { /// request_id and comes back on every callback. scope_seq: std::sync::atomic::AtomicU64, pub leases: LeaseTable, + // TEMP instrumentation: count callbacks to size the resolve-path fan-out. + result_calls: std::sync::atomic::AtomicU64, + note_dep_calls: std::sync::atomic::AtomicU64, + query_calls: std::sync::atomic::AtomicU64, +} + +impl Drop for HostInner { + fn drop(&mut self) { + use std::sync::atomic::Ordering::Relaxed; + tracing::info!( + result = self.result_calls.load(Relaxed), + note_dep = self.note_dep_calls.load(Relaxed), + query = self.query_calls.load(Relaxed), + "remote plugin host callback totals" + ); + } } impl HostInner { @@ -116,6 +132,7 @@ impl InboundHandler for HostCallbackHandler { impl HostCallbackHandler { async fn handle_result(&self, id: u64, req: pb::ResultRequest, mux: &Arc) { + self.inner.result_calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let Some(scope) = self.inner.scope(&req.request_id) else { mux.send_body( id, @@ -158,6 +175,7 @@ impl HostCallbackHandler { } async fn handle_note_dep(&self, id: u64, req: pb::NoteDepRequest, mux: &Arc) { + self.inner.note_dep_calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let Some(scope) = self.inner.scope(&req.request_id) else { mux.send_body( id, @@ -183,6 +201,7 @@ impl HostCallbackHandler { } async fn handle_query(&self, id: u64, req: pb::QueryRequest, mux: &Arc) { + self.inner.query_calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let Some(scope) = self.inner.scope(&req.request_id) else { mux.send_body( id, From 0d2824ef5d1cbfef603c36b9205e906348d294b2 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 13:30:45 +0200 Subject: [PATCH 31/44] perf(plugin-go): note_dep on golist cache hit (cut remote resolve cost) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remote resolve path called executor.result() for every transitive import edge — 22k+ on a real graph — each re-running the full engine result pipeline (cache lookup, ResultLock, memoizer, lease + artifact handles) just to register a dep edge plugingo already had cached. That's the dominant out-of-process cost (note_dep was 0). read_golist_package / read_golist_package_addrs now peek the plugin-side cache (new Memoizer::peek, completed-only, non-inserting): on a hit, register the parent->addr edge with the cheap edge-only note_dep (a bare DepDag insert) and return the cached parse, falling back to result() only on a real miss. Cycle detection is preserved (note_dep registers the same edge and returns CycleError). Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/core/src/hmemoizer/mod.rs | 9 +++++++++ crates/plugin-go/src/plugingo/provider.rs | 20 +++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/core/src/hmemoizer/mod.rs b/crates/core/src/hmemoizer/mod.rs index 68f8e63c..5fc1ebfc 100644 --- a/crates/core/src/hmemoizer/mod.rs +++ b/crates/core/src/hmemoizer/mod.rs @@ -468,6 +468,15 @@ where } } + /// Non-inserting peek: returns the memoized value only if it is already + /// *completed* (not in-flight, not absent). Lets a caller take a cheap path + /// on a cache hit (e.g. registering a dep edge with `note_dep` instead of a + /// full `result`) without disturbing the cache or deduping with in-flight work. + pub fn peek(&self, key: &K) -> Option { + let cache = self.cache.lock().expect("memoizer lock poisoned"); + cache.get(key).and_then(|shared| shared.peek().cloned()) + } + pub async fn process(&self, key: K, f: F) -> V where F: FnOnce() -> Fut, diff --git a/crates/plugin-go/src/plugingo/provider.rs b/crates/plugin-go/src/plugingo/provider.rs index ac4463bf..730506e2 100644 --- a/crates/plugin-go/src/plugingo/provider.rs +++ b/crates/plugin-go/src/plugingo/provider.rs @@ -1728,6 +1728,16 @@ impl ProviderInner { executor: Arc, golist_addr: &Addr, ) -> anyhow::Result> { + // Fast path: the package is already parsed plugin-side, so we only need + // to register the `parent → golist_addr` dep edge (the host's cycle + // check). That's a cheap edge-only `note_dep` instead of a full + // `result()`, which would re-run the engine's whole result pipeline plus + // a lease round-trip for every edge — the dominant cost on the remote + // resolve path (every transitive import hits this). + if let Some(cached) = self.pkg_cache.peek(golist_addr) { + executor.note_dep(golist_addr).await?; + return cached.map_err(unwrap_arc_err); + } let result = executor.result(golist_addr).await?; self.pkg_cache .once( @@ -1778,9 +1788,13 @@ impl ProviderInner { executor: Arc, golist_addr: &Addr, ) -> anyhow::Result> { - // executor.result is called outside the once closure for the same - // reason as in `read_golist_package`: waiters must register the dep - // edge, not just the cache owner. + // executor.result is called outside the once closure so waiters register + // the dep edge, not just the cache owner. Cache hit: cheap edge-only + // note_dep (see read_golist_package). + if let Some(cached) = self.pkg_addrs_cache.peek(golist_addr) { + executor.note_dep(golist_addr).await?; + return cached.map_err(unwrap_arc_err); + } let result = executor.result(golist_addr).await?; self.pkg_addrs_cache .once( From ab676c4efac9c991f114aeb8bfa6c4545803c1ba Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 13:43:23 +0200 Subject: [PATCH 32/44] perf(plugin-go): dedup golist result() to once-per-package (edge via note_dep) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the note_dep split. Previously each caller did note_dep on a cache hit but a full result() on a miss — and concurrent importers of the same golist all peek-missed before the first cached it, so the expensive byte-streaming result() fired thousands of times (result=2523 on a real graph). Now every caller registers its edge with the cheap note_dep, and the data result()+parse runs inside the cache `once` — exactly one result() per distinct golist regardless of how many importers race it. Cycle detection is preserved: the per-caller note_dep registers the edge synchronously, so a dep cycle is caught by DepDag at the closing edge rather than deadlocking on the `once`. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-go/src/plugingo/provider.rs | 36 ++++++++++------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/crates/plugin-go/src/plugingo/provider.rs b/crates/plugin-go/src/plugingo/provider.rs index 730506e2..5d38050b 100644 --- a/crates/plugin-go/src/plugingo/provider.rs +++ b/crates/plugin-go/src/plugingo/provider.rs @@ -1728,21 +1728,19 @@ impl ProviderInner { executor: Arc, golist_addr: &Addr, ) -> anyhow::Result> { - // Fast path: the package is already parsed plugin-side, so we only need - // to register the `parent → golist_addr` dep edge (the host's cycle - // check). That's a cheap edge-only `note_dep` instead of a full - // `result()`, which would re-run the engine's whole result pipeline plus - // a lease round-trip for every edge — the dominant cost on the remote - // resolve path (every transitive import hits this). - if let Some(cached) = self.pkg_cache.peek(golist_addr) { - executor.note_dep(golist_addr).await?; - return cached.map_err(unwrap_arc_err); - } - let result = executor.result(golist_addr).await?; + // Every caller registers its `parent → golist_addr` edge (the host's + // cycle check) with the cheap edge-only `note_dep`. The expensive data + // fetch (`executor.result` + parse) is deduped to a single call per + // distinct golist via `pkg_cache` — so N importers of a shared package + // cost N cheap note_deps plus one `result()`, not N full `result()` + // round-trips (each of which re-runs the engine result pipeline + streams + // the artifact). This is the dominant cost on the remote resolve path. + executor.note_dep(golist_addr).await?; self.pkg_cache .once( golist_addr.clone(), - enclose!((result) move || async move { + enclose!((executor, golist_addr) move || async move { + let result = executor.result(&golist_addr).await?; let pkg = hproc::process_supervisor::block_or_inline(move || -> anyhow::Result<_> { for artifact in &result.artifacts { for entry_result in artifact.walk()? { @@ -1788,18 +1786,14 @@ impl ProviderInner { executor: Arc, golist_addr: &Addr, ) -> anyhow::Result> { - // executor.result is called outside the once closure so waiters register - // the dep edge, not just the cache owner. Cache hit: cheap edge-only - // note_dep (see read_golist_package). - if let Some(cached) = self.pkg_addrs_cache.peek(golist_addr) { - executor.note_dep(golist_addr).await?; - return cached.map_err(unwrap_arc_err); - } - let result = executor.result(golist_addr).await?; + // Cheap edge-only note_dep for every caller; data result() deduped to one + // per distinct golist (see read_golist_package). + executor.note_dep(golist_addr).await?; self.pkg_addrs_cache .once( golist_addr.clone(), - enclose!((result) move || async move { + enclose!((executor, golist_addr) move || async move { + let result = executor.result(&golist_addr).await?; let addrs = hproc::process_supervisor::block_or_inline(move || -> anyhow::Result<_> { for artifact in &result.artifacts { for entry_result in artifact.walk()? { From dd0cb75801891e6f56f3b5124bcea648bcd3778a Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 14:18:25 +0200 Subject: [PATCH 33/44] Revert "perf(plugin-go): dedup golist result() to once-per-package (edge via note_dep)" This reverts commit ab676c4efac9c991f114aeb8bfa6c4545803c1ba. --- crates/plugin-go/src/plugingo/provider.rs | 36 +++++++++++++---------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/crates/plugin-go/src/plugingo/provider.rs b/crates/plugin-go/src/plugingo/provider.rs index 5d38050b..730506e2 100644 --- a/crates/plugin-go/src/plugingo/provider.rs +++ b/crates/plugin-go/src/plugingo/provider.rs @@ -1728,19 +1728,21 @@ impl ProviderInner { executor: Arc, golist_addr: &Addr, ) -> anyhow::Result> { - // Every caller registers its `parent → golist_addr` edge (the host's - // cycle check) with the cheap edge-only `note_dep`. The expensive data - // fetch (`executor.result` + parse) is deduped to a single call per - // distinct golist via `pkg_cache` — so N importers of a shared package - // cost N cheap note_deps plus one `result()`, not N full `result()` - // round-trips (each of which re-runs the engine result pipeline + streams - // the artifact). This is the dominant cost on the remote resolve path. - executor.note_dep(golist_addr).await?; + // Fast path: the package is already parsed plugin-side, so we only need + // to register the `parent → golist_addr` dep edge (the host's cycle + // check). That's a cheap edge-only `note_dep` instead of a full + // `result()`, which would re-run the engine's whole result pipeline plus + // a lease round-trip for every edge — the dominant cost on the remote + // resolve path (every transitive import hits this). + if let Some(cached) = self.pkg_cache.peek(golist_addr) { + executor.note_dep(golist_addr).await?; + return cached.map_err(unwrap_arc_err); + } + let result = executor.result(golist_addr).await?; self.pkg_cache .once( golist_addr.clone(), - enclose!((executor, golist_addr) move || async move { - let result = executor.result(&golist_addr).await?; + enclose!((result) move || async move { let pkg = hproc::process_supervisor::block_or_inline(move || -> anyhow::Result<_> { for artifact in &result.artifacts { for entry_result in artifact.walk()? { @@ -1786,14 +1788,18 @@ impl ProviderInner { executor: Arc, golist_addr: &Addr, ) -> anyhow::Result> { - // Cheap edge-only note_dep for every caller; data result() deduped to one - // per distinct golist (see read_golist_package). - executor.note_dep(golist_addr).await?; + // executor.result is called outside the once closure so waiters register + // the dep edge, not just the cache owner. Cache hit: cheap edge-only + // note_dep (see read_golist_package). + if let Some(cached) = self.pkg_addrs_cache.peek(golist_addr) { + executor.note_dep(golist_addr).await?; + return cached.map_err(unwrap_arc_err); + } + let result = executor.result(golist_addr).await?; self.pkg_addrs_cache .once( golist_addr.clone(), - enclose!((executor, golist_addr) move || async move { - let result = executor.result(&golist_addr).await?; + enclose!((result) move || async move { let addrs = hproc::process_supervisor::block_or_inline(move || -> anyhow::Result<_> { for artifact in &result.artifacts { for entry_result in artifact.walk()? { From c8aa1ae12533f9a447774a3e26898e93b23145e5 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 14:29:52 +0200 Subject: [PATCH 34/44] perf(plugin-abi): coalesce mux writes + buffer reads (cut transport syscalls) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remote resolve path is syscall-heavy: one write (len+body) and two reads (length prefix, then body) per frame, ×140k frames. Two transport-only changes: - writer_loop drains every queued frame and writes them in a single buffer → one write syscall per burst instead of one per frame. - the read half is wrapped in a 64 KiB BufReader → the length-prefix + body reads (and pipelined frames) come from one syscall's worth of data. - write_frame now emits len+body in a single write_all. On the plugingo //... resolve: 8.2s -> 6.5s, sys 6.1s -> 4.3s. No protocol or plugin-logic change. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-abi/src/frame.rs | 18 ++++++++++++++---- crates/plugin-abi/src/mux.rs | 27 +++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/crates/plugin-abi/src/frame.rs b/crates/plugin-abi/src/frame.rs index cc0ebd99..a1c6f790 100644 --- a/crates/plugin-abi/src/frame.rs +++ b/crates/plugin-abi/src/frame.rs @@ -13,11 +13,21 @@ use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; /// length prefix allocating unbounded memory. const MAX_FRAME_LEN: u32 = 16 * 1024 * 1024; -/// Encode and write one frame (length prefix + payload), then flush. +/// Append one length-prefixed frame to `buf` (no I/O). Lets the writer coalesce +/// many frames into a single `write` syscall — the dominant transport cost under +/// the high callback fan-out is per-frame syscalls, not bytes. +pub fn encode_frame_into(buf: &mut Vec, f: &pb::Frame) -> anyhow::Result<()> { + let len = u32::try_from(f.encoded_len()).map_err(|_e| anyhow::anyhow!("frame too large"))?; + buf.extend_from_slice(&len.to_le_bytes()); + f.encode(buf)?; + Ok(()) +} + +/// Encode and write one frame (length prefix + payload) in a single write, then +/// flush. (Bulk paths use [`encode_frame_into`] + one write for many frames.) pub async fn write_frame(w: &mut W, f: &pb::Frame) -> anyhow::Result<()> { - let buf = f.encode_to_vec(); - let len = u32::try_from(buf.len()).map_err(|_e| anyhow::anyhow!("frame too large"))?; - w.write_u32_le(len).await?; + let mut buf = Vec::with_capacity(f.encoded_len() + 4); + encode_frame_into(&mut buf, f)?; w.write_all(&buf).await?; w.flush().await?; Ok(()) diff --git a/crates/plugin-abi/src/mux.rs b/crates/plugin-abi/src/mux.rs index 38a2a6a5..efd24249 100644 --- a/crates/plugin-abi/src/mux.rs +++ b/crates/plugin-abi/src/mux.rs @@ -9,14 +9,14 @@ //! On the host this carries plugin calls outbound + callbacks inbound; on the //! guest, the reverse. -use crate::frame::{read_frame, write_frame}; +use crate::frame::{encode_frame_into, read_frame}; use crate::pb; use async_trait::async_trait; use std::collections::HashMap; use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; -use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tokio::sync::{mpsc, oneshot, Notify}; pub use pb::frame::Body; @@ -95,7 +95,9 @@ impl Mux { // writer task tokio::spawn(writer_loop(write, out_rx)); - // reader task + // reader task — buffered so the length-prefix + body reads (and many + // pipelined frames) come from one syscall's worth of data. + let read = tokio::io::BufReader::with_capacity(64 * 1024, read); tokio::spawn(reader_loop(read, Arc::clone(&mux), handler)); mux @@ -224,11 +226,28 @@ impl Mux { } async fn writer_loop(mut write: W, mut rx: mpsc::UnboundedReceiver) { + let mut buf: Vec = Vec::with_capacity(64 * 1024); while let Some(frame) = rx.recv().await { - if let Err(e) = write_frame(&mut write, &frame).await { + // Coalesce every frame already queued into one buffer, so a burst of + // callbacks/responses costs a single write syscall instead of one each. + buf.clear(); + if let Err(e) = encode_frame_into(&mut buf, &frame) { + tracing::warn!(error = %e, "frame encode failed; dropping"); + continue; + } + while let Ok(frame) = rx.try_recv() { + if let Err(e) = encode_frame_into(&mut buf, &frame) { + tracing::warn!(error = %e, "frame encode failed; dropping"); + } + } + if let Err(e) = write.write_all(&buf).await { tracing::warn!(error = %e, "frame write failed; closing writer"); break; } + if let Err(e) = write.flush().await { + tracing::warn!(error = %e, "frame flush failed; closing writer"); + break; + } } } From 2985c5e775a02b615c8a1e17a21a7fa1c835912d Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 15:53:07 +0200 Subject: [PATCH 35/44] feat(plugin): shm transport (iceoryx2) wired + tested, off by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a working shared-memory transport for out-of-process plugins, but keeps proto/UDS as the default launch — shm measured slower here (the per-call cost is async wakeup latency, not syscalls), so it's an opt-in `shm` cargo feature. shm transport (feature-gated): - plugin-remote::spawn_shm: host connects an iceoryx2 byte-pipe, spawns the plugin with HEPH_PLUGIN_SHM, and uses fd 3 (UDS) for a readiness handshake (pub/sub drops pre-subscribe messages) + bidirectional liveness (EOF = peer gone). plugin-sdk::serve_components_shm is the guest side. - shm.rs hardening: adaptive poll (spin while active, back off when idle), update_connections so a publisher discovers a late subscriber, 0-length EOF sentinel, large no-drop subscriber buffer (4 KiB chunks). Cross-process delivery test (re-exec) proves two processes share the services. - Mux::close for liveness watchers; mux/host temp instrumentation removed. Default launch stays proto (bootstrap uses spawn_streams). Measured on plugingo `test //...`: proto ~6.5s, shm ~8.7s, in-process ~2.8s. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-abi/Cargo.toml | 2 +- crates/plugin-abi/src/convert.rs | 23 +++- crates/plugin-abi/src/mux.rs | 41 +++--- crates/plugin-abi/src/shm.rs | 109 ++++++++++++++- crates/plugin-abi/tests/parity.rs | 38 +++--- crates/plugin-exec/Cargo.toml | 4 + .../plugin-exec/src/bin/heph-plugin-exec.rs | 15 ++- crates/plugin-go/Cargo.toml | 4 + crates/plugin-go/src/bin/heph-plugin-go.rs | 10 +- crates/plugin-remote/src/conn.rs | 6 + crates/plugin-remote/src/host.rs | 19 --- crates/plugin-remote/src/lib.rs | 2 + crates/plugin-remote/src/spawn.rs | 87 ++++++++++++ crates/plugin-sdk/Cargo.toml | 4 + crates/plugin-sdk/src/lib.rs | 2 + crates/plugin-sdk/src/serve.rs | 127 +++++++++++++----- src/commands/bootstrap.rs | 6 +- 17 files changed, 391 insertions(+), 108 deletions(-) diff --git a/crates/plugin-abi/Cargo.toml b/crates/plugin-abi/Cargo.toml index 2ec49803..be0577fc 100644 --- a/crates/plugin-abi/Cargo.toml +++ b/crates/plugin-abi/Cargo.toml @@ -19,7 +19,7 @@ serde_json = { version = "1.0", optional = true } # transport feature: generic length-prefixed Frame framing + bidirectional mux # over any AsyncRead/AsyncWrite. Shared by the host adapter and the guest SDK. prost = { version = "0.14", optional = true } -tokio = { version = "1.52", features = ["io-util", "rt", "sync", "macros"], optional = true } +tokio = { version = "1.52", features = ["io-util", "rt", "sync", "macros", "time"], optional = true } async-trait = { version = "0.1.89", optional = true } tracing = { version = "0.1", optional = true } # shm tier (iceoryx2 byte-pipe under the existing Mux) diff --git a/crates/plugin-abi/src/convert.rs b/crates/plugin-abi/src/convert.rs index 0eee763f..74cfb455 100644 --- a/crates/plugin-abi/src/convert.rs +++ b/crates/plugin-abi/src/convert.rs @@ -11,9 +11,9 @@ use hcore::htvalue::Value; use hmodel::htaddr::Addr; use hmodel::htmatcher::Matcher; use hmodel::htpkg::PkgBuf; +use hplugin::driver::TargetAddr; use hplugin::driver::sandbox::{Dep, Env, EnvValue, Mode, Sandbox, Tool}; use hplugin::driver::targetdef::{RawDef, RawDefBytes}; -use hplugin::driver::TargetAddr; use hplugin::provider::{State, TargetSpec}; use std::collections::BTreeMap; use std::sync::Arc; @@ -198,7 +198,11 @@ pub fn sandbox_to_pb(s: &Sandbox) -> pb::Sandbox { pb::Sandbox { tools: s.tools.iter().map(tool_to_pb).collect(), deps: s.deps.iter().map(dep_to_pb).collect(), - env: s.env.iter().map(|(k, v)| (k.clone(), env_to_pb(v))).collect(), + env: s + .env + .iter() + .map(|(k, v)| (k.clone(), env_to_pb(v))) + .collect(), } } @@ -312,7 +316,11 @@ fn input_to_pb(i: &Input) -> pb::Input { r#ref: Some(target_addr_to_pb(&i.r#ref)), mode: input_mode_to_pb(&i.mode) as i32, origin_id: i.origin_id.clone(), - annotations: i.annotations.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + annotations: i + .annotations + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), hashed: i.hashed, runtime: i.runtime, } @@ -434,7 +442,9 @@ pub fn target_def_from_pb(td: pb::TargetDef) -> anyhow::Result { // ---- OutputArtifact (driver run outputs) ---- -use hplugin::driver::outputartifact::{Content as OaContent, ContentFile, ContentRaw, OutputArtifact, Type as OaType}; +use hplugin::driver::outputartifact::{ + Content as OaContent, ContentFile, ContentRaw, OutputArtifact, Type as OaType, +}; fn oa_type_to_pb(t: &OaType) -> pb::ArtifactType { match t { @@ -550,7 +560,10 @@ mod tests { ("f".to_string(), Value::Float(1.5)), ("b".to_string(), Value::Bool(true)), ("n".to_string(), Value::Null()), - ("l".to_string(), Value::List(vec![Value::Int(1), Value::Int(2)])), + ( + "l".to_string(), + Value::List(vec![Value::Int(1), Value::Int(2)]), + ), ])); assert_eq!(value_from_pb(value_to_pb(&v)), v); } diff --git a/crates/plugin-abi/src/mux.rs b/crates/plugin-abi/src/mux.rs index efd24249..45b0dc12 100644 --- a/crates/plugin-abi/src/mux.rs +++ b/crates/plugin-abi/src/mux.rs @@ -13,11 +13,11 @@ use crate::frame::{encode_frame_into, read_frame}; use crate::pb; use async_trait::async_trait; use std::collections::HashMap; +use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; -use tokio::sync::{mpsc, oneshot, Notify}; +use tokio::sync::{Notify, mpsc, oneshot}; pub use pb::frame::Body; @@ -69,8 +69,6 @@ pub struct Mux { pending: Mutex>, closed: AtomicBool, closed_notify: Notify, - frames_in: AtomicU64, - frames_out: AtomicU64, } impl Mux { @@ -89,8 +87,6 @@ impl Mux { pending: Mutex::new(HashMap::new()), closed: AtomicBool::new(false), closed_notify: Notify::new(), - frames_in: AtomicU64::new(0), - frames_out: AtomicU64::new(0), }); // writer task @@ -127,6 +123,12 @@ impl Mux { } } + /// Externally close the connection (e.g. a liveness watcher detected the + /// peer process died). Fails pending calls and ends `wait_closed`. + pub fn close(&self) { + self.mark_closed(); + } + fn mark_closed(&self) { self.closed.store(true, Ordering::Release); // Drop every pending sender so in-flight unary calls observe @@ -139,7 +141,6 @@ impl Mux { /// Send a frame with an explicit id (used by handlers to reply). pub fn send_body(&self, id: u64, body: Body) { - self.frames_out.fetch_add(1, Ordering::Relaxed); drop(self.out.send(pb::Frame { id, body: Some(body), @@ -168,7 +169,10 @@ impl Mux { } let id = self.alloc_id(); let (tx, rx) = oneshot::channel(); - self.pending.lock().expect("mux pending").insert(id, Pending::Unary(tx)); + self.pending + .lock() + .expect("mux pending") + .insert(id, Pending::Unary(tx)); self.send_body(id, body); tokio::pin!(cancel); tokio::select! { @@ -198,7 +202,10 @@ impl Mux { pub fn call_stream(&self, body: Body) -> mpsc::UnboundedReceiver { let id = self.alloc_id(); let (tx, rx) = mpsc::unbounded_channel(); - self.pending.lock().expect("mux pending").insert(id, Pending::Stream(tx)); + self.pending + .lock() + .expect("mux pending") + .insert(id, Pending::Stream(tx)); self.send_body(id, body); rx } @@ -225,7 +232,10 @@ impl Mux { } } -async fn writer_loop(mut write: W, mut rx: mpsc::UnboundedReceiver) { +async fn writer_loop( + mut write: W, + mut rx: mpsc::UnboundedReceiver, +) { let mut buf: Vec = Vec::with_capacity(64 * 1024); while let Some(frame) = rx.recv().await { // Coalesce every frame already queued into one buffer, so a burst of @@ -259,7 +269,6 @@ async fn reader_loop( loop { match read_frame(&mut read).await { Ok(Some(frame)) => { - mux.frames_in.fetch_add(1, Ordering::Relaxed); let id = frame.id; let Some(body) = frame.body else { continue }; if is_response(&body) { @@ -281,13 +290,3 @@ async fn reader_loop( } mux.mark_closed(); } - -impl Drop for Mux { - fn drop(&mut self) { - tracing::info!( - frames_in = self.frames_in.load(Ordering::Relaxed), - frames_out = self.frames_out.load(Ordering::Relaxed), - "mux total frames" - ); - } -} diff --git a/crates/plugin-abi/src/shm.rs b/crates/plugin-abi/src/shm.rs index 3f347751..a239ca6f 100644 --- a/crates/plugin-abi/src/shm.rs +++ b/crates/plugin-abi/src/shm.rs @@ -16,6 +16,7 @@ //! in the loaned sample) and event-driven wakeup (`Listener`/`WaitSet`) are //! follow-up optimizations. +use iceoryx2::port::update_connections::UpdateConnections; use iceoryx2::prelude::*; use std::io; use std::pin::Pin; @@ -25,8 +26,15 @@ use std::time::Duration; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio::sync::mpsc; -/// Max bytes per iceoryx2 sample; larger writes are split across samples. -const MAX_CHUNK: usize = 64 * 1024; +/// Max bytes per iceoryx2 sample; larger writes are split across samples. Kept +/// small so a deep subscriber buffer (below) stays affordable — buffer memory is +/// `SUB_BUFFER * MAX_CHUNK` per service. +const MAX_CHUNK: usize = 4 * 1024; + +/// Subscriber queue depth. pub/sub is lossy (a full queue drops samples), so this +/// must exceed the protocol's peak in-flight frame count or a Mux call stalls +/// forever on a dropped reply. `SUB_BUFFER * MAX_CHUNK` (= 32 MiB) per service. +const SUB_BUFFER: usize = 8192; pub struct ShmReadHalf { rx: mpsc::UnboundedReceiver>, @@ -55,17 +63,26 @@ pub fn connect( std::thread::spawn(move || { let ports = (|| -> anyhow::Result<_> { let node = NodeBuilder::new().create::()?; + // pub/sub is lossy: a full subscriber buffer drops samples. A build's + // concurrent request/callback fan-out bursts hundreds of frames, so + // size the buffer (and loaned samples) generously to never overflow — + // a dropped frame would stall a Mux call forever. let send_svc = node .service_builder(&send_service.as_str().try_into()?) .publish_subscribe::<[u8]>() + .subscriber_max_buffer_size(SUB_BUFFER) + .history_size(0) .open_or_create()?; let recv_svc = node .service_builder(&recv_service.as_str().try_into()?) .publish_subscribe::<[u8]>() + .subscriber_max_buffer_size(SUB_BUFFER) + .history_size(0) .open_or_create()?; let publisher = send_svc .publisher_builder() .initial_max_slice_len(MAX_CHUNK) + .max_loaned_samples(8) .create()?; let subscriber = recv_svc.subscriber_builder().create()?; Ok((node, publisher, subscriber)) @@ -108,10 +125,26 @@ fn run_loop( wrx: &smpsc::Receiver>, rtx: &mpsc::UnboundedSender>, ) { + // Adaptive backoff: while frames are flowing (a build's hot resolve), spin so + // round-trip latency is ~microseconds, not the per-hop poll interval. After a + // stretch of idleness (between builds), back off to sleeping so an idle plugin + // doesn't peg a core. The hot path is what the cost model cares about. + let mut idle_iters: u32 = 0; + let mut connected = false; loop { let mut idle = true; - // Outbound: publish queued chunks. + // Discover subscribers that connected after this publisher was created + // (the host's publisher exists before the plugin subscribes). Only needed + // until the link is live; once we've received anything the peer is + // connected, so stop paying for it on the hot path. + if !connected { + drop(publisher.update_connections()); + } + + // Outbound: publish queued chunks. On disconnect, publish a 0-length EOF + // marker so the peer's reader observes a clean close (shm has no implicit + // EOF), then exit. loop { match wrx.try_recv() { Ok(chunk) => { @@ -121,16 +154,25 @@ fn run_loop( } } Err(smpsc::TryRecvError::Empty) => break, - Err(smpsc::TryRecvError::Disconnected) => return, + Err(smpsc::TryRecvError::Disconnected) => { + drop(publish(publisher, &[])); + return; + } } } - // Inbound: forward received payloads. + // Inbound: forward received payloads. A 0-length payload is the peer's EOF + // marker → drop our forwarding channel (reader sees end) and exit. loop { match subscriber.receive() { Ok(Some(sample)) => { + let payload = sample.payload(); + if payload.is_empty() { + return; + } idle = false; - if rtx.send(sample.payload().to_vec()).is_err() { + connected = true; + if rtx.send(payload.to_vec()).is_err() { return; } } @@ -140,7 +182,16 @@ fn run_loop( } if idle { - std::thread::sleep(Duration::from_micros(50)); + idle_iters = idle_iters.saturating_add(1); + if idle_iters < 256 { + std::hint::spin_loop(); + } else if idle_iters < 4096 { + std::thread::yield_now(); + } else { + std::thread::sleep(Duration::from_micros(200)); + } + } else { + idle_iters = 0; } } } @@ -216,6 +267,50 @@ mod tests { static N: AtomicU32 = AtomicU32::new(0); + /// Cross-process delivery: re-exec this test binary as a child (subscriber) + /// and ping/pong over shm. shm_byte_pipe_roundtrip only proves the same + /// process; this proves two processes actually share the iceoryx2 services. + #[test] + fn shm_cross_process_delivery() { + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + if std::env::var("HEPH_SHM_XTEST").is_ok() { + // Child: subscribe h2g, publish g2h; echo ping->pong. + rt.block_on(async { + let (mut r, mut w) = connect("xt_g2h", "xt_h2g").expect("child connect"); + let mut b = [0u8; 4]; + r.read_exact(&mut b).await.expect("child read"); + w.write_all(b"pong").await.expect("child write"); + tokio::time::sleep(Duration::from_millis(300)).await; + }); + return; + } + // Mimic heph's order: parent (publisher) connects BEFORE the child + // (subscriber) — so update_connections must rediscover the late subscriber. + let (mut r, mut w) = connect("xt_h2g", "xt_g2h").expect("parent connect"); + let exe = std::env::current_exe().unwrap(); + let mut child = std::process::Command::new(&exe) + .env("HEPH_SHM_XTEST", "1") + .args(["--exact", "shm::tests::shm_cross_process_delivery"]) + .spawn() + .expect("spawn child"); + std::thread::sleep(Duration::from_millis(400)); // let child subscribe + rt.block_on(async { + w.write_all(b"ping").await.expect("parent write"); + let mut b = [0u8; 4]; + tokio::time::timeout(Duration::from_secs(5), r.read_exact(&mut b)) + .await + .expect("cross-process shm delivery timed out") + .expect("parent read"); + assert_eq!(&b, b"pong"); + }); + let _ = child.wait(); + } + #[tokio::test] async fn shm_byte_pipe_roundtrip() { let id = format!( diff --git a/crates/plugin-abi/tests/parity.rs b/crates/plugin-abi/tests/parity.rs index 326e4255..77f58595 100644 --- a/crates/plugin-abi/tests/parity.rs +++ b/crates/plugin-abi/tests/parity.rs @@ -55,25 +55,25 @@ fn result_response_parity() { pb::ResultResponse, shm_types::ResultResponse, pb::ResultResponse { - lease_id: "lease-7".to_string(), - artifacts: vec![ - pb::ArtifactHandle { - handle_id: "h1".to_string(), - group: "out".to_string(), - name: "package.bin".to_string(), - hashout: "abc123".to_string(), - byte_size: 4096, - support: false, - }, - pb::ArtifactHandle { - handle_id: "h2".to_string(), - group: "log".to_string(), - name: "stderr".to_string(), - hashout: String::new(), - byte_size: 0, - support: true, - }, - ], + lease_id: "lease-7".to_string(), + artifacts: vec![ + pb::ArtifactHandle { + handle_id: "h1".to_string(), + group: "out".to_string(), + name: "package.bin".to_string(), + hashout: "abc123".to_string(), + byte_size: 4096, + support: false, + }, + pb::ArtifactHandle { + handle_id: "h2".to_string(), + group: "log".to_string(), + name: "stderr".to_string(), + hashout: String::new(), + byte_size: 0, + support: true, + }, + ], } ); } diff --git a/crates/plugin-exec/Cargo.toml b/crates/plugin-exec/Cargo.toml index 7f60520d..26178f45 100644 --- a/crates/plugin-exec/Cargo.toml +++ b/crates/plugin-exec/Cargo.toml @@ -25,6 +25,10 @@ minijinja = "2" crossterm = { version = "0.29", features = ["event-stream"] } tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "io-util", "io-std", "process", "macros", "sync", "signal", "fs", "net"] } +[features] +# shm transport (iceoryx2); opt-in (slower than proto here on macOS). +shm = ["plugin-sdk/shm"] + [[bin]] name = "heph-plugin-exec" path = "src/bin/heph-plugin-exec.rs" diff --git a/crates/plugin-exec/src/bin/heph-plugin-exec.rs b/crates/plugin-exec/src/bin/heph-plugin-exec.rs index 377b5ee2..3d288e57 100644 --- a/crates/plugin-exec/src/bin/heph-plugin-exec.rs +++ b/crates/plugin-exec/src/bin/heph-plugin-exec.rs @@ -30,8 +30,19 @@ fn main() -> anyhow::Result<()> { async fn run() -> anyhow::Result<()> { let opts = hplugin::config::Options::new(); let mut managed: HashMap> = HashMap::new(); - managed.insert("exec".to_string(), Arc::new(Driver::from_options_exec(&opts)?)); - managed.insert("bash".to_string(), Arc::new(Driver::from_options_bash(&opts)?)); + managed.insert( + "exec".to_string(), + Arc::new(Driver::from_options_exec(&opts)?), + ); + managed.insert( + "bash".to_string(), + Arc::new(Driver::from_options_bash(&opts)?), + ); managed.insert("sh".to_string(), Arc::new(Driver::from_options_sh(&opts)?)); + + #[cfg(feature = "shm")] + if let Ok(id) = std::env::var("HEPH_PLUGIN_SHM") { + return plugin_sdk::serve_components_shm(&id, None, managed).await; + } plugin_sdk::serve_components_inherited(None, managed).await } diff --git a/crates/plugin-go/Cargo.toml b/crates/plugin-go/Cargo.toml index dcc691db..dc6e9c56 100644 --- a/crates/plugin-go/Cargo.toml +++ b/crates/plugin-go/Cargo.toml @@ -30,6 +30,10 @@ glob = "0.3" xxhash-rust = { version = "0.8.15", features = ["xxh3", "std"] } tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "io-util", "process", "macros", "sync", "fs"] } +[features] +# shm transport (iceoryx2); opt-in (slower than proto here on macOS). +shm = ["plugin-sdk/shm"] + [[bin]] name = "heph-plugin-go" path = "src/bin/heph-plugin-go.rs" diff --git a/crates/plugin-go/src/bin/heph-plugin-go.rs b/crates/plugin-go/src/bin/heph-plugin-go.rs index 88c2049b..5e2ea1d1 100644 --- a/crates/plugin-go/src/bin/heph-plugin-go.rs +++ b/crates/plugin-go/src/bin/heph-plugin-go.rs @@ -61,5 +61,13 @@ async fn run() -> anyhow::Result<()> { managed.insert("go_embed".to_string(), Arc::new(GoEmbedDriver)); managed.insert("go_testmain".to_string(), Arc::new(GoTestmainDriver)); - plugin_sdk::serve_components_inherited(Some(Arc::new(provider)), managed).await + let provider: Option> = Some(Arc::new(provider)); + + // Data over shm when the host launched us with HEPH_PLUGIN_SHM (fd 3 stays as + // the liveness channel); otherwise the plain fd-3 UDS transport. + #[cfg(feature = "shm")] + if let Ok(id) = std::env::var("HEPH_PLUGIN_SHM") { + return plugin_sdk::serve_components_shm(&id, provider, managed).await; + } + plugin_sdk::serve_components_inherited(provider, managed).await } diff --git a/crates/plugin-remote/src/conn.rs b/crates/plugin-remote/src/conn.rs index 8400fd81..47d70bec 100644 --- a/crates/plugin-remote/src/conn.rs +++ b/crates/plugin-remote/src/conn.rs @@ -35,6 +35,12 @@ impl RemotePlugin { Self { mux, inner } } + /// The shared mux, for liveness watchers that close the connection when the + /// peer process dies. + pub fn mux_handle(&self) -> Arc { + Arc::clone(&self.mux) + } + /// A handle to this plugin's provider (`name` is its registered name). pub fn provider(&self, name: impl Into) -> RemoteProvider { RemoteProvider::from_parts(Arc::clone(&self.mux), Arc::clone(&self.inner), name.into()) diff --git a/crates/plugin-remote/src/host.rs b/crates/plugin-remote/src/host.rs index 58e89499..9432644a 100644 --- a/crates/plugin-remote/src/host.rs +++ b/crates/plugin-remote/src/host.rs @@ -32,22 +32,6 @@ pub(crate) struct HostInner { /// request_id and comes back on every callback. scope_seq: std::sync::atomic::AtomicU64, pub leases: LeaseTable, - // TEMP instrumentation: count callbacks to size the resolve-path fan-out. - result_calls: std::sync::atomic::AtomicU64, - note_dep_calls: std::sync::atomic::AtomicU64, - query_calls: std::sync::atomic::AtomicU64, -} - -impl Drop for HostInner { - fn drop(&mut self) { - use std::sync::atomic::Ordering::Relaxed; - tracing::info!( - result = self.result_calls.load(Relaxed), - note_dep = self.note_dep_calls.load(Relaxed), - query = self.query_calls.load(Relaxed), - "remote plugin host callback totals" - ); - } } impl HostInner { @@ -132,7 +116,6 @@ impl InboundHandler for HostCallbackHandler { impl HostCallbackHandler { async fn handle_result(&self, id: u64, req: pb::ResultRequest, mux: &Arc) { - self.inner.result_calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let Some(scope) = self.inner.scope(&req.request_id) else { mux.send_body( id, @@ -175,7 +158,6 @@ impl HostCallbackHandler { } async fn handle_note_dep(&self, id: u64, req: pb::NoteDepRequest, mux: &Arc) { - self.inner.note_dep_calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let Some(scope) = self.inner.scope(&req.request_id) else { mux.send_body( id, @@ -201,7 +183,6 @@ impl HostCallbackHandler { } async fn handle_query(&self, id: u64, req: pb::QueryRequest, mux: &Arc) { - self.inner.query_calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let Some(scope) = self.inner.scope(&req.request_id) else { mux.send_body( id, diff --git a/crates/plugin-remote/src/lib.rs b/crates/plugin-remote/src/lib.rs index aab55d02..63a39ef5 100644 --- a/crates/plugin-remote/src/lib.rs +++ b/crates/plugin-remote/src/lib.rs @@ -22,6 +22,8 @@ pub use managed::RemoteManagedDriver; pub use provider::RemoteProvider; #[cfg(unix)] pub use spawn::{PLUGIN_FD, spawn_plugin, spawn_streams}; +#[cfg(all(unix, feature = "shm"))] +pub use spawn::{PLUGIN_SHM_ENV, spawn_shm}; #[cfg(feature = "shm")] pub mod shm; diff --git a/crates/plugin-remote/src/spawn.rs b/crates/plugin-remote/src/spawn.rs index 99323bd7..7f58574b 100644 --- a/crates/plugin-remote/src/spawn.rs +++ b/crates/plugin-remote/src/spawn.rs @@ -64,6 +64,93 @@ pub fn spawn_streams( Ok((tokio_parent.into_split(), child)) } +/// Env var carrying the iceoryx2 service-id base for the shm data transport. +/// When set, the plugin moves its protocol onto shared memory and keeps fd 3 +/// (the UDS) only as a liveness channel (its EOF means the host went away). +#[cfg(feature = "shm")] +pub const PLUGIN_SHM_ENV: &str = "HEPH_PLUGIN_SHM"; + +/// Spawn `program` and connect over the **shm** transport: the protocol runs on +/// an iceoryx2 byte-pipe (no per-frame syscalls), while the inherited fd 3 (UDS) +/// is kept open purely as a bidirectional liveness signal — either side's EOF +/// (process exit/crash) closes the connection. Returns a [`RemotePlugin`]. +#[cfg(feature = "shm")] +pub fn spawn_shm( + program: &Path, + args: &[String], + env: &[(String, String)], + shm_id: &str, +) -> anyhow::Result { + use std::io::Read; + use tokio::io::AsyncReadExt; + + let h2g = format!("{shm_id}_h2g"); + let g2h = format!("{shm_id}_g2h"); + // Host subscribes g2h before the guest ever publishes it (guest only + // publishes in response to our requests), so this direction can't race. + let (hr, hw) = plugin_abi::shm::connect(&h2g, &g2h) + .map_err(|e| anyhow::anyhow!("host shm connect: {e}"))?; + + // fd 3 UDS: a one-byte readiness handshake (the guest signals once its shm + // subscriber is up — iceoryx2 pub/sub drops messages sent before a subscriber + // connects), then a bidirectional liveness channel (EOF = peer gone). + let (parent, child_end) = std::os::unix::net::UnixStream::pair()?; + let child_fd = child_end.as_raw_fd(); + + let mut env = env.to_vec(); + env.push((PLUGIN_SHM_ENV.to_string(), shm_id.to_string())); + let mut cmd = Command::new(program); + cmd.args(args); + for (k, v) in &env { + cmd.env(k, v); + } + let pre = move || -> std::io::Result<()> { + // SAFETY: dup2 is async-signal-safe; child_fd is a valid inherited fd. + let rc = unsafe { libc::dup2(child_fd, PLUGIN_FD) }; + if rc < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }; + // SAFETY: `pre` only calls the async-signal-safe dup2. + unsafe { + cmd.pre_exec(pre); + } + let child = cmd + .spawn() + .map_err(|e| anyhow::anyhow!("spawn plugin {}: {e}", program.display()))?; + drop(child_end); + + // Block (briefly) until the guest's shm subscriber is live. Bounded so a + // plugin that fails to start surfaces as an error (caller falls back to proto) + // instead of hanging. + let mut parent = parent; + parent.set_read_timeout(Some(std::time::Duration::from_secs(10)))?; + let mut ready = [0u8; 1]; + parent + .read_exact(&mut ready) + .map_err(|e| anyhow::anyhow!("plugin shm readiness handshake: {e}"))?; + + parent.set_nonblocking(true)?; + let tokio_parent = tokio::net::UnixStream::from_std(parent)?; + let (mut uds_r, uds_w) = tokio_parent.into_split(); + + let plugin = crate::RemotePlugin::connect(hr, hw); + + // Liveness watcher: hold the UDS write half + child alive; when the guest's + // end closes (exit/crash) our read half hits EOF → close the connection so + // pending calls fail instead of hanging on a dead peer. + let mux = plugin.mux_handle(); + tokio::spawn(async move { + let mut byte = [0u8; 1]; + let _ = uds_r.read(&mut byte).await; // resolves on EOF (no more data sent) + mux.close(); + drop(uds_w); + drop(child); + }); + Ok(plugin) +} + /// Spawn `program` as a single-provider plugin and connect over proto. Returns /// the host adapter plus the child handle (kill/wait it to control its lifetime). pub fn spawn_plugin( diff --git a/crates/plugin-sdk/Cargo.toml b/crates/plugin-sdk/Cargo.toml index 2d934799..ca0610fa 100644 --- a/crates/plugin-sdk/Cargo.toml +++ b/crates/plugin-sdk/Cargo.toml @@ -18,3 +18,7 @@ futures = "0.3.32" prost = "0.14" tokio = { version = "1.52", features = ["rt", "sync", "io-util", "macros", "net"] } tracing = "0.1" + +[features] +# shm transport (iceoryx2) for the guest serve path; pulled in by plugin bins. +shm = ["plugin-abi/shm"] diff --git a/crates/plugin-sdk/src/lib.rs b/crates/plugin-sdk/src/lib.rs index 95b7a512..e3064319 100644 --- a/crates/plugin-sdk/src/lib.rs +++ b/crates/plugin-sdk/src/lib.rs @@ -19,6 +19,8 @@ pub mod serve; pub use ctx::Ctx; pub use host::HostClient; +#[cfg(all(unix, feature = "shm"))] +pub use serve::serve_components_shm; pub use serve::{serve, serve_components, serve_driver, serve_managed_driver, serve_plugin}; #[cfg(unix)] pub use serve::{serve_components_inherited, serve_inherited}; diff --git a/crates/plugin-sdk/src/serve.rs b/crates/plugin-sdk/src/serve.rs index 810177fa..2cd1e0f7 100644 --- a/crates/plugin-sdk/src/serve.rs +++ b/crates/plugin-sdk/src/serve.rs @@ -7,20 +7,19 @@ use async_trait::async_trait; use futures::future::BoxFuture; use hcore::hartifactcontent::{Content, WalkEntry}; use hcore::hasync::StdCancellationToken; -use hplugin::eresult::{ArtifactMeta, EResult}; use hdriver_support::driver_managed::{ManagedDriver, ManagedRunInput, ManagedRunRequest}; +use hmodel::htaddr::Addr; +use hmodel::htmatcher::Matcher; +use hmodel::htpkg::PkgBuf; use hplugin::driver::{ - inputartifact, ApplyTransitiveRequest, ConfigRequest as DriverConfigRequest, Driver, - ParseRequest, RunInput, RunRequest, + ApplyTransitiveRequest, ConfigRequest as DriverConfigRequest, Driver, ParseRequest, RunInput, + RunRequest, inputartifact, }; -use std::path::PathBuf; +use hplugin::eresult::{ArtifactMeta, EResult}; use hplugin::provider::{ ConfigRequest, GetError, GetRequest, ListPackagesRequest, ListRequest, ProbeRequest, Provider, ProviderExecutor, }; -use hmodel::htaddr::Addr; -use hmodel::htmatcher::Matcher; -use hmodel::htpkg::PkgBuf; use plugin_abi::convert; use plugin_abi::error::{WireError, WireErrorKind}; use plugin_abi::mux::{Body, InboundHandler, Mux}; @@ -28,6 +27,7 @@ use plugin_abi::pb; use prost::Message; use std::collections::HashMap; use std::io::Read; +use std::path::PathBuf; use std::sync::{Arc, Mutex}; /// Serve `provider` over an established duplex byte stream. Returns the mux @@ -120,7 +120,49 @@ pub async fn serve_components_inherited( managed: HashMap>, ) -> anyhow::Result<()> { let (r, w) = inherited_fd3()?; - serve_components(provider, managed, r, w).wait_closed().await; + serve_components(provider, managed, r, w) + .wait_closed() + .await; + Ok(()) +} + +/// Guest entry point over the **shm** transport: serve the protocol on an +/// iceoryx2 byte-pipe (no per-frame syscalls) while watching the inherited fd 3 +/// (UDS) for EOF as the host-liveness signal. Used when the host launched us via +/// `spawn_shm` (env `HEPH_PLUGIN_SHM`). +#[cfg(all(unix, feature = "shm"))] +pub async fn serve_components_shm( + shm_id: &str, + provider: Option>, + managed: HashMap>, +) -> anyhow::Result<()> { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let h2g = format!("{shm_id}_h2g"); + let g2h = format!("{shm_id}_g2h"); + // Guest: publish on g2h, subscribe on h2g (mirror of the host). After this + // returns our subscriber is live. + let (gr, gw) = plugin_abi::shm::connect(&g2h, &h2g) + .map_err(|e| anyhow::anyhow!("guest shm connect: {e}"))?; + let mux = serve_components(provider, managed, gr, gw); + + let (mut uds_r, mut uds_w) = inherited_fd3()?; + // Readiness handshake: tell the host our shm subscriber is up so it doesn't + // publish before we can receive (iceoryx2 drops pre-subscribe messages). + uds_w + .write_all(&[1u8]) + .await + .map_err(|e| anyhow::anyhow!("shm readiness signal: {e}"))?; + + // Liveness: hold the UDS write half open; when the host closes its end + // (exit/crash) our read half hits EOF → close the connection → we exit. + let watch_mux = Arc::clone(&mux); + tokio::spawn(async move { + let mut byte = [0u8; 1]; + let _ = uds_r.read(&mut byte).await; + watch_mux.close(); + }); + let _hold = uds_w; // keep the write half open for host-side liveness + mux.wait_closed().await; Ok(()) } @@ -162,7 +204,10 @@ impl GuestHandler { impl GuestHandler { fn new_token(&self, id: u64) -> Arc { let tok = Arc::new(StdCancellationToken::new()); - self.tokens.lock().expect("tokens").insert(id, Arc::clone(&tok)); + self.tokens + .lock() + .expect("tokens") + .insert(id, Arc::clone(&tok)); tok } fn drop_token(&self, id: u64) { @@ -189,7 +234,9 @@ impl InboundHandler for GuestHandler { } else if let Some(m) = self.managed.values().next() { m.config(DriverConfigRequest {}).map(|r| r.name) } else { - Err(anyhow::anyhow!("plugin exports neither provider nor driver")) + Err(anyhow::anyhow!( + "plugin exports neither provider nor driver" + )) }; match name { Ok(name) => mux.send_body(id, Body::ConfigResp(pb::ConfigResponse { name })), @@ -231,7 +278,8 @@ impl InboundHandler for GuestHandler { item: pb::ListPackageResponse { pkg: lpr.pkg.as_str().to_string(), } - .encode_to_vec().into(), + .encode_to_vec() + .into(), }), ), Err(e) => { @@ -340,15 +388,15 @@ impl InboundHandler for GuestHandler { } Body::ApplyTransitiveReq(req) => { let tok = self.new_token(id); - let target_def = match convert::target_def_from_pb(req.target_def.unwrap_or_default()) - { - Ok(td) => td, - Err(e) => { - mux.send_body(id, err_frame(e.to_string())); - self.drop_token(id); - return; - } - }; + let target_def = + match convert::target_def_from_pb(req.target_def.unwrap_or_default()) { + Ok(td) => td, + Err(e) => { + mux.send_body(id, err_frame(e.to_string())); + self.drop_token(id); + return; + } + }; let sel = req.driver.clone(); let areq = ApplyTransitiveRequest { request_id: req.request_id, @@ -390,13 +438,17 @@ impl InboundHandler for GuestHandler { tok.cancel(); } } - other => mux.send_body(id, err_frame(format!("unhandled inbound request: {other:?}"))), + other => mux.send_body( + id, + err_frame(format!("unhandled inbound request: {other:?}")), + ), } } } fn run_input_from_pb(mi: &pb::ManagedRunInput) -> RunInput { - let ty = match pb::InputArtifactType::try_from(mi.r#type).unwrap_or(pb::InputArtifactType::Dep) { + let ty = match pb::InputArtifactType::try_from(mi.r#type).unwrap_or(pb::InputArtifactType::Dep) + { pb::InputArtifactType::Support => inputartifact::Type::Support, _ => inputartifact::Type::Dep, }; @@ -493,7 +545,11 @@ impl GuestHandler { Ok(resp) => mux.send_body( id, Body::ManagedRunResp(pb::ManagedRunResponse { - artifacts: resp.artifacts.iter().map(convert::output_artifact_to_pb).collect(), + artifacts: resp + .artifacts + .iter() + .map(convert::output_artifact_to_pb) + .collect(), }), ), Err(e) => mux.send_body(id, err_frame(e.to_string())), @@ -516,7 +572,8 @@ impl GuestHandler { item: pb::ListResponse { addr: Some(convert::addr_to_pb(&lr.addr)), } - .encode_to_vec().into(), + .encode_to_vec() + .into(), }), ), Err(e) => { @@ -588,11 +645,13 @@ struct MuxExecutor { impl MuxExecutor { /// Pull all bytes of one artifact via `open_artifact` (streamed as chunks). async fn fetch_artifact(&self, lease_id: &str, handle_id: &str) -> Result> { - let mut rx = self.mux.call_stream(Body::OpenArtifactReq(pb::OpenArtifactRequest { - lease_id: lease_id.to_string(), - handle_id: handle_id.to_string(), - offset: 0, - })); + let mut rx = self + .mux + .call_stream(Body::OpenArtifactReq(pb::OpenArtifactRequest { + lease_id: lease_id.to_string(), + handle_id: handle_id.to_string(), + offset: 0, + })); let mut bytes = Vec::new(); while let Some(b) = rx.recv().await { match b { @@ -629,7 +688,8 @@ impl ProviderExecutor for MuxExecutor { // artifact in one fetch; lazy/offset chunking is M3. Most // plugins read a tiny file (e.g. package.bin) so this is // cheap in practice. - let mut artifacts: Vec> = Vec::with_capacity(rr.artifacts.len()); + let mut artifacts: Vec> = + Vec::with_capacity(rr.artifacts.len()); let mut artifacts_meta = Vec::with_capacity(rr.artifacts.len()); for h in &rr.artifacts { let bytes = self.fetch_artifact(&rr.lease_id, &h.handle_id).await?; @@ -689,7 +749,12 @@ impl ProviderExecutor for MuxExecutor { parent: None, addr: Some(convert::addr_to_pb(addr)), }); - match self.mux.call(body).await.map_err(|e| map_wire_err(e, addr))? { + match self + .mux + .call(body) + .await + .map_err(|e| map_wire_err(e, addr))? + { Body::NoteDepResp(r) => { if r.ok { Ok(()) diff --git a/src/commands/bootstrap.rs b/src/commands/bootstrap.rs index 38c790b5..6ab427a3 100644 --- a/src/commands/bootstrap.rs +++ b/src/commands/bootstrap.rs @@ -616,11 +616,13 @@ fn register_bin_plugins( let (program, args) = argv .split_first() .context("internal: empty plugin argv after resolution")?; + // Proto transport over a UDS socketpair (fd 3). The shm transport exists + // (feature `shm`, plugin-remote::spawn_shm) but is slower here — the + // per-call cost is async wakeup latency, not syscalls — so the default + // launch uses proto. let ((r, w), child) = hplugin_remote::spawn_streams(std::path::Path::new(program), args, &env) .with_context(|| format!("spawn plugin `{program}`"))?; - // Dropping the handle does not kill the child (std detaches); the plugin - // self-exits when our socket end closes at engine drop. drop(child); let plugin = hplugin_remote::RemotePlugin::connect(r, w); for (name, kind) in members { From fb88a29ef91e2bdd18e38d2669005d736754ca05 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Tue, 16 Jun 2026 23:45:35 +0200 Subject: [PATCH 36/44] feat(plugin-stabby): in-process stable-ABI plugin transport (dylib) at native speed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Out-of-process plugins paid ~2x: a `test //...` over the go plugin ran ~7s vs ~3.1s compiled-in. Profiling (ai-docs/PERFORMANCE.md) pinned the cost on the async Mux transport (tokio mpsc + duplex wakeups, frame allocs), not on serialization or the ~22k note_dep/result callbacks (which fan out concurrently, hiding latency). Add a native stable-ABI transport so a plugin loads in-process as a cdylib and is called via direct stabby vtable dispatch — no IPC, no serialization on the hot path — while staying a separately-built, hot-swappable, ABI-checked artifact. - crates/plugin-stabby: the stable ABI (stabby). StableExecutor (result/note_dep/ query) crosses callbacks natively; StableProvider/StableManagedDriver cross cold methods as prost pb::Frame bytes; get() takes the native executor. host/guest adapters bridge to/from the in-process hplugin traits. The cdylib owns its own tokio runtime (its statically-linked tokio differs from the host's) so a driver run shelling out via proc_exec works. - crates/plugin-go-cdylib: the go plugin as a loadable cdylib (#[stabby::export]). - Host loader: libloading + stabby get_stabbied type-report ABI check. - config: `dylib: { path | url }` and `wasm: { path | url }` (parallel to `bin:`), sharing one ArtifactConfig/ArtifactSource + url download path. wasm wires the existing wasmtime tier (provider + driver); gated on the `wasm` feature. - gen-example builds + installs the cdylib; example .hephconfig2 uses `dylib:`. - mux: try_inline fast path serves synchronous note_dep without a task spawn. Result: go plugin via the dylib runs ~3.4s (vs 3.1s compiled-in floor), fresh and cached builds correct. bin: (out-of-process UDS) and wasm: coexist. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 73 ++++ Cargo.toml | 7 +- crates/engine/src/engine/config_yaml.rs | 46 +++ crates/plugin-abi/src/mux.rs | 28 +- crates/plugin-abi/src/shm.rs | 5 +- crates/plugin-go-cdylib/Cargo.toml | 21 ++ crates/plugin-go-cdylib/src/lib.rs | 70 ++++ crates/plugin-remote/src/conn.rs | 10 + crates/plugin-remote/src/host.rs | 115 ++++-- crates/plugin-sdk/src/lib.rs | 5 +- crates/plugin-sdk/src/serve.rs | 86 ++++- crates/plugin-stabby/Cargo.toml | 33 ++ crates/plugin-stabby/src/abi.rs | 135 +++++++ crates/plugin-stabby/src/guest.rs | 203 +++++++++++ crates/plugin-stabby/src/host.rs | 159 +++++++++ crates/plugin-stabby/src/lib.rs | 33 ++ crates/plugin-stabby/src/load_stable.rs | 434 ++++++++++++++++++++++ crates/plugin-stabby/src/serve_stable.rs | 436 +++++++++++++++++++++++ crates/plugin/src/provider.rs | 27 ++ devenv.nix | 24 +- example/.hephconfig2 | 24 +- src/commands/bootstrap.rs | 157 +++++++- 22 files changed, 2058 insertions(+), 73 deletions(-) create mode 100644 crates/plugin-go-cdylib/Cargo.toml create mode 100644 crates/plugin-go-cdylib/src/lib.rs create mode 100644 crates/plugin-stabby/Cargo.toml create mode 100644 crates/plugin-stabby/src/abi.rs create mode 100644 crates/plugin-stabby/src/guest.rs create mode 100644 crates/plugin-stabby/src/host.rs create mode 100644 crates/plugin-stabby/src/lib.rs create mode 100644 crates/plugin-stabby/src/load_stable.rs create mode 100644 crates/plugin-stabby/src/serve_stable.rs diff --git a/Cargo.lock b/Cargo.lock index e7beb872..936693ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2335,6 +2335,7 @@ dependencies = [ "plugin-nix", "plugin-query", "plugin-remote", + "plugin-stabby", "posthog-rs", "pprof", "proc", @@ -4382,6 +4383,19 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "plugin-go-cdylib" +version = "0.1.0" +dependencies = [ + "anyhow", + "driver-support", + "plugin", + "plugin-go", + "plugin-stabby", + "stabby", + "walk", +] + [[package]] name = "plugin-nix" version = "0.1.0" @@ -4455,6 +4469,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "plugin-stabby" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "core", + "driver-support", + "futures", + "libloading", + "model", + "plugin", + "plugin-abi", + "prost 0.14.4", + "stabby", + "tokio", +] + [[package]] name = "plugingo-e2e" version = "0.1.0" @@ -5738,6 +5770,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2-const-stable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9" + [[package]] name = "sharded-slab" version = "0.1.7" @@ -5903,6 +5941,41 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "stabby" +version = "72.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7b834ec7ced12095fea1e4b07dcb7e8cf2b59b18afa3eac52494d835965a5ec" +dependencies = [ + "libloading", + "rustversion", + "stabby-abi", +] + +[[package]] +name = "stabby-abi" +version = "72.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1a4f477858a5bdf927c9fab7f579899de9b13e39f8b3b3b300c89fbab632f4" +dependencies = [ + "rustc_version", + "rustversion", + "sha2-const-stable", + "stabby-macros", +] + +[[package]] +name = "stabby-macros" +version = "72.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b31c4b2434980b67ad83f300a58088ba14d59454dcd79ba3d87419bbd924d31e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" diff --git a/Cargo.toml b/Cargo.toml index 3e9519ec..fd3cf9b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2024" [workspace] -members = ["gen/proto", "crates/e2e", "crates/testkit", "crates/plugingo-e2e", "crates/htspec-derive", "crates/core", "crates/walk", "crates/proc", "crates/model", "crates/sandboxfuse", "crates/plugin", "crates/plugin-abi", "crates/plugin-sdk", "crates/plugin-remote", "crates/plugin-echo", "crates/builtins", "crates/plugin-buildfile", "crates/driver-support", "crates/plugin-exec", "crates/plugin-nix", "crates/plugin-query", "crates/plugin-go", "crates/telemetry", "crates/tui", "crates/lock", "crates/engine"] +members = ["gen/proto", "crates/e2e", "crates/testkit", "crates/plugingo-e2e", "crates/htspec-derive", "crates/core", "crates/walk", "crates/proc", "crates/model", "crates/sandboxfuse", "crates/plugin", "crates/plugin-abi", "crates/plugin-sdk", "crates/plugin-remote", "crates/plugin-stabby", "crates/plugin-go-cdylib", "crates/plugin-echo", "crates/builtins", "crates/plugin-buildfile", "crates/driver-support", "crates/plugin-exec", "crates/plugin-nix", "crates/plugin-query", "crates/plugin-go", "crates/telemetry", "crates/tui", "crates/lock", "crates/engine"] # wasm guest components are built with cargo-component for wasm32-wasip2, not by # the host `cargo build --workspace`; keep them out of the workspace. exclude = ["crates/wasm-guests"] @@ -114,6 +114,7 @@ hplugin-nix = { package = "plugin-nix", path = "crates/plugin-nix" } hplugin-query = { package = "plugin-query", path = "crates/plugin-query" } hplugin-go = { package = "plugin-go", path = "crates/plugin-go" } hplugin-remote = { package = "plugin-remote", path = "crates/plugin-remote" } +hplugin-stabby = { package = "plugin-stabby", path = "crates/plugin-stabby", features = ["host", "guest"] } htelemetry = { package = "telemetry", path = "crates/telemetry" } htui = { package = "tui", path = "crates/tui" } hlock = { package = "lock", path = "crates/lock" } @@ -180,6 +181,10 @@ reqwest = { version = "0.13", default-features = false, features = ["blocking", # crate's build.rs (the final binary link). default = ["fuse-sandbox"] fuse-sandbox = ["hsandboxfuse/fuse-sandbox"] +# Out-of-process plugin shm tier (iceoryx2 byte-pipe under the Mux). Opt-in. +shm = ["hplugin-remote/shm"] +# In-process wasm-component plugins (`wasm:` config), via wasmtime. Opt-in. +wasm = ["hplugin-remote/wasm"] [dev-dependencies] serde-jsonlines = "0.7.0" diff --git a/crates/engine/src/engine/config_yaml.rs b/crates/engine/src/engine/config_yaml.rs index 961408a6..b9f84b6e 100644 --- a/crates/engine/src/engine/config_yaml.rs +++ b/crates/engine/src/engine/config_yaml.rs @@ -341,6 +341,52 @@ pub struct PluginEntry { /// the name resolves to an in-process built-in factory. #[serde(default)] pub bin: Option, + /// In-process loadable-plugin cdylib spec (stable ABI, native-speed). A + /// separately-built, hot-swappable artifact loaded in-process. Mutually + /// exclusive with `bin`/`wasm`. + #[serde(default)] + pub dylib: Option, + /// In-process wasm-component plugin spec (wasmtime, capability-sandboxed). + /// For pure-compute plugins. Mutually exclusive with `bin`/`dylib`. + #[serde(default)] + pub wasm: Option, +} + +/// How to locate a loadable in-process plugin artifact (a cdylib or a wasm +/// component). Exactly one of `path`/`url` must be set: +/// - `path`: a file on disk, relative to the workspace root. +/// - `url`: a URL with `{os}`/`{arch}` placeholders; downloaded + cached. +#[derive(Debug, Deserialize, Clone, PartialEq, Eq)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ArtifactConfig { + #[serde(default)] + pub path: Option, + #[serde(default)] + pub url: Option, +} + +/// The resolved, mutually-exclusive source of an [`ArtifactConfig`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ArtifactSource { + /// A path relative to the workspace root. + Path(String), + /// A `{os}`/`{arch}` URL to download + cache. + Url(String), +} + +impl ArtifactConfig { + /// Validate that exactly one of `path`/`url` is set and return it. `ctx` names + /// the entry for error messages. + pub fn resolve(&self, ctx: &str) -> anyhow::Result { + match (&self.path, &self.url) { + (Some(p), None) => Ok(ArtifactSource::Path(p.clone())), + (None, Some(u)) => Ok(ArtifactSource::Url(u.clone())), + (Some(_), Some(_)) => { + anyhow::bail!("artifact for `{ctx}` sets both `path` and `url`; set exactly one") + } + (None, None) => anyhow::bail!("artifact for `{ctx}` sets neither `path` nor `url`"), + } + } } /// How to launch an out-of-process plugin. Exactly one of `path`/`exec`/`url` diff --git a/crates/plugin-abi/src/mux.rs b/crates/plugin-abi/src/mux.rs index 45b0dc12..79da142b 100644 --- a/crates/plugin-abi/src/mux.rs +++ b/crates/plugin-abi/src/mux.rs @@ -27,6 +27,15 @@ pub use pb::frame::Body; #[async_trait] pub trait InboundHandler: Send + Sync + 'static { async fn handle(&self, id: u64, body: Body, mux: Arc); + + /// Fast path for cheap, synchronous, non-reentrant requests: if this body can + /// be served without awaiting (e.g. a `note_dep` edge insert), return the + /// reply to send inline — skipping the per-request `tokio::spawn` + task-hop + /// the async `handle` path costs. Return `Err(body)` to fall back to `handle` + /// (the body is boxed — it's a large prost enum). Default: everything async. + fn try_inline(&self, _id: u64, body: Body) -> Result> { + Err(Box::new(body)) + } } enum Pending { @@ -274,11 +283,20 @@ async fn reader_loop( if is_response(&body) { mux.route_response(id, body); } else { - let mux = Arc::clone(&mux); - let handler = Arc::clone(&handler); - tokio::spawn(async move { - handler.handle(id, body, mux).await; - }); + // Inline cheap synchronous requests (no spawn, no task-hop); + // spawn the rest so a slow/reentrant handler can't stall the + // reader from draining further frames. + match handler.try_inline(id, body) { + Ok(reply) => mux.send_body(id, reply), + Err(body) => { + let mux = Arc::clone(&mux); + let handler = Arc::clone(&handler); + let body = *body; + tokio::spawn(async move { + handler.handle(id, body, mux).await; + }); + } + } } } Ok(None) => break, // peer closed diff --git a/crates/plugin-abi/src/shm.rs b/crates/plugin-abi/src/shm.rs index a239ca6f..bfd0f873 100644 --- a/crates/plugin-abi/src/shm.rs +++ b/crates/plugin-abi/src/shm.rs @@ -139,7 +139,8 @@ fn run_loop( // until the link is live; once we've received anything the peer is // connected, so stop paying for it on the hot path. if !connected { - drop(publisher.update_connections()); + // Discover a late-connecting subscriber; ignore the result. + let _discovered = publisher.update_connections(); } // Outbound: publish queued chunks. On disconnect, publish a 0-length EOF @@ -308,7 +309,7 @@ mod tests { .expect("parent read"); assert_eq!(&b, b"pong"); }); - let _ = child.wait(); + let _status = child.wait(); } #[tokio::test] diff --git a/crates/plugin-go-cdylib/Cargo.toml b/crates/plugin-go-cdylib/Cargo.toml new file mode 100644 index 00000000..2c177c38 --- /dev/null +++ b/crates/plugin-go-cdylib/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "plugin-go-cdylib" +version = "0.1.0" +edition = "2024" + +[lints] +workspace = true + +# Loadable plugin artifact: the go provider + drivers behind the stable ABI, +# loaded in-process by the host via `hplugin_stabby::load_stable::load`. +[lib] +crate-type = ["cdylib"] + +[dependencies] +hplugin-go = { package = "plugin-go", path = "../plugin-go" } +hplugin-stabby = { package = "plugin-stabby", path = "../plugin-stabby", features = ["guest"] } +hplugin = { package = "plugin", path = "../plugin" } +hwalk = { package = "walk", path = "../walk" } +hdriver-support = { package = "driver-support", path = "../driver-support" } +stabby = "72.1.8" +anyhow = "1.0.102" diff --git a/crates/plugin-go-cdylib/src/lib.rs b/crates/plugin-go-cdylib/src/lib.rs new file mode 100644 index 00000000..24e579cd --- /dev/null +++ b/crates/plugin-go-cdylib/src/lib.rs @@ -0,0 +1,70 @@ +//! The go plugin as a loadable cdylib behind the stable ABI. +//! +//! Exports a single stabby `create` entry that constructs the go provider + its +//! managed drivers and hands them back as ABI-stable handles. The host loads this +//! with `hplugin_stabby::load_stable::load`, which verifies ABI compatibility via +//! stabby's type reports before use. Calls then run in-process at native speed — +//! no serialization on the hot path, no IPC (see ai-docs/PERFORMANCE.md). +//! +//! Plugin-specific settings are read from the environment (as the spawned `bin:` +//! variant does); only the workspace root crosses in [`CreateConfig`]. + +use hdriver_support::driver_managed::ManagedDriver; +use hplugin_go::plugingo::{GoEmbedDriver, GoGolistDriver, GoTestmainDriver, Provider}; +use hplugin_stabby::abi::{CreateConfig, NamedDriver, PluginComponents}; +use hplugin_stabby::serve_stable::{make_dyn_managed_driver, make_dyn_provider}; +use std::path::PathBuf; +use std::sync::Arc; + +/// Stable ABI create entry. `#[stabby::export]` emits the type-report symbols the +/// host's `get_stabbied` checks for ABI compatibility. +#[stabby::export] +pub extern "C" fn heph_plugin_create(cfg: CreateConfig) -> PluginComponents { + match build(cfg) { + Ok(c) => c, + Err(e) => { + // No safe way to surface an error through the stable bundle (it must + // carry a valid provider handle), and unwinding across the FFI + // boundary is UB — so fail loudly and abort. + eprintln!("heph-plugin-go: plugin construction failed: {e:#}"); + std::process::abort(); + } + } +} + +fn build(cfg: CreateConfig) -> anyhow::Result { + let root = PathBuf::from(cfg.root.to_string()); + let go_bin = + std::env::var("HEPH_PLUGIN_GO_BIN").unwrap_or_else(|_| "//@heph/bin:go".to_string()); + let walk_db = std::env::var_os("HEPH_PLUGIN_GO_WALK_DB") + .map(PathBuf::from) + .unwrap_or_else(|| root.join(".heph-plugin-go-fswalk.db")); + + let walker = Arc::new(hwalk::CachedWalker::open(&walk_db)); + let opts = hplugin::config::Options::new(); + let provider: Arc = + Arc::new(Provider::from_options(root, &[], &[], &opts, walker)?); + + let mut drivers = stabby::vec::Vec::new(); + let golist: Arc = Arc::new(GoGolistDriver::new(go_bin)); + drivers.push(NamedDriver { + name: "go_golist".into(), + driver: make_dyn_managed_driver(golist), + }); + let embed: Arc = Arc::new(GoEmbedDriver); + drivers.push(NamedDriver { + name: "go_embed".into(), + driver: make_dyn_managed_driver(embed), + }); + let testmain: Arc = Arc::new(GoTestmainDriver); + drivers.push(NamedDriver { + name: "go_testmain".into(), + driver: make_dyn_managed_driver(testmain), + }); + + Ok(PluginComponents { + provider_name: "go".into(), + provider: make_dyn_provider(provider), + drivers, + }) +} diff --git a/crates/plugin-remote/src/conn.rs b/crates/plugin-remote/src/conn.rs index 47d70bec..0319156b 100644 --- a/crates/plugin-remote/src/conn.rs +++ b/crates/plugin-remote/src/conn.rs @@ -55,4 +55,14 @@ impl RemotePlugin { pub fn driver(&self, name: impl Into) -> RemoteDriver { RemoteDriver::from_parts(Arc::clone(&self.mux), Arc::clone(&self.inner), name.into()) } + + /// A native [`ScopedExecutor`](hplugin::provider::ScopedExecutor) over this + /// plugin's scope registry — for in-process / dylib transports that route the + /// hot callbacks (result/note_dep/query) as direct calls instead of over the + /// mux. The scope ids it dispatches are the ones `RemoteProvider::get` mints. + pub fn scoped_executor(&self) -> Arc { + Arc::new(crate::host::ScopedHostExecutor { + inner: Arc::clone(&self.inner), + }) + } } diff --git a/crates/plugin-remote/src/host.rs b/crates/plugin-remote/src/host.rs index 9432644a..2288fc20 100644 --- a/crates/plugin-remote/src/host.rs +++ b/crates/plugin-remote/src/host.rs @@ -57,6 +57,87 @@ impl HostInner { fn scope(&self, scope_id: &str) -> Option> { self.scopes.lock().expect("scopes").get(scope_id).cloned() } + + /// Register the `parent -> addr` dep edge for a `note_dep` request and build + /// the response — fully synchronous. + /// + /// `note_dep` is inline-synchronous host-side: it is a dep-DAG edge insert + /// (`RequestState::track_dep`) with no real await, so `block_on` drives the + /// future to completion with a single poll and never touches the scheduler. + /// That lets the hot, high-volume callback be served without a `tokio::spawn` + /// (the Mux inline fast path) or even a tokio worker at all (the spin mailbox). + pub(crate) fn note_dep_resp(&self, req: pb::NoteDepRequest) -> pb::NoteDepResponse { + match self.scope(&req.request_id) { + Some(scope) => { + let addr = convert::addr_from_pb(req.addr.unwrap_or_default()); + match futures::executor::block_on(scope.executor.note_dep(&addr)) { + Ok(()) => pb::NoteDepResponse { + ok: true, + cycle: false, + message: String::new(), + }, + Err(e) => pb::NoteDepResponse { + ok: false, + cycle: is_cycle(&e), + message: e.to_string(), + }, + } + } + None => pb::NoteDepResponse { + ok: false, + cycle: false, + message: format!("unknown request scope {}", req.request_id), + }, + } + } +} + +/// A [`ScopedExecutor`] over the host scope registry: routes a plugin's callbacks +/// (by request scope id) directly into the per-request engine executor. Used by +/// the in-process / dylib transports where the callback is a native call rather +/// than a serialized round-trip — the whole hot-path win. +pub(crate) struct ScopedHostExecutor { + pub inner: Arc, +} + +impl hplugin::provider::ScopedExecutor for ScopedHostExecutor { + fn result<'a>( + &'a self, + request_id: &'a str, + addr: &'a hmodel::htaddr::Addr, + ) -> futures::future::BoxFuture<'a, anyhow::Result>> { + Box::pin(async move { + match self.inner.scope(request_id) { + Some(scope) => scope.executor.result(addr).await, + None => anyhow::bail!("unknown request scope {request_id}"), + } + }) + } + fn note_dep<'a>( + &'a self, + request_id: &'a str, + addr: &'a hmodel::htaddr::Addr, + ) -> futures::future::BoxFuture<'a, anyhow::Result<()>> { + Box::pin(async move { + match self.inner.scope(request_id) { + Some(scope) => scope.executor.note_dep(addr).await, + None => anyhow::bail!("unknown request scope {request_id}"), + } + }) + } + fn query<'a>( + &'a self, + request_id: &'a str, + m: &'a hmodel::htmatcher::Matcher, + extra_skip: &'a [String], + ) -> futures::future::BoxFuture<'a, anyhow::Result>> { + Box::pin(async move { + match self.inner.scope(request_id) { + Some(scope) => scope.executor.query(m, extra_skip).await, + None => anyhow::bail!("unknown request scope {request_id}"), + } + }) + } } pub(crate) struct HostCallbackHandler { @@ -112,6 +193,15 @@ impl InboundHandler for HostCallbackHandler { } } } + + /// `note_dep` is the dominant callback and fully synchronous, so serve it + /// inline in the reader loop instead of spawning a task per call. + fn try_inline(&self, _id: u64, body: Body) -> Result> { + match body { + Body::NoteDepReq(req) => Ok(Body::NoteDepResp(self.inner.note_dep_resp(req))), + other => Err(Box::new(other)), + } + } } impl HostCallbackHandler { @@ -158,28 +248,9 @@ impl HostCallbackHandler { } async fn handle_note_dep(&self, id: u64, req: pb::NoteDepRequest, mux: &Arc) { - let Some(scope) = self.inner.scope(&req.request_id) else { - mux.send_body( - id, - err_frame(format!("unknown request scope {}", req.request_id)), - ); - return; - }; - let addr = convert::addr_from_pb(req.addr.unwrap_or_default()); - // Edge-only registration (cheap); cycle is detected by type, not message. - let resp = match scope.executor.note_dep(&addr).await { - Ok(()) => pb::NoteDepResponse { - ok: true, - cycle: false, - message: String::new(), - }, - Err(e) => pb::NoteDepResponse { - ok: false, - cycle: is_cycle(&e), - message: e.to_string(), - }, - }; - mux.send_body(id, Body::NoteDepResp(resp)); + // note_dep is synchronous; the inline fast path normally serves it before + // a spawn. This async arm stays for completeness / non-inline transports. + mux.send_body(id, Body::NoteDepResp(self.inner.note_dep_resp(req))); } async fn handle_query(&self, id: u64, req: pb::QueryRequest, mux: &Arc) { diff --git a/crates/plugin-sdk/src/lib.rs b/crates/plugin-sdk/src/lib.rs index e3064319..505af921 100644 --- a/crates/plugin-sdk/src/lib.rs +++ b/crates/plugin-sdk/src/lib.rs @@ -21,7 +21,10 @@ pub use ctx::Ctx; pub use host::HostClient; #[cfg(all(unix, feature = "shm"))] pub use serve::serve_components_shm; -pub use serve::{serve, serve_components, serve_driver, serve_managed_driver, serve_plugin}; +pub use serve::{ + serve, serve_components, serve_components_with_scoped, serve_driver, serve_managed_driver, + serve_plugin, +}; #[cfg(unix)] pub use serve::{serve_components_inherited, serve_inherited}; diff --git a/crates/plugin-sdk/src/serve.rs b/crates/plugin-sdk/src/serve.rs index 2cd1e0f7..02018169 100644 --- a/crates/plugin-sdk/src/serve.rs +++ b/crates/plugin-sdk/src/serve.rs @@ -76,6 +76,7 @@ where driver, managed: HashMap::new(), tokens: Mutex::new(HashMap::new()), + scoped: None, }); Mux::start(read, write, handler) } @@ -89,6 +90,38 @@ pub fn serve_components( read: R, write: W, ) -> Arc +where + R: tokio::io::AsyncRead + Unpin + Send + 'static, + W: tokio::io::AsyncWrite + Unpin + Send + 'static, +{ + serve_components_inner(provider, managed, None, read, write) +} + +/// Serve a multi-component plugin with a **native** scoped executor for the hot +/// callbacks (in-process / dylib transport). result/note_dep/query become direct +/// calls into `scoped` instead of serialized mux round-trips; cold methods still +/// cross over `read`/`write`. +pub fn serve_components_with_scoped( + provider: Option>, + managed: HashMap>, + scoped: Arc, + read: R, + write: W, +) -> Arc +where + R: tokio::io::AsyncRead + Unpin + Send + 'static, + W: tokio::io::AsyncWrite + Unpin + Send + 'static, +{ + serve_components_inner(provider, managed, Some(scoped), read, write) +} + +fn serve_components_inner( + provider: Option>, + managed: HashMap>, + scoped: Option>, + read: R, + write: W, +) -> Arc where R: tokio::io::AsyncRead + Unpin + Send + 'static, W: tokio::io::AsyncWrite + Unpin + Send + 'static, @@ -98,6 +131,7 @@ where driver: None, managed, tokens: Mutex::new(HashMap::new()), + scoped, }); Mux::start(read, write, handler) } @@ -120,7 +154,7 @@ pub async fn serve_components_inherited( managed: HashMap>, ) -> anyhow::Result<()> { let (r, w) = inherited_fd3()?; - serve_components(provider, managed, r, w) + serve_components_inner(provider, managed, None, r, w) .wait_closed() .await; Ok(()) @@ -185,6 +219,10 @@ struct GuestHandler { managed: HashMap>, // frame-id -> cancellation token for the in-flight method call tokens: Mutex>>, + // Native callback path (in-process / dylib): when set, the per-request + // executor's result/note_dep/query are direct calls into the host scope + // registry instead of serialized mux round-trips — the hot-path win. + scoped: Option>, } impl GuestHandler { @@ -304,6 +342,7 @@ impl InboundHandler for GuestHandler { let executor: Arc = Arc::new(MuxExecutor { mux: Arc::clone(&mux), request_id: req.request_id.clone(), + scoped: self.scoped.clone(), }); let greq = GetRequest { request_id: req.request_id, @@ -640,6 +679,23 @@ fn map_wire_err(e: anyhow::Error, addr: &Addr) -> anyhow::Error { struct MuxExecutor { mux: Arc, request_id: String, + /// Native callback path (in-process / dylib): direct calls into the host + /// scope registry, no serialization. Takes priority over the mux. + scoped: Option>, +} + +/// Interpret a `NoteDepResponse` (from either transport) as a typed result. +fn note_dep_result(r: pb::NoteDepResponse, addr: &Addr) -> Result<()> { + if r.ok { + Ok(()) + } else if r.cycle { + Err(anyhow::Error::new(hplugin::error::CycleError { + from: addr.clone(), + to: addr.clone(), + })) + } else { + anyhow::bail!("{}", r.message) + } } impl MuxExecutor { @@ -672,6 +728,9 @@ impl MuxExecutor { impl ProviderExecutor for MuxExecutor { fn result<'a>(&'a self, addr: &'a Addr) -> BoxFuture<'a, Result>> { + if let Some(scoped) = &self.scoped { + return scoped.result(&self.request_id, addr); + } Box::pin(async move { let body = Body::ResultReq(pb::ResultRequest { request_id: self.request_id.clone(), @@ -725,6 +784,9 @@ impl ProviderExecutor for MuxExecutor { m: &'a Matcher, extra_skip: &'a [String], ) -> BoxFuture<'a, Result>> { + if let Some(scoped) = &self.scoped { + return scoped.query(&self.request_id, m, extra_skip); + } Box::pin(async move { let body = Body::QueryReq(pb::QueryRequest { request_id: self.request_id.clone(), @@ -741,32 +803,24 @@ impl ProviderExecutor for MuxExecutor { } fn note_dep<'a>(&'a self, addr: &'a Addr) -> BoxFuture<'a, Result<()>> { + if let Some(scoped) = &self.scoped { + return scoped.note_dep(&self.request_id, addr); + } Box::pin(async move { // The host registers parent -> addr from its own request context, so // `parent` here is informational only. - let body = Body::NoteDepReq(pb::NoteDepRequest { + let req = pb::NoteDepRequest { request_id: self.request_id.clone(), parent: None, addr: Some(convert::addr_to_pb(addr)), - }); + }; match self .mux - .call(body) + .call(Body::NoteDepReq(req)) .await .map_err(|e| map_wire_err(e, addr))? { - Body::NoteDepResp(r) => { - if r.ok { - Ok(()) - } else if r.cycle { - Err(anyhow::Error::new(hplugin::error::CycleError { - from: addr.clone(), - to: addr.clone(), - })) - } else { - anyhow::bail!("{}", r.message) - } - } + Body::NoteDepResp(r) => note_dep_result(r, addr), other => anyhow::bail!("unexpected note_dep response: {other:?}"), } }) diff --git a/crates/plugin-stabby/Cargo.toml b/crates/plugin-stabby/Cargo.toml new file mode 100644 index 00000000..cb4061ec --- /dev/null +++ b/crates/plugin-stabby/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "plugin-stabby" +version = "0.1.0" +edition = "2024" + +[lints] +workspace = true + +[dependencies] +stabby = { version = "72.1.8", features = ["libloading"] } +libloading = "0.8" +anyhow = "1.0.102" +futures = "0.3.32" +hplugin = { package = "plugin", path = "../plugin", optional = true } +hmodel = { package = "model", path = "../model", optional = true } +hcore = { package = "core", path = "../core", optional = true } +hdriver-support = { package = "driver-support", path = "../driver-support", optional = true } +plugin-abi = { path = "../plugin-abi", features = ["convert"], optional = true } +prost = { version = "0.14", optional = true } +async-trait = { version = "0.1.89", optional = true } +# Guest only: a cdylib owns its own tokio runtime — its statically-linked tokio is +# distinct from the host's, so async work that touches the reactor (e.g. plugin-go +# shelling out to `go list` in a driver `run`) must execute on the cdylib's runtime. +tokio = { version = "1.52", features = ["rt-multi-thread", "sync"], optional = true } + +[features] +# host: adapters from the engine's ProviderExecutor onto the stable ABI. +host = ["dep:hplugin", "dep:hmodel", "dep:hcore", "dep:hdriver-support", "dep:plugin-abi", "dep:prost", "dep:async-trait"] +# guest: adapters wrapping the stable ABI back into hplugin traits. +guest = ["dep:hplugin", "dep:hmodel", "dep:hcore", "dep:hdriver-support", "dep:plugin-abi", "dep:prost", "dep:async-trait", "dep:tokio"] + +[dev-dependencies] +futures = "0.3.32" diff --git a/crates/plugin-stabby/src/abi.rs b/crates/plugin-stabby/src/abi.rs new file mode 100644 index 00000000..06685b5d --- /dev/null +++ b/crates/plugin-stabby/src/abi.rs @@ -0,0 +1,135 @@ +//! The stable wire-ABI surface: stabby-stable types + traits that cross the +//! cdylib boundary. Pure stabby — no domain deps — so both host and guest depend +//! on the exact same layout. Conversions to/from the engine's native types live +//! in [`crate::host`] / [`crate::guest`]. + +// Engine futures are `Send` but not `Sync` (BoxFuture), so the ABI returns the +// Send-only stabby future (`DynFuture` would additionally require `Sync`). +use stabby::future::DynFutureUnsync as DynFuture; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; + +/// Outcome of a `note_dep` dep-edge registration. +#[stabby::stabby] +pub struct NoteDepOutcome { + pub ok: bool, + /// The edge closed a dependency cycle (typed, not message-matched). + pub cycle: bool, + pub message: SString, +} + +/// One result artifact, materialized as bytes at the seam (plugin-go reads a +/// tiny `package.bin`; lazy streaming is a later optimization). +#[stabby::stabby] +pub struct StableArtifact { + pub hashout: SString, + pub bytes: SVec, +} + +/// Outcome of a `result` resolution. +#[stabby::stabby] +pub struct ResultOutcome { + pub ok: bool, + pub cycle: bool, + pub cancelled: bool, + pub message: SString, + pub artifacts: SVec, +} + +/// Outcome of a `query`. +#[stabby::stabby] +pub struct QueryOutcome { + pub ok: bool, + pub message: SString, + /// Canonical `//pkg:name` addr strings. + pub addrs: SVec, +} + +/// The host callback surface, called by the plugin while serving `get`/`parse`. +/// Mirrors `hplugin::provider::ProviderExecutor`. Implemented host-side over the +/// real engine executor ([`crate::host`]); consumed guest-side wrapped back into +/// a `ProviderExecutor` ([`crate::guest`]). Calls are direct vtable dispatch — +/// no serialization, no message-passing. +/// +/// `addr` is the canonical `//pkg:name` string; `query`'s matcher crosses as +/// prost bytes (query is rare/zero on the hot path). +#[stabby::stabby] +pub trait StableExecutor { + extern "C" fn note_dep<'a>(&'a self, addr: SString) -> DynFuture<'a, NoteDepOutcome>; + extern "C" fn result<'a>(&'a self, addr: SString) -> DynFuture<'a, ResultOutcome>; + extern "C" fn query<'a>( + &'a self, + matcher_pb: SVec, + extra_skip: SVec, + ) -> DynFuture<'a, QueryOutcome>; +} + +/// An owned, ABI-stable handle to a host executor — what the host passes into the +/// plugin's `get`/`parse`. +pub type DynExecutor = stabby::dynptr!(stabby::boxed::Box); + +/// The cold provider surface, called by the host. Direct stabby vtable dispatch — +/// no async mux, no channels, no duplex (that machinery was the entire cold-path +/// cost; see ai-docs/PERFORMANCE.md). Requests/responses cross as prost-encoded +/// `pb::Frame` bytes (cheap, low-volume, lenient via protobuf); the host decodes +/// the response `Body`. `get` additionally takes the native [`DynExecutor`] so the +/// plugin's hot callbacks during resolution are direct calls. +/// +/// Stream methods (`list`, `list_packages`) return all items length-delimited: +/// a sequence of prost length-delimited `pb::Frame`s (StreamItem… then StreamEnd). +#[stabby::stabby] +pub trait StableProvider { + /// The provider's registered name. + extern "C" fn config(&self) -> SString; + extern "C" fn list<'a>(&'a self, req: SVec) -> DynFuture<'a, SVec>; + extern "C" fn list_packages<'a>(&'a self, req: SVec) -> DynFuture<'a, SVec>; + extern "C" fn get<'a>(&'a self, req: SVec, exec: DynExecutor) -> DynFuture<'a, SVec>; + extern "C" fn probe<'a>(&'a self, req: SVec) -> DynFuture<'a, SVec>; +} + +/// The cold managed-driver surface (same transport contract as [`StableProvider`]). +#[stabby::stabby] +pub trait StableManagedDriver { + extern "C" fn config(&self) -> SString; + extern "C" fn parse<'a>(&'a self, req: SVec) -> DynFuture<'a, SVec>; + extern "C" fn apply_transitive<'a>(&'a self, req: SVec) -> DynFuture<'a, SVec>; + /// `shell` selects `run_shell` over `run`. + extern "C" fn run<'a>(&'a self, req: SVec, shell: bool) -> DynFuture<'a, SVec>; +} + +/// Owned ABI-stable handles to a loaded plugin's components. +pub type DynProvider = stabby::dynptr!(stabby::boxed::Box); +pub type DynManagedDriver = + stabby::dynptr!(stabby::boxed::Box); + +/// Config handed to a cdylib's create entry. Generic: the workspace root; the +/// plugin reads any plugin-specific settings from its own env (as the spawned +/// `bin:` variant does). +#[stabby::stabby] +pub struct CreateConfig { + pub root: SString, +} + +/// A named managed driver in a plugin's component bundle. +#[stabby::stabby] +pub struct NamedDriver { + pub name: SString, + pub driver: DynManagedDriver, +} + +/// What a cdylib's create entry returns: a provider + named drivers, all as owned +/// ABI-stable handles that the host wraps with [`crate::load_stable`]. (plugin-go +/// always exports a provider; driver-only bundles can carry an empty name.) +#[stabby::stabby] +pub struct PluginComponents { + pub provider_name: SString, + pub provider: DynProvider, + pub drivers: SVec, +} + +/// The cdylib create-entry symbol name (exported with `#[stabby::export]`, +/// loaded host-side with `get_stabbied`). +pub const CREATE_SYMBOL: &[u8] = b"heph_plugin_create"; + +/// The create entry's function-pointer type. +pub type CreateFn = extern "C" fn(CreateConfig) -> PluginComponents; diff --git a/crates/plugin-stabby/src/guest.rs b/crates/plugin-stabby/src/guest.rs new file mode 100644 index 00000000..3eff9969 --- /dev/null +++ b/crates/plugin-stabby/src/guest.rs @@ -0,0 +1,203 @@ +//! Guest adapter: wrap a host [`DynExecutor`] back into an [`hplugin::provider:: +//! ProviderExecutor`] so plugin code calls back exactly as in-process — the calls +//! are direct stabby vtable dispatch into the host, no serialization. + +use crate::abi::{DynExecutor, StableExecutorDyn}; +use anyhow::Result; +use futures::future::BoxFuture; +use hcore::hartifactcontent::{Content, WalkEntry}; +use hmodel::htaddr::Addr; +use hmodel::htmatcher::Matcher; +use hplugin::eresult::{ArtifactMeta, EResult}; +use hplugin::provider::ProviderExecutor; +use prost::Message; +use std::io::Read; +use std::sync::Arc; + +/// Forwards `ProviderExecutor` calls to the host over the stable ABI. +pub struct GuestExecutor { + exec: DynExecutor, +} + +impl GuestExecutor { + pub fn new(exec: DynExecutor) -> Self { + Self { exec } + } +} + +impl ProviderExecutor for GuestExecutor { + fn note_dep<'a>(&'a self, addr: &'a Addr) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let r = self.exec.note_dep(addr.to_string().into()).await; + if r.ok { + Ok(()) + } else if r.cycle { + Err(anyhow::Error::new(hplugin::error::CycleError { + from: addr.clone(), + to: addr.clone(), + })) + } else { + anyhow::bail!("{}", r.message) + } + }) + } + + fn result<'a>(&'a self, addr: &'a Addr) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let r = self.exec.result(addr.to_string().into()).await; + if r.ok { + let mut artifacts: Vec> = Vec::with_capacity(r.artifacts.len()); + let mut meta = Vec::with_capacity(r.artifacts.len()); + for a in r.artifacts.iter() { + let hashout = a.hashout.to_string(); + artifacts.push(Arc::new(StableContent { + bytes: a.bytes.to_vec(), + hashout: hashout.clone(), + }) as Arc); + meta.push(ArtifactMeta { hashout }); + } + Ok(Arc::new(EResult { + artifacts, + support_artifacts: vec![], + artifacts_meta: meta, + })) + } else if r.cycle { + Err(anyhow::Error::new(hplugin::error::CycleError { + from: addr.clone(), + to: addr.clone(), + })) + } else if r.cancelled { + Err(anyhow::Error::new(hplugin::error::CancelledError)) + } else { + anyhow::bail!("{}", r.message) + } + }) + } + + fn query<'a>( + &'a self, + m: &'a Matcher, + extra_skip: &'a [String], + ) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let matcher_pb = plugin_abi::convert::matcher_to_pb(m).encode_to_vec(); + let skip: stabby::vec::Vec = + extra_skip.iter().map(|s| s.clone().into()).collect(); + let r = self + .exec + .query(stabby::vec::Vec::from(matcher_pb.as_slice()), skip) + .await; + if r.ok { + let mut out = Vec::with_capacity(r.addrs.len()); + for a in r.addrs.iter() { + out.push(hmodel::htaddr::parse_addr(a)?); + } + Ok(out) + } else { + anyhow::bail!("{}", r.message) + } + }) + } +} + +/// A host artifact materialized guest-side from eagerly-read bytes. Artifacts are +/// tar (the only content type the cache produces today), so `walk` uses the tar +/// walker. Mirrors `plugin_sdk::serve::RemoteContent`. +struct StableContent { + bytes: Vec, + hashout: String, +} + +impl Content for StableContent { + fn reader(&self) -> Result> { + Ok(Box::new(std::io::Cursor::new(self.bytes.clone()))) + } + fn walk(&self) -> Result> + '_>> { + Ok(Box::new(hcore::hartifactcontent::tar::TarWalker::new( + std::io::Cursor::new(self.bytes.clone()), + )?)) + } + fn hashout(&self) -> Result { + Ok(self.hashout.clone()) + } + fn byte_size(&self) -> Option { + Some(self.bytes.len() as u64) + } +} + +#[cfg(all(test, feature = "host"))] +mod tests { + use super::*; + use crate::host::HostExecutor; + use hcore::hartifactcontent::WalkEntry; + use hmodel::htaddr::parse_addr; + use std::io::Read; + use std::sync::atomic::{AtomicU32, Ordering}; + + struct MockExec { + note_deps: AtomicU32, + } + impl ProviderExecutor for MockExec { + fn note_dep<'a>(&'a self, _addr: &'a Addr) -> BoxFuture<'a, Result<()>> { + self.note_deps.fetch_add(1, Ordering::Relaxed); + Box::pin(async { Ok(()) }) + } + fn result<'a>(&'a self, _addr: &'a Addr) -> BoxFuture<'a, Result>> { + Box::pin(async { + Ok(Arc::new(EResult { + artifacts: vec![Arc::new(Bytes(b"hello".to_vec())) as Arc], + support_artifacts: vec![], + artifacts_meta: vec![ArtifactMeta { + hashout: "h1".into(), + }], + })) + }) + } + fn query<'a>( + &'a self, + _m: &'a Matcher, + _s: &'a [String], + ) -> BoxFuture<'a, Result>> { + Box::pin(async { Ok(vec![]) }) + } + } + + struct Bytes(Vec); + impl Content for Bytes { + fn reader(&self) -> Result> { + Ok(Box::new(std::io::Cursor::new(self.0.clone()))) + } + fn walk(&self) -> Result> + '_>> { + anyhow::bail!("no walk in test") + } + fn hashout(&self) -> Result { + Ok("h1".into()) + } + } + + // The hot path crosses the stable ABI (host adapter -> stabby dyn -> guest + // adapter) and back, same process — proving the conversions before the cdylib. + #[test] + fn hot_path_roundtrip() { + let mock = Arc::new(MockExec { + note_deps: AtomicU32::new(0), + }); + let dynexec = HostExecutor::wrap(mock.clone() as Arc); + let guest = GuestExecutor::new(dynexec); + let addr = parse_addr("//pkg/a:b").expect("parse addr"); + + futures::executor::block_on(guest.note_dep(&addr)).expect("note_dep"); + assert_eq!(mock.note_deps.load(Ordering::Relaxed), 1); + + let eres = futures::executor::block_on(guest.result(&addr)).expect("result"); + assert_eq!(eres.artifacts.len(), 1); + assert_eq!(eres.artifacts_meta[0].hashout, "h1"); + let mut buf = String::new(); + eres.artifacts[0] + .reader() + .unwrap() + .read_to_string(&mut buf) + .unwrap(); + assert_eq!(buf, "hello"); + } +} diff --git a/crates/plugin-stabby/src/host.rs b/crates/plugin-stabby/src/host.rs new file mode 100644 index 00000000..36541f2a --- /dev/null +++ b/crates/plugin-stabby/src/host.rs @@ -0,0 +1,159 @@ +//! Host adapter: expose the engine's [`ProviderExecutor`] over the stable ABI so +//! a loaded plugin can call back via direct stabby vtable dispatch. + +use crate::abi::{ + DynExecutor, NoteDepOutcome, QueryOutcome, ResultOutcome, StableArtifact, StableExecutor, +}; +use hmodel::htaddr::parse_addr; +use hplugin::provider::ProviderExecutor; +use prost::Message; +use stabby::future::DynFutureUnsync as DynFuture; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; +use std::io::Read; +use std::sync::Arc; + +/// Wraps the per-request engine executor; handed to the plugin as a [`DynExecutor`]. +pub struct HostExecutor { + inner: Arc, +} + +impl HostExecutor { + /// Wrap a per-request engine executor as an ABI-stable [`DynExecutor`]. + pub fn wrap(inner: Arc) -> DynExecutor { + stabby::boxed::Box::new(HostExecutor { inner }).into() + } +} + +fn is_cycle(e: &anyhow::Error) -> bool { + hcore::hmemoizer::downcast_chain_ref::(e).is_some() +} + +impl StableExecutor for HostExecutor { + extern "C" fn note_dep<'a>(&'a self, addr: SString) -> DynFuture<'a, NoteDepOutcome> { + stabby::boxed::Box::new(async move { + let parsed = match parse_addr(&addr) { + Ok(a) => a, + Err(e) => { + return NoteDepOutcome { + ok: false, + cycle: false, + message: format!("addr parse: {e}").into(), + }; + } + }; + match self.inner.note_dep(&parsed).await { + Ok(()) => NoteDepOutcome { + ok: true, + cycle: false, + message: SString::new(), + }, + Err(e) => NoteDepOutcome { + ok: false, + cycle: is_cycle(&e), + message: e.to_string().into(), + }, + } + }) + .into() + } + + extern "C" fn result<'a>(&'a self, addr: SString) -> DynFuture<'a, ResultOutcome> { + stabby::boxed::Box::new(async move { + let parsed = match parse_addr(&addr) { + Ok(a) => a, + Err(e) => { + return ResultOutcome { + ok: false, + cycle: false, + cancelled: false, + message: format!("addr parse: {e}").into(), + artifacts: SVec::new(), + }; + } + }; + match self.inner.result(&parsed).await { + Ok(eres) => { + let mut artifacts = SVec::new(); + for (idx, art) in eres.artifacts.iter().enumerate() { + let hashout = eres + .artifacts_meta + .get(idx) + .map(|m| m.hashout.clone()) + .or_else(|| art.hashout().ok()) + .unwrap_or_default(); + // Eagerly read the bytes (plugin-go reads a tiny file). + let bytes = match art.reader().and_then(|mut r| { + let mut b = Vec::new(); + r.read_to_end(&mut b)?; + Ok(b) + }) { + Ok(b) => b, + Err(e) => { + return ResultOutcome { + ok: false, + cycle: false, + cancelled: false, + message: format!("artifact read: {e}").into(), + artifacts: SVec::new(), + }; + } + }; + artifacts.push(StableArtifact { + hashout: hashout.into(), + bytes: SVec::from(bytes.as_slice()), + }); + } + ResultOutcome { + ok: true, + cycle: false, + cancelled: false, + message: SString::new(), + artifacts, + } + } + Err(e) => ResultOutcome { + ok: false, + cycle: is_cycle(&e), + cancelled: hplugin::error::is_cancelled(&e), + message: e.to_string().into(), + artifacts: SVec::new(), + }, + } + }) + .into() + } + + extern "C" fn query<'a>( + &'a self, + matcher_pb: SVec, + extra_skip: SVec, + ) -> DynFuture<'a, QueryOutcome> { + stabby::boxed::Box::new(async move { + let matcher = match plugin_abi::pb::Matcher::decode(&matcher_pb[..]) { + Ok(m) => plugin_abi::convert::matcher_from_pb(m), + Err(e) => { + return QueryOutcome { + ok: false, + message: format!("matcher decode: {e}").into(), + addrs: SVec::new(), + }; + } + }; + let skip: Vec = extra_skip.iter().map(|s| s.to_string()).collect(); + match self.inner.query(&matcher, &skip).await { + Ok(addrs) => QueryOutcome { + ok: true, + message: SString::new(), + addrs: addrs.iter().map(|a| a.to_string().into()).collect(), + }, + Err(e) => QueryOutcome { + ok: false, + message: e.to_string().into(), + addrs: SVec::new(), + }, + } + }) + .into() + } +} diff --git a/crates/plugin-stabby/src/lib.rs b/crates/plugin-stabby/src/lib.rs new file mode 100644 index 00000000..79c6e77e --- /dev/null +++ b/crates/plugin-stabby/src/lib.rs @@ -0,0 +1,33 @@ +//! Stable-ABI boundary for in-process dynamically-loaded heph plugins. +//! +//! Goal: load a plugin as a cdylib and call its surface with ZERO serialization +//! on the hot path — the high-volume `ProviderExecutor` callbacks (result / +//! note_dep / query, ~22k per `test //...`) cross as direct stabby vtable calls +//! over native-ish types, recovering the in-process floor (~3.1s vs ~7s +//! out-of-process) while keeping the plugin a separately-built, hot-swappable, +//! ABI-stable artifact. +//! +//! Scoping: only the hot callback path is native. The cold, low-volume +//! Provider/Driver methods (config/list/get/parse/run, ~2k calls ≈ 130ms total) +//! cross as prost bytes — cheap, and lenient via protobuf — so the gnarly +//! `TargetSpec`/`TargetDef`/`raw_def` types need no stabby mirror. +//! +//! `Addr` crosses as its canonical `//pkg:name` string (parsed at the seam; +//! ~22k parses ≈ tens of ms). Result artifacts cross as eagerly-read bytes +//! (plugin-go reads a tiny `package.bin`), wrapped back into a local `Content`. + +#![allow( + clippy::expl_impl_clone_on_copy, + reason = "stabby's #[stabby] macro derives Clone on the zero-sized Copy vtable marker types it generates; the lint fires on macro output we don't control" +)] + +pub mod abi; + +#[cfg(feature = "guest")] +pub mod guest; +#[cfg(feature = "host")] +pub mod host; +#[cfg(feature = "host")] +pub mod load_stable; +#[cfg(feature = "guest")] +pub mod serve_stable; diff --git a/crates/plugin-stabby/src/load_stable.rs b/crates/plugin-stabby/src/load_stable.rs new file mode 100644 index 00000000..25892f13 --- /dev/null +++ b/crates/plugin-stabby/src/load_stable.rs @@ -0,0 +1,434 @@ +//! Host side of the native (mux-free) stable transport: wrap a loaded plugin's +//! [`DynProvider`] / [`DynManagedDriver`] as `hplugin::Provider` / +//! `hdriver_support::ManagedDriver`, so the engine drives them through its normal +//! traits. Each cold method is a direct stabby call; `get` passes the engine's +//! executor across natively (`HostExecutor`) so callbacks are direct. + +use crate::abi::{ + CREATE_SYMBOL, CreateConfig, CreateFn, DynExecutor, DynManagedDriver, DynProvider, + StableManagedDriverDyn, StableProviderDyn, +}; +use crate::host::HostExecutor; +use async_trait::async_trait; +use futures::future::BoxFuture; +use hcore::hasync::Cancellable; +use hdriver_support::driver_managed::{ + ManagedDriver, ManagedRunInput, ManagedRunRequest, ManagedRunResponse, +}; +use hmodel::htpkg::PkgBuf; +use hplugin::driver::{ + ApplyTransitiveRequest, ApplyTransitiveResponse, ConfigRequest as DriverConfigRequest, + ConfigResponse as DriverConfigResponse, DriverSchema, ParseRequest, ParseResponse, + inputartifact, +}; +use hplugin::provider::{ + ConfigRequest, ConfigResponse, GetError, GetRequest, GetResponse, ListPackageResponse, + ListPackagesRequest, ListRequest, ListResponse, ProbeRequest, ProbeResponse, Provider, +}; +use plugin_abi::pb::frame::Body; +use plugin_abi::{convert, pb}; +use prost::Message; +use stabby::vec::Vec as SVec; +use std::sync::Arc; + +fn sv(bytes: &[u8]) -> SVec { + SVec::from(bytes) +} + +/// A loaded plugin's host-side handles: an optional provider + named drivers. +pub type LoadedComponents = ( + Option, + Vec<(String, StableRemoteManagedDriver)>, +); + +/// Load a plugin cdylib and construct the host-side handles. The library's ABI is +/// verified against ours via stabby's type reports (`get_stabbied`); a mismatch +/// (different stabby version, or drifted boundary types) is a hard error. The +/// `Library` is intentionally leaked: the returned trait objects' vtables live in +/// the dylib's code, which must stay mapped for the process lifetime. +pub fn load(path: &std::path::Path, root: &str) -> anyhow::Result { + use crate::abi::PluginComponents; + use anyhow::Context; + use stabby::libloading::StabbyLibrary; + + // SAFETY: loading a plugin dylib runs its initializers; the path is operator- + // controlled config. The ABI of what we call is checked below via get_stabbied. + let lib = unsafe { libloading::Library::new(path) } + .with_context(|| format!("dlopen plugin {}", path.display()))?; + + // Scope the symbol borrow so the library can be leaked after the call. + let comps: PluginComponents = { + // SAFETY: get_stabbied verifies the symbol's stabby type report matches + // `CreateFn` before returning it; calling it is then ABI-sound. + let create = unsafe { lib.get_stabbied::(CREATE_SYMBOL) } + .map_err(|e| anyhow::anyhow!("stabby ABI check failed for {}: {e}", path.display()))?; + create(CreateConfig { root: root.into() }) + }; + // Keep the dylib mapped for the process lifetime (the returned trait objects' + // vtables point into its code); leaking the handle is intentional. + let _: &'static mut libloading::Library = Box::leak(Box::new(lib)); + + let PluginComponents { + provider_name, + provider, + drivers, + } = comps; + let pname = provider_name.to_string(); + let host_provider = if pname.is_empty() { + None + } else { + Some(StableRemoteProvider::new(provider, pname)) + }; + + let mut host_drivers = Vec::new(); + for nd in drivers { + let name = nd.name.to_string(); + host_drivers.push(( + name.clone(), + StableRemoteManagedDriver::new(nd.driver, name), + )); + } + Ok((host_provider, host_drivers)) +} + +fn decode_unary(bytes: &[u8]) -> anyhow::Result { + pb::Frame::decode(bytes)? + .body + .ok_or_else(|| anyhow::anyhow!("empty stable response frame")) +} + +fn decode_stream(bytes: &[u8]) -> anyhow::Result> { + use prost::bytes::Buf; + let mut cur = bytes; + let mut out = Vec::new(); + while cur.has_remaining() { + let f = pb::Frame::decode_length_delimited(&mut cur)?; + if let Some(b) = f.body { + out.push(b); + } + } + Ok(out) +} + +/// Host handle to a loaded plugin's provider. `Clone` (cheap — shares the loaded +/// component) so the engine's provider factory can mint handles. +#[derive(Clone)] +pub struct StableRemoteProvider { + inner: Arc, + name: String, +} + +impl StableRemoteProvider { + pub fn new(inner: DynProvider, name: impl Into) -> Self { + Self { + inner: Arc::new(inner), + name: name.into(), + } + } + + pub fn name(&self) -> &str { + &self.name + } +} + +impl Provider for StableRemoteProvider { + fn config(&self, _req: ConfigRequest) -> anyhow::Result { + Ok(ConfigResponse { + name: self.name.clone(), + }) + } + + fn list<'a>( + &'a self, + req: ListRequest, + _ct: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> + Send>>> + { + Box::pin(async move { + let pb_req = pb::ListRequest { + request_id: req.request_id, + package: req.package.as_str().to_string(), + states: req.states.iter().map(convert::state_to_pb).collect(), + } + .encode_to_vec(); + let bytes = self.inner.list(sv(&pb_req)).await; + let mut out: Vec> = Vec::new(); + for b in decode_stream(&bytes)? { + match b { + Body::StreamItem(si) => { + let lr = pb::ListResponse::decode(&si.item[..])?; + out.push(Ok(ListResponse { + addr: convert::addr_from_pb(lr.addr.unwrap_or_default()), + })); + } + Body::StreamEnd(se) => { + if let Some(e) = se.error { + anyhow::bail!("{}", e.message); + } + break; + } + Body::Error(e) => anyhow::bail!("{}", e.message), + _ => {} + } + } + Ok(Box::new(out.into_iter()) + as Box< + dyn Iterator> + Send, + >) + }) + } + + fn list_packages<'a>( + &'a self, + req: ListPackagesRequest, + _ct: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture< + 'a, + anyhow::Result> + Send>>, + > { + Box::pin(async move { + let pb_req = pb::ListPackagesRequest { + prefix: req.prefix.as_str().to_string(), + } + .encode_to_vec(); + let bytes = self.inner.list_packages(sv(&pb_req)).await; + let mut out: Vec> = Vec::new(); + for b in decode_stream(&bytes)? { + match b { + Body::StreamItem(si) => { + let lpr = pb::ListPackageResponse::decode(&si.item[..])?; + out.push(Ok(ListPackageResponse { + pkg: PkgBuf::from(lpr.pkg), + })); + } + Body::StreamEnd(se) => { + if let Some(e) = se.error { + anyhow::bail!("{}", e.message); + } + break; + } + Body::Error(e) => anyhow::bail!("{}", e.message), + _ => {} + } + } + Ok(Box::new(out.into_iter()) + as Box< + dyn Iterator> + Send, + >) + }) + } + + fn get<'a>( + &'a self, + req: GetRequest, + _ct: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, std::result::Result> { + Box::pin(async move { + let pb_req = pb::GetRequest { + request_id: req.request_id.clone(), + addr: Some(convert::addr_to_pb(&req.addr)), + states: req.states.iter().map(convert::state_to_pb).collect(), + } + .encode_to_vec(); + let exec: DynExecutor = HostExecutor::wrap(Arc::clone(&req.executor)); + let bytes = self.inner.get(sv(&pb_req), exec).await; + let body = decode_unary(&bytes).map_err(GetError::Other)?; + match body { + Body::GetResp(gr) => Ok(GetResponse { + target_spec: convert::target_spec_from_pb(gr.target_spec.unwrap_or_default()), + }), + Body::GetErr(ge) => match pb::get_error::Kind::try_from(ge.kind) + .unwrap_or(pb::get_error::Kind::Other) + { + pb::get_error::Kind::NotFound => Err(GetError::NotFound), + pb::get_error::Kind::Cycle => Err(GetError::Other(anyhow::Error::new( + hplugin::error::CycleError { + from: req.addr.clone(), + to: req.addr.clone(), + }, + ))), + pb::get_error::Kind::Cancelled => Err(GetError::Other(anyhow::Error::new( + hplugin::error::CancelledError, + ))), + _ => Err(GetError::Other(anyhow::anyhow!("{}", ge.message))), + }, + other => Err(GetError::Other(anyhow::anyhow!( + "unexpected get response: {other:?}" + ))), + } + }) + } + + fn probe<'a>( + &'a self, + req: ProbeRequest, + _ct: &'a (dyn Cancellable + Send + Sync), + ) -> BoxFuture<'a, anyhow::Result> { + Box::pin(async move { + let pb_req = pb::ProbeRequest { + request_id: req.request_id, + package: req.package.as_str().to_string(), + } + .encode_to_vec(); + let bytes = self.inner.probe(sv(&pb_req)).await; + match decode_unary(&bytes)? { + Body::ProbeResp(pr) => Ok(ProbeResponse { + states: pr.states.into_iter().map(convert::state_from_pb).collect(), + }), + Body::Error(e) => anyhow::bail!("{}", e.message), + other => anyhow::bail!("unexpected probe response: {other:?}"), + } + }) + } +} + +/// Host handle to a loaded plugin's managed driver. `Clone` (shares the loaded +/// component) for the engine's driver factory. +#[derive(Clone)] +pub struct StableRemoteManagedDriver { + inner: Arc, + name: String, +} + +impl StableRemoteManagedDriver { + pub fn new(inner: DynManagedDriver, name: impl Into) -> Self { + Self { + inner: Arc::new(inner), + name: name.into(), + } + } +} + +#[async_trait] +impl ManagedDriver for StableRemoteManagedDriver { + fn config(&self, _req: DriverConfigRequest) -> anyhow::Result { + Ok(DriverConfigResponse { + name: self.name.clone(), + }) + } + + fn schema(&self) -> DriverSchema { + DriverSchema::default() + } + + async fn parse( + &self, + req: ParseRequest, + _ct: &(dyn Cancellable + Send + Sync), + ) -> anyhow::Result { + let pb_req = pb::ParseRequest { + request_id: req.request_id, + target_spec: Some(convert::target_spec_to_pb(req.target_spec.as_ref())), + driver: self.name.clone(), + } + .encode_to_vec(); + let bytes = self.inner.parse(sv(&pb_req)).await; + match decode_unary(&bytes)? { + Body::ParseResp(pr) => Ok(ParseResponse { + target_def: convert::target_def_from_pb(pr.target_def.unwrap_or_default())?, + }), + Body::Error(e) => anyhow::bail!("{}", e.message), + other => anyhow::bail!("unexpected parse response: {other:?}"), + } + } + + async fn apply_transitive( + &self, + req: ApplyTransitiveRequest, + _ct: &(dyn Cancellable + Send + Sync), + ) -> anyhow::Result { + let pb_req = pb::ApplyTransitiveRequest { + request_id: req.request_id, + target_def: Some(convert::target_def_to_pb(&req.target_def)?), + sandbox: Some(convert::sandbox_to_pb(&req.sandbox)), + driver: self.name.clone(), + } + .encode_to_vec(); + let bytes = self.inner.apply_transitive(sv(&pb_req)).await; + match decode_unary(&bytes)? { + Body::ApplyTransitiveResp(r) => Ok(ApplyTransitiveResponse { + target_def: convert::target_def_from_pb(r.target_def.unwrap_or_default())?, + }), + Body::Error(e) => anyhow::bail!("{}", e.message), + other => anyhow::bail!("unexpected apply_transitive response: {other:?}"), + } + } + + fn supports_shell(&self) -> bool { + true + } + + async fn run<'a, 'io>( + &self, + req: ManagedRunRequest<'a, 'io>, + _ct: &(dyn Cancellable + Send + Sync), + ) -> anyhow::Result { + self.dispatch_run(req, false).await + } + + async fn run_shell<'a, 'io>( + &self, + req: ManagedRunRequest<'a, 'io>, + _ct: &(dyn Cancellable + Send + Sync), + ) -> anyhow::Result { + self.dispatch_run(req, true).await + } +} + +impl StableRemoteManagedDriver { + async fn dispatch_run( + &self, + req: ManagedRunRequest<'_, '_>, + shell: bool, + ) -> anyhow::Result { + let pb_req = pb::ManagedRunRequest { + request_id: req.request.request_id.clone(), + target: Some(convert::target_def_to_pb(req.request.target)?), + tree_root_path: req.request.tree_root_path.to_string_lossy().into_owned(), + hashin: req.request.hashin.to_string(), + sandbox_dir: req.sandbox_dir.to_string_lossy().into_owned(), + sandbox_ws_dir: req.sandbox_ws_dir.to_string_lossy().into_owned(), + sandbox_pkg_dir: req.sandbox_pkg_dir.to_string_lossy().into_owned(), + inputs: req.inputs.iter().map(managed_input_to_pb).collect(), + shell, + driver: self.name.clone(), + } + .encode_to_vec(); + let bytes = self.inner.run(sv(&pb_req), shell).await; + match decode_unary(&bytes)? { + Body::ManagedRunResp(r) => Ok(ManagedRunResponse { + artifacts: r + .artifacts + .into_iter() + .map(convert::output_artifact_from_pb) + .collect(), + }), + Body::Error(e) => anyhow::bail!("{}", e.message), + other => anyhow::bail!("unexpected managed run response: {other:?}"), + } + } +} + +fn managed_input_to_pb(mi: &ManagedRunInput) -> pb::ManagedRunInput { + let ty = match mi.input.artifact.r#type { + inputartifact::Type::Dep => pb::InputArtifactType::Dep, + inputartifact::Type::Support => pb::InputArtifactType::Support, + }; + pb::ManagedRunInput { + r#type: ty as i32, + origin_id: mi.input.origin_id.clone(), + source_addr: Some(convert::addr_to_pb(&mi.input.source_addr)), + filters: mi.input.filters.clone(), + annotations: mi + .input + .annotations + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + unpack_root: mi.unpack_root.to_string_lossy().into_owned(), + list_path: mi + .list_path + .as_ref() + .map(|p| p.to_string_lossy().into_owned()), + } +} diff --git a/crates/plugin-stabby/src/serve_stable.rs b/crates/plugin-stabby/src/serve_stable.rs new file mode 100644 index 00000000..c341208e --- /dev/null +++ b/crates/plugin-stabby/src/serve_stable.rs @@ -0,0 +1,436 @@ +//! Guest side of the native (mux-free) stable transport: wrap an author's +//! `Provider` / `ManagedDriver` as [`StableProvider`] / [`StableManagedDriver`]. +//! +//! Cold requests/responses cross as prost `pb::Frame` bytes (the response `Body` +//! the mux serve loop would have sent, returned directly instead). `get` receives +//! the host executor natively ([`DynExecutor`]) so the plugin's hot callbacks are +//! direct calls — no mux, no channels, no task spawn (see ai-docs/PERFORMANCE.md). + +use crate::abi::{DynExecutor, StableManagedDriver, StableProvider}; +use crate::guest::GuestExecutor; +use anyhow::Result; +use hcore::hartifactcontent::{Content, WalkEntry}; +use hcore::hasync::StdCancellationToken; +use hdriver_support::driver_managed::{ManagedDriver, ManagedRunInput, ManagedRunRequest}; +use hmodel::htpkg::PkgBuf; +use hplugin::driver::{ + ApplyTransitiveRequest, ConfigRequest as DriverConfigRequest, ParseRequest, RunInput, + RunRequest, inputartifact, +}; +use hplugin::provider::{ + ConfigRequest, GetError, GetRequest, ListPackagesRequest, ListRequest, ProbeRequest, Provider, + ProviderExecutor, +}; +use plugin_abi::convert; +use plugin_abi::pb; +use plugin_abi::pb::frame::Body; +use prost::Message; +use stabby::future::DynFutureUnsync as DynFuture; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; +use std::io::Read; +use std::path::PathBuf; +use std::sync::Arc; + +/// The cdylib's own tokio runtime. A loaded cdylib's statically-linked tokio is a +/// separate instance from the host's, so async work that touches the reactor (a +/// driver `run` shelling out via `proc_exec`) must run here, not on the host +/// worker that polls our returned future. Sized like the engine's runtime +/// (plugin-go parks workers via `block_in_place` per subprocess chunk). +fn cdylib_runtime() -> &'static tokio::runtime::Runtime { + use std::sync::OnceLock; + static RT: OnceLock = OnceLock::new(); + RT.get_or_init(|| { + let n = std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(8); + tokio::runtime::Builder::new_multi_thread() + .worker_threads(n) + .max_blocking_threads(8 * n + 64) + .enable_all() + .build() + .expect("build cdylib plugin runtime") + }) +} + +fn unary(body: Body) -> SVec { + let f = pb::Frame { + id: 0, + body: Some(body), + }; + SVec::from(f.encode_to_vec().as_slice()) +} + +fn stream(bodies: Vec) -> SVec { + let mut buf = Vec::new(); + for b in bodies { + let f = pb::Frame { + id: 0, + body: Some(b), + }; + // Encoding into a growable Vec is infallible; bind the Result to satisfy + // #[must_use] without an explicit drop (the value is Copy). + let _encoded = f.encode_length_delimited(&mut buf); + } + SVec::from(buf.as_slice()) +} + +fn err_body(message: String) -> Body { + Body::Error(pb::Error { + kind: pb::error::Kind::Other as i32, + message, + }) +} + +fn is_cycle(e: &anyhow::Error) -> bool { + hcore::hmemoizer::downcast_chain_ref::(e).is_some() +} + +fn get_error_kind(e: &anyhow::Error) -> pb::get_error::Kind { + if is_cycle(e) { + pb::get_error::Kind::Cycle + } else if hplugin::error::is_cancelled(e) { + pb::get_error::Kind::Cancelled + } else { + pb::get_error::Kind::Other + } +} + +/// Wrap a real provider as an ABI-stable [`crate::abi::DynProvider`] handle +/// (in-process; the cdylib entry produces the same handle across the boundary). +pub fn make_dyn_provider(provider: Arc) -> crate::abi::DynProvider { + stabby::boxed::Box::new(StableProviderImpl { provider }).into() +} + +/// Wrap a real managed driver as an ABI-stable [`crate::abi::DynManagedDriver`]. +pub fn make_dyn_managed_driver(driver: Arc) -> crate::abi::DynManagedDriver { + stabby::boxed::Box::new(StableManagedDriverImpl { driver }).into() +} + +/// Wraps an author `Provider` as a [`StableProvider`]. +pub struct StableProviderImpl { + pub provider: Arc, +} + +impl StableProvider for StableProviderImpl { + extern "C" fn config(&self) -> SString { + self.provider + .config(ConfigRequest {}) + .map(|r| r.name) + .unwrap_or_default() + .into() + } + + extern "C" fn list<'a>(&'a self, req: SVec) -> DynFuture<'a, SVec> { + let provider = Arc::clone(&self.provider); + stabby::boxed::Box::new(async move { + let req = pb::ListRequest::decode(&req[..]).unwrap_or_default(); + let tok = StdCancellationToken::new(); + let lreq = ListRequest { + request_id: req.request_id, + package: PkgBuf::from(req.package), + states: req.states.into_iter().map(convert::state_from_pb).collect(), + }; + let mut bodies = Vec::new(); + match provider.list(lreq, &tok).await { + Ok(iter) => { + for item in iter { + match item { + Ok(lr) => bodies.push(Body::StreamItem(pb::StreamItem { + item: pb::ListResponse { + addr: Some(convert::addr_to_pb(&lr.addr)), + } + .encode_to_vec() + .into(), + })), + Err(e) => { + bodies.push(stream_err(e.to_string())); + return stream(bodies); + } + } + } + bodies.push(Body::StreamEnd(pb::StreamEnd { error: None })); + } + Err(e) => bodies.push(stream_err(e.to_string())), + } + stream(bodies) + }) + .into() + } + + extern "C" fn list_packages<'a>(&'a self, req: SVec) -> DynFuture<'a, SVec> { + let provider = Arc::clone(&self.provider); + stabby::boxed::Box::new(async move { + let req = pb::ListPackagesRequest::decode(&req[..]).unwrap_or_default(); + let tok = StdCancellationToken::new(); + let lreq = ListPackagesRequest { + prefix: PkgBuf::from(req.prefix), + }; + let mut bodies = Vec::new(); + match provider.list_packages(lreq, &tok).await { + Ok(iter) => { + for item in iter { + match item { + Ok(lpr) => bodies.push(Body::StreamItem(pb::StreamItem { + item: pb::ListPackageResponse { + pkg: lpr.pkg.as_str().to_string(), + } + .encode_to_vec() + .into(), + })), + Err(e) => { + bodies.push(stream_err(e.to_string())); + return stream(bodies); + } + } + } + bodies.push(Body::StreamEnd(pb::StreamEnd { error: None })); + } + Err(e) => bodies.push(stream_err(e.to_string())), + } + stream(bodies) + }) + .into() + } + + extern "C" fn get<'a>(&'a self, req: SVec, exec: DynExecutor) -> DynFuture<'a, SVec> { + let provider = Arc::clone(&self.provider); + stabby::boxed::Box::new(async move { + let req = pb::GetRequest::decode(&req[..]).unwrap_or_default(); + let executor: Arc = Arc::new(GuestExecutor::new(exec)); + let tok = StdCancellationToken::new(); + let greq = GetRequest { + request_id: req.request_id, + addr: convert::addr_from_pb(req.addr.unwrap_or_default()), + states: req.states.into_iter().map(convert::state_from_pb).collect(), + executor, + }; + let body = match provider.get(greq, &tok).await { + Ok(gr) => Body::GetResp(pb::GetResponse { + target_spec: Some(convert::target_spec_to_pb(&gr.target_spec)), + }), + Err(GetError::NotFound) => Body::GetErr(pb::GetError { + kind: pb::get_error::Kind::NotFound as i32, + message: String::new(), + }), + Err(GetError::Other(e)) => Body::GetErr(pb::GetError { + kind: get_error_kind(&e) as i32, + message: e.to_string(), + }), + }; + unary(body) + }) + .into() + } + + extern "C" fn probe<'a>(&'a self, req: SVec) -> DynFuture<'a, SVec> { + let provider = Arc::clone(&self.provider); + stabby::boxed::Box::new(async move { + let req = pb::ProbeRequest::decode(&req[..]).unwrap_or_default(); + let tok = StdCancellationToken::new(); + let preq = ProbeRequest { + request_id: req.request_id, + package: PkgBuf::from(req.package), + }; + let body = match provider.probe(preq, &tok).await { + Ok(pr) => Body::ProbeResp(pb::ProbeResponse { + states: pr.states.iter().map(convert::state_to_pb).collect(), + }), + Err(e) => err_body(e.to_string()), + }; + unary(body) + }) + .into() + } +} + +fn stream_err(message: String) -> Body { + Body::StreamEnd(pb::StreamEnd { + error: Some(pb::Error { + kind: pb::error::Kind::Other as i32, + message, + }), + }) +} + +/// Wraps an author `ManagedDriver` as a [`StableManagedDriver`]. +pub struct StableManagedDriverImpl { + pub driver: Arc, +} + +impl StableManagedDriver for StableManagedDriverImpl { + extern "C" fn config(&self) -> SString { + self.driver + .config(DriverConfigRequest {}) + .map(|r| r.name) + .unwrap_or_default() + .into() + } + + extern "C" fn parse<'a>(&'a self, req: SVec) -> DynFuture<'a, SVec> { + let driver = Arc::clone(&self.driver); + stabby::boxed::Box::new(async move { + let req = pb::ParseRequest::decode(&req[..]).unwrap_or_default(); + let tok = StdCancellationToken::new(); + let preq = ParseRequest { + request_id: req.request_id, + target_spec: Arc::new(convert::target_spec_from_pb( + req.target_spec.unwrap_or_default(), + )), + }; + let body = match driver.parse(preq, &tok).await { + Ok(resp) => match convert::target_def_to_pb(&resp.target_def) { + Ok(td) => Body::ParseResp(pb::ParseResponse { + target_def: Some(td), + }), + Err(e) => err_body(e.to_string()), + }, + Err(e) => err_body(e.to_string()), + }; + unary(body) + }) + .into() + } + + extern "C" fn apply_transitive<'a>(&'a self, req: SVec) -> DynFuture<'a, SVec> { + let driver = Arc::clone(&self.driver); + stabby::boxed::Box::new(async move { + let req = pb::ApplyTransitiveRequest::decode(&req[..]).unwrap_or_default(); + let tok = StdCancellationToken::new(); + let target_def = match convert::target_def_from_pb(req.target_def.unwrap_or_default()) { + Ok(td) => td, + Err(e) => return unary(err_body(e.to_string())), + }; + let areq = ApplyTransitiveRequest { + request_id: req.request_id, + target_def, + sandbox: convert::sandbox_from_pb(req.sandbox.unwrap_or_default()), + }; + let body = match driver.apply_transitive(areq, &tok).await { + Ok(resp) => match convert::target_def_to_pb(&resp.target_def) { + Ok(td) => Body::ApplyTransitiveResp(pb::ApplyTransitiveResponse { + target_def: Some(td), + }), + Err(e) => err_body(e.to_string()), + }, + Err(e) => err_body(e.to_string()), + }; + unary(body) + }) + .into() + } + + extern "C" fn run<'a>(&'a self, req: SVec, shell: bool) -> DynFuture<'a, SVec> { + let driver = Arc::clone(&self.driver); + stabby::boxed::Box::new(async move { + // `run` shells out via the reactor — execute on the cdylib's own + // runtime, bridge the result back to the host's polling task. + let (tx, rx) = tokio::sync::oneshot::channel(); + cdylib_runtime().spawn(async move { + // Receiver dropped only if the host gave up; ignore send failure. + drop(tx.send(run_impl(driver, req, shell).await)); + }); + rx.await + .unwrap_or_else(|_| unary(err_body("cdylib run task dropped".into()))) + }) + .into() + } +} + +async fn run_impl(driver: Arc, req: SVec, shell: bool) -> SVec { + let req = pb::ManagedRunRequest::decode(&req[..]).unwrap_or_default(); + let tok = StdCancellationToken::new(); + let target = match convert::target_def_from_pb(req.target.unwrap_or_default()) { + Ok(t) => t, + Err(e) => return unary(err_body(e.to_string())), + }; + let request_id = req.request_id; + let hashin = req.hashin; + let sandbox_dir = PathBuf::from(req.sandbox_dir); + let run_inputs: Vec = req.inputs.iter().map(run_input_from_pb).collect(); + let managed_inputs: Vec = + req.inputs.into_iter().map(managed_input_from_pb).collect(); + let rr = RunRequest { + request_id: &request_id, + target: &target, + tree_root_path: PathBuf::from(req.tree_root_path), + inputs: run_inputs, + hashin: hashin.as_str(), + stdin: None, + stdout: None, + stderr: None, + sandbox_dir: sandbox_dir.clone(), + }; + let mrr = ManagedRunRequest { + request: rr, + sandbox_dir, + sandbox_ws_dir: PathBuf::from(req.sandbox_ws_dir), + sandbox_pkg_dir: PathBuf::from(req.sandbox_pkg_dir), + inputs: managed_inputs, + }; + let result = if shell { + driver.run_shell(mrr, &tok).await + } else { + driver.run(mrr, &tok).await + }; + let body = match result { + Ok(resp) => Body::ManagedRunResp(pb::ManagedRunResponse { + artifacts: resp + .artifacts + .iter() + .map(convert::output_artifact_to_pb) + .collect(), + }), + Err(e) => err_body(e.to_string()), + }; + unary(body) +} + +fn run_input_from_pb(mi: &pb::ManagedRunInput) -> RunInput { + let ty = match pb::InputArtifactType::try_from(mi.r#type).unwrap_or(pb::InputArtifactType::Dep) + { + pb::InputArtifactType::Support => inputartifact::Type::Support, + _ => inputartifact::Type::Dep, + }; + RunInput { + artifact: inputartifact::InputArtifact { + r#type: ty, + origin_id: mi.origin_id.clone(), + // Input bytes live on the shared filesystem (unpack_root); read from + // disk, never from this Content. + content: Arc::new(NullContent), + }, + origin_id: mi.origin_id.clone(), + source_addr: convert::addr_from_pb(mi.source_addr.clone().unwrap_or_default()), + filters: mi.filters.clone(), + annotations: mi + .annotations + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + } +} + +fn managed_input_from_pb(mi: pb::ManagedRunInput) -> ManagedRunInput { + let input = run_input_from_pb(&mi); + ManagedRunInput { + input, + list_path: mi.list_path.map(PathBuf::from), + unpack_root: PathBuf::from(mi.unpack_root), + } +} + +/// Placeholder Content for materialized run inputs — bytes live on the shared +/// filesystem (unpack_root), so this is never read. +struct NullContent; +impl Content for NullContent { + fn reader(&self) -> Result> { + anyhow::bail!("managed run input content is on disk (unpack_root), not streamed") + } + fn walk(&self) -> Result> + '_>> { + anyhow::bail!("managed run input content is on disk (unpack_root), not streamed") + } + fn hashout(&self) -> Result { + Ok(String::new()) + } +} diff --git a/crates/plugin/src/provider.rs b/crates/plugin/src/provider.rs index 9470e326..2fe509e3 100644 --- a/crates/plugin/src/provider.rs +++ b/crates/plugin/src/provider.rs @@ -110,6 +110,33 @@ pub trait ProviderExecutor: Send + Sync { } } +/// The [`ProviderExecutor`] callback surface, addressed by request scope id +/// instead of carrying the executor inline. Used to route a plugin's callbacks +/// back to the right per-request executor over a transport boundary where the +/// executor itself cannot cross (out-of-process, or a dylib): the host registers +/// each `get`'s executor under a scope id, the plugin's callbacks carry that id, +/// and this trait dispatches them. In-process the implementation is a direct +/// call into the engine executor (no serialization); across a cdylib it is the +/// stable-ABI mirror. +pub trait ScopedExecutor: Send + Sync { + fn result<'a>( + &'a self, + request_id: &'a str, + addr: &'a Addr, + ) -> BoxFuture<'a, anyhow::Result>>; + fn note_dep<'a>( + &'a self, + request_id: &'a str, + addr: &'a Addr, + ) -> BoxFuture<'a, anyhow::Result<()>>; + fn query<'a>( + &'a self, + request_id: &'a str, + m: &'a Matcher, + extra_skip: &'a [String], + ) -> BoxFuture<'a, anyhow::Result>>; +} + pub struct GetRequest { pub request_id: String, pub addr: Addr, diff --git a/devenv.nix b/devenv.nix index 099bad5d..2f81fb32 100644 --- a/devenv.nix +++ b/devenv.nix @@ -67,22 +67,28 @@ in cd $DEVENV_ROOT/example/go/large && go mod tidy ''; # Set up the example workspace end to end: regenerate the large go repo and - # build the out-of-process go plugin into example/.heph3/heph-go-plugin, which - # example/.hephconfig2 launches via `bin: { path: .heph3/heph-go-plugin }`. + # Build the go plugin as a loadable cdylib into example/.heph3/heph-go-plugin.dylib, + # which example/.hephconfig2 loads in-process via `dylib: { path: ... }` behind the + # stable ABI (native speed — see ai-docs/PERFORMANCE.md). The cdylib basename is + # platform-specific (lib*.dylib on macOS, lib*.so on Linux). scripts.gen-example.exec = '' gen gen-go-large - cargo build --release -p plugin-go --bin heph-plugin-go - bin="$CARGO_TARGET_DIR/release/heph-plugin-go" + cargo build --release -p plugin-go-cdylib if [ "$(uname -s)" = "Darwin" ]; then - # Rewrite the nix-store libiconv load command to /usr/lib so the spawned - # plugin keeps launching after the store path is GC'd (same as the CLI). - bash "$DEVENV_ROOT/scripts/macos-portable.sh" "$bin" + lib="$CARGO_TARGET_DIR/release/libplugin_go_cdylib.dylib" + else + lib="$CARGO_TARGET_DIR/release/libplugin_go_cdylib.so" + fi + if [ "$(uname -s)" = "Darwin" ]; then + # Rewrite the nix-store libiconv load command to /usr/lib so the loaded + # dylib keeps resolving after the store path is GC'd (same as the CLI). + bash "$DEVENV_ROOT/scripts/macos-portable.sh" "$lib" fi - dest="$DEVENV_ROOT/example/.heph3/heph-go-plugin" + dest="$DEVENV_ROOT/example/.heph3/heph-go-plugin.dylib" mkdir -p "$(dirname "$dest")" # Atomic replace (new inode) so a running macOS process keeps its signature. - cp "$bin" "$dest.new" + cp "$lib" "$dest.new" mv -f "$dest.new" "$dest" ''; scripts.lint.exec = "echo '> clippy' && cargo clippy --all-targets --locked -- -D warnings && echo '> fmt' && cargo fmt --check ${qualityCrates}"; diff --git a/example/.hephconfig2 b/example/.hephconfig2 index 3a9136bc..3ba39de0 100644 --- a/example/.hephconfig2 +++ b/example/.hephconfig2 @@ -3,26 +3,26 @@ providers: options: patterns: - BUILD - # The go plugin runs out-of-process. `bin.path` (relative to the workspace - # root) points at the binary built by the `gen-example` devenv script. The - # provider + its go_* drivers share one process because they resolve to the - # same launch command. + # The go plugin is a loadable cdylib, loaded in-process behind the stable ABI + # (native speed, no IPC). `dylib.path` (relative to the workspace root) points at + # the artifact built by the `gen-example` devenv script. The provider + its go_* + # drivers share the one loaded library. - name: go - bin: - path: .heph3/heph-go-plugin + dylib: + path: .heph3/heph-go-plugin.dylib drivers: - name: exec - name: bash - name: sh - name: go_golist - bin: - path: .heph3/heph-go-plugin + dylib: + path: .heph3/heph-go-plugin.dylib - name: go_embed - bin: - path: .heph3/heph-go-plugin + dylib: + path: .heph3/heph-go-plugin.dylib - name: go_testmain - bin: - path: .heph3/heph-go-plugin + dylib: + path: .heph3/heph-go-plugin.dylib #fuse: # enabled: true # Remote (shared) caches. Each entry is keyed by a name and has a `uri` plus diff --git a/src/commands/bootstrap.rs b/src/commands/bootstrap.rs index 6ab427a3..9fb18191 100644 --- a/src/commands/bootstrap.rs +++ b/src/commands/bootstrap.rs @@ -100,11 +100,14 @@ pub fn new_engine() -> anyhow::Result<(Arc, ShutdownTrigger)> { // Names a `bin:` config entry routes to an out-of-process plugin. For those, // skip the in-process built-in factory below and register a remote handle // (spawned by `register_bin_plugins`) under the same name instead. + // A name routes out-of-process (`bin:`) OR to an in-process loadable cdylib + // (`dylib:`) / wasm component (`wasm:`); either way the built-in in-process + // factory below is skipped. let bin_names: std::collections::HashSet<&str> = file .providers .iter() .chain(file.drivers.iter()) - .filter(|en| en.bin.is_some()) + .filter(|en| en.bin.is_some() || en.dylib.is_some() || en.wasm.is_some()) .map(|en| en.name.as_str()) .collect(); @@ -174,6 +177,14 @@ pub fn new_engine() -> anyhow::Result<(Arc, ShutdownTrigger)> { // `go` provider + its `go_*` drivers) register against that one connection. register_bin_plugins(&mut e, &root, &home_dir, &file)?; + // Load + register every in-process loadable plugin declared via `dylib:`. One + // cdylib per distinct path serves a provider + its drivers (e.g. `go` plus the + // `go_*` drivers), loaded once behind the stable ABI at native speed. + register_dylib_plugins(&mut e, &root, &home_dir, &file)?; + + // Load + register every wasm-component plugin declared via `wasm:`. + register_wasm_plugins(&mut e, &root, &home_dir, &file)?; + e.apply_config(&file.providers, &file.drivers)?; let engine = Arc::new(e); @@ -616,10 +627,9 @@ fn register_bin_plugins( let (program, args) = argv .split_first() .context("internal: empty plugin argv after resolution")?; - // Proto transport over a UDS socketpair (fd 3). The shm transport exists - // (feature `shm`, plugin-remote::spawn_shm) but is slower here — the - // per-call cost is async wakeup latency, not syscalls — so the default - // launch uses proto. + // Cold protocol over a UDS socketpair (fd 3). For native-speed in-process + // plugins use `dylib:` (the stable ABI) instead; this `bin:` path is the + // portable out-of-process transport. let ((r, w), child) = hplugin_remote::spawn_streams(std::path::Path::new(program), args, &env) .with_context(|| format!("spawn plugin `{program}`"))?; @@ -647,6 +657,143 @@ fn register_bin_plugins( Ok(()) } +/// Load + register every `dylib:` plugin. One cdylib per distinct path is loaded +/// in-process behind the stable ABI (ABI-checked via stabby type reports), and the +/// provider + drivers it exports are registered under their own names. The engine +/// then drives them through its normal traits at native speed. +#[cfg(unix)] +fn register_dylib_plugins( + e: &mut engine::Engine, + root: &std::path::Path, + home_dir: &std::path::Path, + file: &config_yaml::ConfigYaml, +) -> anyhow::Result<()> { + use std::collections::BTreeSet; + + let mut paths: BTreeSet = BTreeSet::new(); + for en in file.providers.iter().chain(file.drivers.iter()) { + if let Some(d) = &en.dylib { + paths.insert(resolve_artifact_path(d.resolve(&en.name)?, root, home_dir)?); + } + } + + let root_str = root.to_string_lossy().into_owned(); + for path in paths { + let (provider, drivers) = hplugin_stabby::load_stable::load(&path, &root_str) + .with_context(|| format!("load plugin dylib {}", path.display()))?; + if let Some(p) = provider { + let name = p.name().to_string(); + e.register_provider_factory(&name, move |_init, _opts| Ok(Box::new(p.clone())))?; + } + for (name, drv) in drivers { + e.register_managed_driver_factory(&name, move |_init, _opts| { + Ok(Box::new(drv.clone())) + })?; + } + } + Ok(()) +} + +#[cfg(not(unix))] +fn register_dylib_plugins( + _e: &mut engine::Engine, + _root: &std::path::Path, + _home_dir: &std::path::Path, + file: &config_yaml::ConfigYaml, +) -> anyhow::Result<()> { + if file + .providers + .iter() + .chain(file.drivers.iter()) + .any(|en| en.dylib.is_some()) + { + anyhow::bail!("`dylib:` plugins are only supported on unix"); + } + Ok(()) +} + +/// Resolve a loadable-artifact source to a concrete file path: a `path` is taken +/// relative to the workspace `root`; a `url` is downloaded + cached (same fetch +/// path as `bin:` urls). +#[cfg(unix)] +fn resolve_artifact_path( + src: config_yaml::ArtifactSource, + root: &std::path::Path, + home_dir: &std::path::Path, +) -> anyhow::Result { + Ok(match src { + config_yaml::ArtifactSource::Path(p) => root.join(p), + config_yaml::ArtifactSource::Url(u) => download_plugin(&u, home_dir)?, + }) +} + +/// Load + register every `wasm:` plugin. One wasm component per distinct path is +/// loaded in-process via wasmtime (capability-sandboxed); the provider and/or +/// driver it exports register under the config entry names that point at it. +#[cfg(all(unix, feature = "wasm"))] +fn register_wasm_plugins( + e: &mut engine::Engine, + root: &std::path::Path, + home_dir: &std::path::Path, + file: &config_yaml::ConfigYaml, +) -> anyhow::Result<()> { + use std::collections::BTreeMap; + + // path -> (provider entry name, driver entry name) + let mut by_path: BTreeMap, Option)> = + BTreeMap::new(); + for en in &file.providers { + if let Some(w) = &en.wasm { + let p = resolve_artifact_path(w.resolve(&en.name)?, root, home_dir)?; + by_path.entry(p).or_default().0 = Some(en.name.clone()); + } + } + for en in &file.drivers { + if let Some(w) = &en.wasm { + let p = resolve_artifact_path(w.resolve(&en.name)?, root, home_dir)?; + by_path.entry(p).or_default().1 = Some(en.name.clone()); + } + } + + for (path, (prov, drv)) in by_path { + let bytes = + std::fs::read(&path).with_context(|| format!("read wasm plugin {}", path.display()))?; + let plugin = hplugin_remote::wasm::WasmPlugin::load( + &bytes, + prov.clone().unwrap_or_default(), + drv.clone().unwrap_or_default(), + ) + .with_context(|| format!("load wasm plugin {}", path.display()))?; + if let Some(name) = prov { + let pl = std::sync::Arc::clone(&plugin); + e.register_provider_factory(&name, move |_init, _opts| Ok(Box::new(pl.provider())))?; + } + if let Some(name) = drv { + let pl = std::sync::Arc::clone(&plugin); + e.register_driver_factory(&name, move |_init, _opts| Ok(Box::new(pl.driver())))?; + } + } + Ok(()) +} + +#[cfg(not(all(unix, feature = "wasm")))] +fn register_wasm_plugins( + _e: &mut engine::Engine, + _root: &std::path::Path, + _home_dir: &std::path::Path, + file: &config_yaml::ConfigYaml, +) -> anyhow::Result<()> { + if file + .providers + .iter() + .chain(file.drivers.iter()) + .any(|en| en.wasm.is_some()) + { + anyhow::bail!("`wasm:` plugins require building heph with the `wasm` feature on unix"); + } + Ok(()) +} + #[cfg(not(unix))] fn register_bin_plugins( _e: &mut engine::Engine, From 91e56bec42db30d139cf2b61c1094176984ba979 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Wed, 17 Jun 2026 00:01:49 +0200 Subject: [PATCH 37/44] refactor(plugin-go): drop go as a compiled-in built-in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The go plugin now ships only as a separate artifact — loaded in-process via `dylib:` (native speed) or spawned via `bin:` — so heph no longer links plugin-go into the binary. Removes the in-process `go`/`go_*` factory registrations, the root `hplugin-go` dependency, and the `heph::plugingo` re-export. A config entry named `go` without a transport now has no factory (by design). plugingo-e2e constructs the provider directly, so it depends on `plugin-go` itself instead of the (removed) `heph::plugingo` re-export. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 2 +- Cargo.toml | 1 - crates/plugingo-e2e/Cargo.toml | 2 ++ crates/plugingo-e2e/tests/common/mod.rs | 7 ++++- src/commands/bootstrap.rs | 36 +++---------------------- src/lib.rs | 1 - 6 files changed, 13 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 936693ff..da6d7c0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2331,7 +2331,6 @@ dependencies = [ "plugin", "plugin-buildfile", "plugin-exec", - "plugin-go", "plugin-nix", "plugin-query", "plugin-remote", @@ -4494,6 +4493,7 @@ dependencies = [ "anyhow", "futures", "heph", + "plugin-go", "tempfile", "testkit", "tokio", diff --git a/Cargo.toml b/Cargo.toml index fd3cf9b3..b5fa85fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -112,7 +112,6 @@ hdriver-support = { package = "driver-support", path = "crates/driver-support" } hplugin-exec = { package = "plugin-exec", path = "crates/plugin-exec" } hplugin-nix = { package = "plugin-nix", path = "crates/plugin-nix" } hplugin-query = { package = "plugin-query", path = "crates/plugin-query" } -hplugin-go = { package = "plugin-go", path = "crates/plugin-go" } hplugin-remote = { package = "plugin-remote", path = "crates/plugin-remote" } hplugin-stabby = { package = "plugin-stabby", path = "crates/plugin-stabby", features = ["host", "guest"] } htelemetry = { package = "telemetry", path = "crates/telemetry" } diff --git a/crates/plugingo-e2e/Cargo.toml b/crates/plugingo-e2e/Cargo.toml index ce8b1e6b..1043e450 100644 --- a/crates/plugingo-e2e/Cargo.toml +++ b/crates/plugingo-e2e/Cargo.toml @@ -9,6 +9,8 @@ workspace = true [dev-dependencies] heph = { path = "../.." } +# go is no longer a heph built-in; the e2e harness constructs it directly. +plugin-go = { path = "../plugin-go" } htestkit = { package = "testkit", path = "../testkit" } tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] } anyhow = "1" diff --git a/crates/plugingo-e2e/tests/common/mod.rs b/crates/plugingo-e2e/tests/common/mod.rs index c617e49f..e1b77044 100644 --- a/crates/plugingo-e2e/tests/common/mod.rs +++ b/crates/plugingo-e2e/tests/common/mod.rs @@ -1,9 +1,14 @@ +// Shared integration-test helper module: each test binary `mod common`s this but +// uses only some helpers, so per-binary dead-code/unused-import warnings are +// expected here (the items are exercised across the suite). +#![allow(dead_code, unused_imports)] + use anyhow::Context as _; use heph::pluginbuildfile; use heph::pluginexec; -use heph::plugingo; use heph::pluginstatictarget; use htestkit::{Workspace, WorkspaceBuilder, copy_dir_to_tempdir}; +use plugin_go::plugingo; use std::path::PathBuf; use tempfile::TempDir; diff --git a/src/commands/bootstrap.rs b/src/commands/bootstrap.rs index 9fb18191..fd7a0ea4 100644 --- a/src/commands/bootstrap.rs +++ b/src/commands/bootstrap.rs @@ -6,9 +6,7 @@ use tokio::runtime::{Builder, Runtime}; use tokio::sync::mpsc; use crate::engine::config_yaml; -use crate::{ - engine, pluginbuildfile, pluginexec, plugingo, pluginhostbin, pluginnix, plugintextfile, -}; +use crate::{engine, pluginbuildfile, pluginexec, pluginhostbin, pluginnix, plugintextfile}; /// Builds the multi-thread runtime used by every command entry point. /// @@ -126,35 +124,9 @@ pub fn new_engine() -> anyhow::Result<(Arc, ShutdownTrigger)> { )) })?; } - if !bin_names.contains("go") { - e.register_provider_factory("go", |init, opts| { - Ok(Box::new(plugingo::Provider::from_options( - init.root.to_path_buf(), - &init.skip_dirs, - &init.skip_globs, - opts, - init.walker.clone(), - )?)) - })?; - } - if !bin_names.contains("go_golist") { - e.register_managed_driver_factory("go_golist", |_init, opts| { - config_yaml::deny_unknown("go_golist driver", opts, &[])?; - Ok(Box::new(plugingo::GoGolistDriver::new("//@heph/bin:go"))) - })?; - } - if !bin_names.contains("go_embed") { - e.register_managed_driver_factory("go_embed", |_init, opts| { - config_yaml::deny_unknown("go_embed driver", opts, &[])?; - Ok(Box::new(plugingo::GoEmbedDriver)) - })?; - } - if !bin_names.contains("go_testmain") { - e.register_managed_driver_factory("go_testmain", |_init, opts| { - config_yaml::deny_unknown("go_testmain driver", opts, &[])?; - Ok(Box::new(plugingo::GoTestmainDriver)) - })?; - } + // The go plugin is no longer a compiled-in built-in: it ships as a separate + // artifact loaded via `dylib:` (native speed) or spawned via `bin:`. A config + // entry named `go`/`go_*` without a transport therefore has no factory. if !bin_names.contains("exec") { e.register_managed_driver_factory("exec", |_init, opts| { Ok(Box::new(pluginexec::Driver::from_options_exec(opts)?)) diff --git a/src/lib.rs b/src/lib.rs index 9c61c483..857b7925 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,7 +29,6 @@ pub use hmodel::{htaddr, htmatcher, htpkg}; pub use hplugin::htspec; pub use hplugin_buildfile::pluginbuildfile; pub use hplugin_exec::pluginexec; -pub use hplugin_go::plugingo; pub use hplugin_nix::pluginnix; pub use hplugin_query::pluginquery; #[cfg(target_os = "macos")] From 8d5126e79598e55ed758e07f5f4ecda5c6268940 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Wed, 17 Jun 2026 00:36:51 +0200 Subject: [PATCH 38/44] ci(lint): clippy with --all-features + fmt the new crates; fix surfaced errors CI lint only checked default-feature code, so the shm/wasm feature-gated transports were never linted, and fmt skipped the new plugin-stabby / plugin-go-cdylib crates. - devenv `lint`: add a `clippy --all-targets --all-features` pass (covers feature-gated code and both arms of `#[cfg(feature)]`); add plugin-stabby and plugin-go-cdylib to the fmt-checked qualityCrates. - Fix the errors this surfaces: - non-binding `let _` on a must_use future result in the shm liveness watchers (plugin-remote spawn_shm, plugin-sdk serve_components_shm). - fmt-align e2e/plugin-echo to the current toolchain's rustfmt. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/e2e/tests/remote_plugin.rs | 9 ++++++--- crates/plugin-echo/src/main.rs | 8 ++++++-- crates/plugin-echo/tests/subprocess.rs | 6 ++---- crates/plugin-remote/src/spawn.rs | 2 +- crates/plugin-sdk/src/serve.rs | 2 +- devenv.nix | 8 ++++++-- 6 files changed, 22 insertions(+), 13 deletions(-) diff --git a/crates/e2e/tests/remote_plugin.rs b/crates/e2e/tests/remote_plugin.rs index bf05cf3f..ab1c48b0 100644 --- a/crates/e2e/tests/remote_plugin.rs +++ b/crates/e2e/tests/remote_plugin.rs @@ -40,7 +40,9 @@ impl Provider for TestProvider { addr: addr("pkg", "a"), })]; Ok(Box::new(items.into_iter()) - as Box> + Send>) + as Box< + dyn Iterator> + Send, + >) }) } @@ -57,7 +59,9 @@ impl Provider for TestProvider { pkg: PkgBuf::from("pkg"), })]; Ok(Box::new(items.into_iter()) - as Box> + Send>) + as Box< + dyn Iterator> + Send, + >) }) } @@ -108,4 +112,3 @@ async fn remote_provider_serves_get_spec_through_engine() -> anyhow::Result<()> Ok(()) } - diff --git a/crates/plugin-echo/src/main.rs b/crates/plugin-echo/src/main.rs index c0af4a6e..8c1277b0 100644 --- a/crates/plugin-echo/src/main.rs +++ b/crates/plugin-echo/src/main.rs @@ -44,7 +44,9 @@ impl Provider for EchoProvider { }), ]; Ok(Box::new(items.into_iter()) - as Box> + Send>) + as Box< + dyn Iterator> + Send, + >) }) } @@ -61,7 +63,9 @@ impl Provider for EchoProvider { pkg: PkgBuf::from("//pkg"), })]; Ok(Box::new(items.into_iter()) - as Box> + Send>) + as Box< + dyn Iterator> + Send, + >) }) } diff --git a/crates/plugin-echo/tests/subprocess.rs b/crates/plugin-echo/tests/subprocess.rs index a04ab468..9f958998 100644 --- a/crates/plugin-echo/tests/subprocess.rs +++ b/crates/plugin-echo/tests/subprocess.rs @@ -5,13 +5,11 @@ use futures::future::BoxFuture; use hcore::hartifactcontent::{Content, WalkEntry}; use hcore::hasync::StdCancellationToken; -use hplugin::eresult::{ArtifactMeta, EResult}; -use hplugin::provider::{ - ConfigRequest, GetRequest, ListRequest, Provider, ProviderExecutor, -}; use hmodel::htaddr::Addr; use hmodel::htmatcher::Matcher; use hmodel::htpkg::PkgBuf; +use hplugin::eresult::{ArtifactMeta, EResult}; +use hplugin::provider::{ConfigRequest, GetRequest, ListRequest, Provider, ProviderExecutor}; use plugin_remote::spawn_plugin; use std::collections::BTreeMap; use std::io::{Cursor, Read}; diff --git a/crates/plugin-remote/src/spawn.rs b/crates/plugin-remote/src/spawn.rs index 7f58574b..d80f8bbd 100644 --- a/crates/plugin-remote/src/spawn.rs +++ b/crates/plugin-remote/src/spawn.rs @@ -143,7 +143,7 @@ pub fn spawn_shm( let mux = plugin.mux_handle(); tokio::spawn(async move { let mut byte = [0u8; 1]; - let _ = uds_r.read(&mut byte).await; // resolves on EOF (no more data sent) + let _n = uds_r.read(&mut byte).await; // resolves on EOF (no more data sent) mux.close(); drop(uds_w); drop(child); diff --git a/crates/plugin-sdk/src/serve.rs b/crates/plugin-sdk/src/serve.rs index 02018169..0fe30870 100644 --- a/crates/plugin-sdk/src/serve.rs +++ b/crates/plugin-sdk/src/serve.rs @@ -192,7 +192,7 @@ pub async fn serve_components_shm( let watch_mux = Arc::clone(&mux); tokio::spawn(async move { let mut byte = [0u8; 1]; - let _ = uds_r.read(&mut byte).await; + let _n = uds_r.read(&mut byte).await; watch_mux.close(); }); let _hold = uds_w; // keep the write half open for host-side liveness diff --git a/devenv.nix b/devenv.nix index 2f81fb32..5ba392d6 100644 --- a/devenv.nix +++ b/devenv.nix @@ -2,7 +2,7 @@ let binLocation = "$HOME/.local/bin/heph3"; - qualityCrates = "-p heph -p e2e -p testkit -p plugingo-e2e -p htspec-derive -p core -p walk -p proc -p model -p sandboxfuse -p plugin -p plugin-abi -p plugin-sdk -p plugin-remote -p plugin-echo -p builtins -p plugin-buildfile -p driver-support -p plugin-exec -p plugin-nix -p plugin-query -p plugin-go -p telemetry -p tui -p lock -p engine"; + qualityCrates = "-p heph -p e2e -p testkit -p plugingo-e2e -p htspec-derive -p core -p walk -p proc -p model -p sandboxfuse -p plugin -p plugin-abi -p plugin-sdk -p plugin-remote -p plugin-stabby -p plugin-go-cdylib -p plugin-echo -p builtins -p plugin-buildfile -p driver-support -p plugin-exec -p plugin-nix -p plugin-query -p plugin-go -p telemetry -p tui -p lock -p engine"; in { # https://devenv.sh/basics/ @@ -91,7 +91,11 @@ in cp "$lib" "$dest.new" mv -f "$dest.new" "$dest" ''; - scripts.lint.exec = "echo '> clippy' && cargo clippy --all-targets --locked -- -D warnings && echo '> fmt' && cargo fmt --check ${qualityCrates}"; + # Lint default-feature code, then again with every feature enabled (so + # feature-gated code — shm/wasm transports, both arms of `#[cfg(feature)]` — is + # covered too), then fmt-check all hand-written crates (qualityCrates; generated + # gen/proto is excluded). + scripts.lint.exec = "echo '> clippy' && cargo clippy --all-targets --locked -- -D warnings && echo '> clippy --all-features' && cargo clippy --all-targets --all-features --locked -- -D warnings && echo '> fmt' && cargo fmt --check ${qualityCrates}"; scripts.fix.exec = "cargo fix --allow-dirty && cargo fmt ${qualityCrates}"; scripts.tst.exec = "cargo test --locked --all"; From e3a551c5f95b52d4b6755e3e81e7676c72c39635 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Wed, 17 Jun 2026 00:45:02 +0200 Subject: [PATCH 39/44] ci: ship shm+wasm by default; test all features - Default features now include `shm` and `wasm`, so the shipped heph binary supports every plugin transport (bin: UDS/shm, dylib:, wasm:) out of the box. - `tst` runs `cargo test --all --all-features`, so CI exercises the feature-gated transports + their e2e tests (shm cross-process, wasm component) instead of only default-feature code. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 8 +++++--- devenv.nix | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b5fa85fd..825f0de6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -178,11 +178,13 @@ reqwest = { version = "0.13", default-features = false, features = ["blocking", # and falls back to the copy path when FUSE is unavailable, so a default build # is safe on hosts without it. The `-weak-lfuse` link arg is emitted by this # crate's build.rs (the final binary link). -default = ["fuse-sandbox"] +# shm + wasm ship by default so the binary supports every plugin transport +# (bin: UDS, bin: shm, dylib:, wasm:) out of the box. +default = ["fuse-sandbox", "shm", "wasm"] fuse-sandbox = ["hsandboxfuse/fuse-sandbox"] -# Out-of-process plugin shm tier (iceoryx2 byte-pipe under the Mux). Opt-in. +# Out-of-process plugin shm tier (iceoryx2 byte-pipe under the Mux). shm = ["hplugin-remote/shm"] -# In-process wasm-component plugins (`wasm:` config), via wasmtime. Opt-in. +# In-process wasm-component plugins (`wasm:` config), via wasmtime. wasm = ["hplugin-remote/wasm"] [dev-dependencies] diff --git a/devenv.nix b/devenv.nix index 5ba392d6..4c5bc93a 100644 --- a/devenv.nix +++ b/devenv.nix @@ -97,7 +97,9 @@ in # gen/proto is excluded). scripts.lint.exec = "echo '> clippy' && cargo clippy --all-targets --locked -- -D warnings && echo '> clippy --all-features' && cargo clippy --all-targets --all-features --locked -- -D warnings && echo '> fmt' && cargo fmt --check ${qualityCrates}"; scripts.fix.exec = "cargo fix --allow-dirty && cargo fmt ${qualityCrates}"; - scripts.tst.exec = "cargo test --locked --all"; + # Test everything, including feature-gated code (shm/wasm transports, the + # cross-process + wasm e2e tests) — `--all-features` enables every crate feature. + scripts.tst.exec = "cargo test --locked --all --all-features"; scripts.build-profile.exec = ''cargo build --profile profiling''; scripts.run-profile.exec = ''$CARGO_TARGET_DIR/profiling/heph "''${@}"''; From 27d735e36b864cc0323f48671f7beccfd5bf15cd Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Wed, 17 Jun 2026 01:06:22 +0200 Subject: [PATCH 40/44] ci: line-tables-only test debuginfo to fit the all-features link `cargo test --all --all-features` links wasmtime + the wasm toolchain + iceoryx2 into the integration-test binaries; full debuginfo made the linker OOM (SIGBUS, ld exit 135) on CI runners. line-tables-only keeps backtraces with line numbers while cutting the link size enough to fit. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 6 ++++++ crates/wasm-guests/helloworld/src/bindings.rs | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 825f0de6..2e2c2b95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,12 @@ inherits = "release" debug = true strip = "none" +# Test binaries link `--all-features` in CI (wasmtime + iceoryx2 + the wasm +# toolchain), so full debuginfo makes the linker OOM on CI runners. Line-tables +# keep backtraces with line numbers while shrinking the link enough to fit. +[profile.test] +debug = "line-tables-only" + [profile.release] # Thin LTO keeps cross-crate inlining but links in parallel (fat `lto = true` # is a single-threaded whole-program pass that dominated wall time and erased diff --git a/crates/wasm-guests/helloworld/src/bindings.rs b/crates/wasm-guests/helloworld/src/bindings.rs index 82993ae7..327ea735 100644 --- a/crates/wasm-guests/helloworld/src/bindings.rs +++ b/crates/wasm-guests/helloworld/src/bindings.rs @@ -861,7 +861,9 @@ macro_rules! __export_plugin_impl { #[doc(inline)] pub(crate) use __export_plugin_impl as export; #[cfg(target_arch = "wasm32")] -#[unsafe(link_section = "component-type:wit-bindgen:0.41.0:heph:plugin:plugin:encoded world")] +#[unsafe( + link_section = "component-type:wit-bindgen:0.41.0:heph:plugin:plugin:encoded world" +)] #[doc(hidden)] #[allow(clippy::octal_escapes)] pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; 555] = *b"\ From 4d31222acbccd369f23ec715e9e57e2b375aaa69 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Wed, 17 Jun 2026 01:29:49 +0200 Subject: [PATCH 41/44] ci: targeted feature test passes instead of --all-features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cargo test --all --all-features` aborted the macOS Test job: --all-features turns on `fuse-sandbox` workspace-wide, which hard-links libfuse into test binaries (e.g. plugin-abi's) that lack the `-weak-lfuse` build.rs, so they fail to launch on the macFUSE-less runner (dyld: Library not loaded: libfuse.2.dylib). Replace with a default `cargo test --all` (heph default now pulls shm+wasm) plus targeted per-crate passes for the feature-gated transports — none enable fuse-sandbox, so no spurious libfuse link: - plugin-abi --features shm (shm cross-process) - plugin-remote --features shm,wasm (proto/shm/wasm e2e) - plugin-stabby --features host,guest (stabby roundtrip) Co-Authored-By: Claude Opus 4.8 (1M context) --- devenv.nix | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/devenv.nix b/devenv.nix index 4c5bc93a..e284a2c1 100644 --- a/devenv.nix +++ b/devenv.nix @@ -97,9 +97,13 @@ in # gen/proto is excluded). scripts.lint.exec = "echo '> clippy' && cargo clippy --all-targets --locked -- -D warnings && echo '> clippy --all-features' && cargo clippy --all-targets --all-features --locked -- -D warnings && echo '> fmt' && cargo fmt --check ${qualityCrates}"; scripts.fix.exec = "cargo fix --allow-dirty && cargo fmt ${qualityCrates}"; - # Test everything, including feature-gated code (shm/wasm transports, the - # cross-process + wasm e2e tests) — `--all-features` enables every crate feature. - scripts.tst.exec = "cargo test --locked --all --all-features"; + # Test everything. The default pass covers all crates with their default + # features (heph default now pulls shm+wasm). The targeted passes then exercise + # the feature-gated transport code + e2e (shm cross-process, wasm component, + # stabby roundtrip). NB: a single `--all-features` run can't be used — it turns + # on `fuse-sandbox` workspace-wide, hard-linking libfuse into crates that lack + # the `-weak-lfuse` build.rs, which then abort at launch on macFUSE-less runners. + scripts.tst.exec = "cargo test --locked --all && cargo test --locked -p plugin-abi --features shm && cargo test --locked -p plugin-remote --features shm,wasm && cargo test --locked -p plugin-stabby --features host,guest"; scripts.build-profile.exec = ''cargo build --profile profiling''; scripts.run-profile.exec = ''$CARGO_TARGET_DIR/profiling/heph "''${@}"''; From ff110bb54f04796ea78499562300c078784847af Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Wed, 17 Jun 2026 01:58:20 +0200 Subject: [PATCH 42/44] ci: weak-link libfuse in the new plugin crates' test binaries `cargo test --all` unifies the default `fuse-sandbox` feature across the workspace, so the new crates' test binaries transitively link `fuser` -> libfuse through their hplugin/sandboxfuse deps. Without `-weak-lfuse` they hard-link /usr/local/lib/libfuse.2.dylib and dyld aborts at launch on the macFUSE-less macOS runner (plugin-abi's test binary, SIGABRT). Add the standard `-weak-lfuse` build.rs (matching plugin-exec/plugingo-e2e/heph) to plugin-abi, plugin-sdk, plugin-remote, plugin-stabby. Verified: all test binaries now use LC_LOAD_WEAK_DYLIB for libfuse (none hard-link it). Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-abi/build.rs | 15 +++++++++++++++ crates/plugin-remote/build.rs | 15 +++++++++++++++ crates/plugin-sdk/build.rs | 15 +++++++++++++++ crates/plugin-stabby/build.rs | 15 +++++++++++++++ 4 files changed, 60 insertions(+) create mode 100644 crates/plugin-abi/build.rs create mode 100644 crates/plugin-remote/build.rs create mode 100644 crates/plugin-sdk/build.rs create mode 100644 crates/plugin-stabby/build.rs diff --git a/crates/plugin-abi/build.rs b/crates/plugin-abi/build.rs new file mode 100644 index 00000000..4cada8f2 --- /dev/null +++ b/crates/plugin-abi/build.rs @@ -0,0 +1,15 @@ +//! Weak-link libfuse on macOS (see `crates/sandboxfuse/build.rs` for the full +//! rationale). This crate transitively links `fuser` -> libfuse through its +//! `hplugin`/`sandboxfuse` dependencies (the `fuse-sandbox` feature, default-on +//! at the `heph` bin and unified across the workspace by `cargo test --all`). +//! `rustc-link-arg` does NOT propagate from a dependency's build script to a +//! dependent's binaries, so every test-binary-producing crate that links fuser +//! must emit `-weak-lfuse` at its own final link or its test binary hard-links +//! `/usr/local/lib/libfuse.2.dylib` and dyld aborts at launch on hosts without +//! macFUSE — including CI runners. +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { + println!("cargo::rustc-link-arg=-weak-lfuse"); + } +} diff --git a/crates/plugin-remote/build.rs b/crates/plugin-remote/build.rs new file mode 100644 index 00000000..4cada8f2 --- /dev/null +++ b/crates/plugin-remote/build.rs @@ -0,0 +1,15 @@ +//! Weak-link libfuse on macOS (see `crates/sandboxfuse/build.rs` for the full +//! rationale). This crate transitively links `fuser` -> libfuse through its +//! `hplugin`/`sandboxfuse` dependencies (the `fuse-sandbox` feature, default-on +//! at the `heph` bin and unified across the workspace by `cargo test --all`). +//! `rustc-link-arg` does NOT propagate from a dependency's build script to a +//! dependent's binaries, so every test-binary-producing crate that links fuser +//! must emit `-weak-lfuse` at its own final link or its test binary hard-links +//! `/usr/local/lib/libfuse.2.dylib` and dyld aborts at launch on hosts without +//! macFUSE — including CI runners. +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { + println!("cargo::rustc-link-arg=-weak-lfuse"); + } +} diff --git a/crates/plugin-sdk/build.rs b/crates/plugin-sdk/build.rs new file mode 100644 index 00000000..4cada8f2 --- /dev/null +++ b/crates/plugin-sdk/build.rs @@ -0,0 +1,15 @@ +//! Weak-link libfuse on macOS (see `crates/sandboxfuse/build.rs` for the full +//! rationale). This crate transitively links `fuser` -> libfuse through its +//! `hplugin`/`sandboxfuse` dependencies (the `fuse-sandbox` feature, default-on +//! at the `heph` bin and unified across the workspace by `cargo test --all`). +//! `rustc-link-arg` does NOT propagate from a dependency's build script to a +//! dependent's binaries, so every test-binary-producing crate that links fuser +//! must emit `-weak-lfuse` at its own final link or its test binary hard-links +//! `/usr/local/lib/libfuse.2.dylib` and dyld aborts at launch on hosts without +//! macFUSE — including CI runners. +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { + println!("cargo::rustc-link-arg=-weak-lfuse"); + } +} diff --git a/crates/plugin-stabby/build.rs b/crates/plugin-stabby/build.rs new file mode 100644 index 00000000..4cada8f2 --- /dev/null +++ b/crates/plugin-stabby/build.rs @@ -0,0 +1,15 @@ +//! Weak-link libfuse on macOS (see `crates/sandboxfuse/build.rs` for the full +//! rationale). This crate transitively links `fuser` -> libfuse through its +//! `hplugin`/`sandboxfuse` dependencies (the `fuse-sandbox` feature, default-on +//! at the `heph` bin and unified across the workspace by `cargo test --all`). +//! `rustc-link-arg` does NOT propagate from a dependency's build script to a +//! dependent's binaries, so every test-binary-producing crate that links fuser +//! must emit `-weak-lfuse` at its own final link or its test binary hard-links +//! `/usr/local/lib/libfuse.2.dylib` and dyld aborts at launch on hosts without +//! macFUSE — including CI runners. +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { + println!("cargo::rustc-link-arg=-weak-lfuse"); + } +} From b7a2f1ab55dd7d8e14de5c3bfe224dda7372b921 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Wed, 17 Jun 2026 02:19:38 +0200 Subject: [PATCH 43/44] ci: weak-link libfuse in plugin-echo bin + plugin-go cdylib Continuing the macOS fuse fix: with plugin-abi fixed, the next binary to abort on the macFUSE-less runner was `heph-plugin-echo` (plugin-echo's bin, launched by its subprocess test). plugin-echo and plugin-go-cdylib both transitively link fuser but lacked the `-weak-lfuse` build.rs. Add it. Critically, the cdylib must weak- link too, or `dlopen` of the shipped plugin fails on a Mac without macFUSE. Verified: no bin/test/cdylib artifact hard-links libfuse (all LC_LOAD_WEAK_DYLIB). Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-echo/build.rs | 13 +++++++++++++ crates/plugin-go-cdylib/build.rs | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 crates/plugin-echo/build.rs create mode 100644 crates/plugin-go-cdylib/build.rs diff --git a/crates/plugin-echo/build.rs b/crates/plugin-echo/build.rs new file mode 100644 index 00000000..b4fc4b10 --- /dev/null +++ b/crates/plugin-echo/build.rs @@ -0,0 +1,13 @@ +//! Weak-link libfuse on macOS (see `crates/sandboxfuse/build.rs` for the full +//! rationale). This crate's binary/cdylib transitively links `fuser` -> libfuse +//! through its hplugin/sandboxfuse deps (the default-on `fuse-sandbox` feature, +//! unified across the workspace). `rustc-link-arg` does NOT propagate from a +//! dependency's build script, so this crate must emit `-weak-lfuse` at its own +//! final link or its artifact hard-links `/usr/local/lib/libfuse.2.dylib` and +//! fails to launch/load on hosts without macFUSE — including CI runners. +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { + println!("cargo::rustc-link-arg=-weak-lfuse"); + } +} diff --git a/crates/plugin-go-cdylib/build.rs b/crates/plugin-go-cdylib/build.rs new file mode 100644 index 00000000..b4fc4b10 --- /dev/null +++ b/crates/plugin-go-cdylib/build.rs @@ -0,0 +1,13 @@ +//! Weak-link libfuse on macOS (see `crates/sandboxfuse/build.rs` for the full +//! rationale). This crate's binary/cdylib transitively links `fuser` -> libfuse +//! through its hplugin/sandboxfuse deps (the default-on `fuse-sandbox` feature, +//! unified across the workspace). `rustc-link-arg` does NOT propagate from a +//! dependency's build script, so this crate must emit `-weak-lfuse` at its own +//! final link or its artifact hard-links `/usr/local/lib/libfuse.2.dylib` and +//! fails to launch/load on hosts without macFUSE — including CI runners. +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { + println!("cargo::rustc-link-arg=-weak-lfuse"); + } +} From 1ff6df9333ae761243b1f876db9e95d87a1920b8 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Wed, 17 Jun 2026 02:40:25 +0200 Subject: [PATCH 44/44] ci: retry iceoryx2 open_or_create on transient SystemInFlux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shm cross-process test flaked on linux: `open_or_create` races another process opening/creating the same service and returns the transient `PublishSubscribeOpenOrCreateError::SystemInFlux`. Retry the service setup briefly (up to ~0.5s) instead of failing — hardens the shm byte-pipe under concurrent plugin starts too, not just the test. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/plugin-abi/src/shm.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/plugin-abi/src/shm.rs b/crates/plugin-abi/src/shm.rs index bfd0f873..c2727c97 100644 --- a/crates/plugin-abi/src/shm.rs +++ b/crates/plugin-abi/src/shm.rs @@ -61,7 +61,7 @@ pub fn connect( // All iceoryx2 handles (!Send) live on this one thread. std::thread::spawn(move || { - let ports = (|| -> anyhow::Result<_> { + let setup = || -> anyhow::Result<_> { let node = NodeBuilder::new().create::()?; // pub/sub is lossy: a full subscriber buffer drops samples. A build's // concurrent request/callback fan-out bursts hundreds of frames, so @@ -86,7 +86,19 @@ pub fn connect( .create()?; let subscriber = recv_svc.subscriber_builder().create()?; Ok((node, publisher, subscriber)) - })(); + }; + // `open_or_create` races another process opening/creating the same + // service and returns the transient `SystemInFlux`; retry briefly. + let mut ports = setup(); + for _ in 0..50 { + match &ports { + Err(e) if e.to_string().contains("InFlux") => { + std::thread::sleep(Duration::from_millis(10)); + ports = setup(); + } + _ => break, + } + } match ports { Ok((_node, publisher, subscriber)) => {