diff --git a/Cargo.lock b/Cargo.lock index 14392c506..5076711bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,6 +72,21 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" +[[package]] +name = "antithesis_sdk" +version = "0.2.9" +source = "git+https://github.com/antithesishq/antithesis-sdk-rust?rev=78c9db56771f0f6b01cb5404765c5a96852c159c#78c9db56771f0f6b01cb5404765c5a96852c159c" +dependencies = [ + "libc", + "libloading", + "linkme", + "once_cell", + "rand 0.8.5", + "rustc_version_runtime", + "serde", + "serde_json", +] + [[package]] name = "anyhow" version = "1.0.103" @@ -1978,6 +1993,14 @@ dependencies = [ "zstd", ] +[[package]] +name = "lading-antithesis" +version = "0.1.0" +dependencies = [ + "antithesis_sdk", + "serde_json", +] + [[package]] name = "lading-capture" version = "0.2.0" @@ -2148,12 +2171,42 @@ version = "0.2.184" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link 0.2.1", +] + [[package]] name = "libm" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -3425,6 +3478,16 @@ dependencies = [ "semver", ] +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + [[package]] name = "rustix" version = "0.38.44" @@ -3912,6 +3975,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 677f879de..32859f801 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "integration/sheepdog", "integration/ducks", "lading", + "lading_antithesis", "lading_capture_schema", "lading_capture", "lading_fuzz", @@ -69,6 +70,7 @@ ddsketch-agent = { git = "https://github.com/DataDog/saluki", rev = "f47a7ef588c datadog-protos = { git = "https://github.com/DataDog/saluki", rev = "f47a7ef588c53aa1da35dcfd93808595ebeb1291" } protobuf = { version = "3.7" } enum_dispatch = { version = "0.3" } +antithesis_sdk = { git = "https://github.com/antithesishq/antithesis-sdk-rust", rev = "78c9db56771f0f6b01cb5404765c5a96852c159c", default-features = false } # 0.2.8 is pinned to rand 0.8, rev version allows us to select workspace rand [workspace.lints.clippy] all = "deny" diff --git a/lading_antithesis/Cargo.toml b/lading_antithesis/Cargo.toml new file mode 100644 index 000000000..dc0ab58e8 --- /dev/null +++ b/lading_antithesis/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "lading-antithesis" +version = "0.1.0" +authors = [ + "Brian L. Troutwine ", + "George Hahn y, ...)`: a more precise assertion gives Antithesis better +exploration. diff --git a/lading_antithesis/src/lib.rs b/lading_antithesis/src/lib.rs new file mode 100644 index 000000000..d3eab98ee --- /dev/null +++ b/lading_antithesis/src/lib.rs @@ -0,0 +1,408 @@ +//! Thin facade over the Antithesis SDK. +//! +//! This crate owns the single `antithesis` feature for the project. It is no-op +//! unless specifically enabled. Each macro forwards to the SDK macro. These +//! macros convert a trailing map into JSON for consumption by the underlying +//! SDK. +//! +//! # Use Guidance +//! +//! A more precise assertion gives Antithesis better exploration. Prefer the +//! numeric macros like `always_gt!(x, y, ...)` over `always!(x > y, ...)`. + +#![deny(missing_docs)] + +/// SDK re-export. The macros forward through this path so call sites need no +/// `antithesis_sdk` dependency of their own. +#[cfg(feature = "antithesis")] +#[doc(hidden)] +pub use antithesis_sdk; +/// `serde_json::json` re-export. The macros wrap detail maps through this path +/// so call sites need no `serde_json` dependency of their own. +#[cfg(feature = "antithesis")] +#[doc(hidden)] +pub use serde_json::json; + +/// Initializes the Antithesis SDK and its assertion catalog. No-op without the +/// `antithesis` feature. +#[cfg(feature = "antithesis")] +pub fn init() { + antithesis_sdk::antithesis_init(); +} + +/// Initializes the Antithesis SDK and its assertion catalog. No-op without the +/// `antithesis` feature. +#[cfg(not(feature = "antithesis"))] +pub fn init() {} + +// Feature on. Every macro forwards to the live SDK macro at the call site. +#[cfg(feature = "antithesis")] +mod enabled { + /// Asserts `condition` holds every time this line runs, and that the line runs at least once. + #[macro_export] + macro_rules! always { + ($condition:expr, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_always!($condition, $message, &$crate::json!({ $($details)* })) + }; + ($condition:expr, $message:literal) => { + $crate::antithesis_sdk::assert_always!($condition, $message) + }; + } + + /// Asserts `condition` holds every time this line runs. Passes even if the line never runs. + #[macro_export] + macro_rules! always_or_unreachable { + ($condition:expr, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_always_or_unreachable!($condition, $message, &$crate::json!({ $($details)* })) + }; + ($condition:expr, $message:literal) => { + $crate::antithesis_sdk::assert_always_or_unreachable!($condition, $message) + }; + } + + /// Asserts `condition` holds at least once across all runs of this line. + #[macro_export] + macro_rules! sometimes { + ($condition:expr, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_sometimes!($condition, $message, &$crate::json!({ $($details)* })) + }; + ($condition:expr, $message:literal) => { + $crate::antithesis_sdk::assert_sometimes!($condition, $message) + }; + } + + /// Asserts this line is reached at least once. + #[macro_export] + macro_rules! reachable { + ($message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_reachable!($message, &$crate::json!({ $($details)* })) + }; + ($message:literal) => { + $crate::antithesis_sdk::assert_reachable!($message) + }; + } + + /// Asserts this line is never reached. + #[macro_export] + macro_rules! unreachable { + ($message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_unreachable!($message, &$crate::json!({ $($details)* })) + }; + ($message:literal) => { + $crate::antithesis_sdk::assert_unreachable!($message) + }; + } + + /// Asserts `left > right` always, with numeric guidance on the two operands. + #[macro_export] + macro_rules! always_gt { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_always_greater_than!($left, $right, $message, &$crate::json!({ $($details)* })) + }; + ($left:expr, $right:expr, $message:literal) => { + $crate::antithesis_sdk::assert_always_greater_than!($left, $right, $message) + }; + } + + /// Asserts `left >= right` always, with numeric guidance on the two operands. + #[macro_export] + macro_rules! always_ge { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_always_greater_than_or_equal_to!( + $left, $right, $message, &$crate::json!({ $($details)* }) + ) + }; + ($left:expr, $right:expr, $message:literal) => { + $crate::antithesis_sdk::assert_always_greater_than_or_equal_to!($left, $right, $message) + }; + } + + /// Asserts `left < right` always, with numeric guidance on the two operands. + #[macro_export] + macro_rules! always_lt { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_always_less_than!($left, $right, $message, &$crate::json!({ $($details)* })) + }; + ($left:expr, $right:expr, $message:literal) => { + $crate::antithesis_sdk::assert_always_less_than!($left, $right, $message) + }; + } + + /// Asserts `left <= right` always, with numeric guidance on the two operands. + #[macro_export] + macro_rules! always_le { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_always_less_than_or_equal_to!( + $left, $right, $message, &$crate::json!({ $($details)* }) + ) + }; + ($left:expr, $right:expr, $message:literal) => { + $crate::antithesis_sdk::assert_always_less_than_or_equal_to!($left, $right, $message) + }; + } + + /// Asserts `left > right` at least once, with numeric guidance on the two operands. + #[macro_export] + macro_rules! sometimes_gt { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_sometimes_greater_than!($left, $right, $message, &$crate::json!({ $($details)* })) + }; + ($left:expr, $right:expr, $message:literal) => { + $crate::antithesis_sdk::assert_sometimes_greater_than!($left, $right, $message) + }; + } + + /// Asserts `left >= right` at least once, with numeric guidance on the two operands. + #[macro_export] + macro_rules! sometimes_ge { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_sometimes_greater_than_or_equal_to!( + $left, $right, $message, &$crate::json!({ $($details)* }) + ) + }; + ($left:expr, $right:expr, $message:literal) => { + $crate::antithesis_sdk::assert_sometimes_greater_than_or_equal_to!($left, $right, $message) + }; + } + + /// Asserts `left < right` at least once, with numeric guidance on the two operands. + #[macro_export] + macro_rules! sometimes_lt { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_sometimes_less_than!($left, $right, $message, &$crate::json!({ $($details)* })) + }; + ($left:expr, $right:expr, $message:literal) => { + $crate::antithesis_sdk::assert_sometimes_less_than!($left, $right, $message) + }; + } + + /// Asserts `left <= right` at least once, with numeric guidance on the two operands. + #[macro_export] + macro_rules! sometimes_le { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_sometimes_less_than_or_equal_to!( + $left, $right, $message, &$crate::json!({ $($details)* }) + ) + }; + ($left:expr, $right:expr, $message:literal) => { + $crate::antithesis_sdk::assert_sometimes_less_than_or_equal_to!($left, $right, $message) + }; + } + + /// Asserts at least one of the named conditions always holds, with per-name guidance. + #[macro_export] + macro_rules! always_some { + ({ $($conditions:tt)* }, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_always_some!({ $($conditions)* }, $message, &$crate::json!({ $($details)* })) + }; + ({ $($conditions:tt)* }, $message:literal) => { + $crate::antithesis_sdk::assert_always_some!({ $($conditions)* }, $message) + }; + } + + /// Asserts every named condition holds at least once, with per-name guidance. + #[macro_export] + macro_rules! sometimes_all { + ({ $($conditions:tt)* }, $message:literal, { $($details:tt)* }) => { + $crate::antithesis_sdk::assert_sometimes_all!({ $($conditions)* }, $message, &$crate::json!({ $($details)* })) + }; + ({ $($conditions:tt)* }, $message:literal) => { + $crate::antithesis_sdk::assert_sometimes_all!({ $($conditions)* }, $message) + }; + } +} + +// Feature off. Every macro is a no-op that elides its arguments unevaluated. +#[cfg(not(feature = "antithesis"))] +mod disabled { + /// Asserts `condition` holds every time this line runs, and that the line runs at least once. + #[macro_export] + macro_rules! always { + ($condition:expr, $message:literal, { $($details:tt)* }) => {{}}; + ($condition:expr, $message:literal) => {{}}; + } + + /// Asserts `condition` holds every time this line runs. Passes even if the line never runs. + #[macro_export] + macro_rules! always_or_unreachable { + ($condition:expr, $message:literal, { $($details:tt)* }) => {{}}; + ($condition:expr, $message:literal) => {{}}; + } + + /// Asserts `condition` holds at least once across all runs of this line. + #[macro_export] + macro_rules! sometimes { + ($condition:expr, $message:literal, { $($details:tt)* }) => {{}}; + ($condition:expr, $message:literal) => {{}}; + } + + /// Asserts this line is reached at least once. + #[macro_export] + macro_rules! reachable { + ($message:literal, { $($details:tt)* }) => {{}}; + ($message:literal) => {{}}; + } + + /// Asserts this line is never reached. + #[macro_export] + macro_rules! unreachable { + ($message:literal, { $($details:tt)* }) => {{}}; + ($message:literal) => {{}}; + } + + /// Asserts `left > right` always, with numeric guidance on the two operands. + #[macro_export] + macro_rules! always_gt { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => {{}}; + ($left:expr, $right:expr, $message:literal) => {{}}; + } + + /// Asserts `left >= right` always, with numeric guidance on the two operands. + #[macro_export] + macro_rules! always_ge { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => {{}}; + ($left:expr, $right:expr, $message:literal) => {{}}; + } + + /// Asserts `left < right` always, with numeric guidance on the two operands. + #[macro_export] + macro_rules! always_lt { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => {{}}; + ($left:expr, $right:expr, $message:literal) => {{}}; + } + + /// Asserts `left <= right` always, with numeric guidance on the two operands. + #[macro_export] + macro_rules! always_le { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => {{}}; + ($left:expr, $right:expr, $message:literal) => {{}}; + } + + /// Asserts `left > right` at least once, with numeric guidance on the two operands. + #[macro_export] + macro_rules! sometimes_gt { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => {{}}; + ($left:expr, $right:expr, $message:literal) => {{}}; + } + + /// Asserts `left >= right` at least once, with numeric guidance on the two operands. + #[macro_export] + macro_rules! sometimes_ge { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => {{}}; + ($left:expr, $right:expr, $message:literal) => {{}}; + } + + /// Asserts `left < right` at least once, with numeric guidance on the two operands. + #[macro_export] + macro_rules! sometimes_lt { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => {{}}; + ($left:expr, $right:expr, $message:literal) => {{}}; + } + + /// Asserts `left <= right` at least once, with numeric guidance on the two operands. + #[macro_export] + macro_rules! sometimes_le { + ($left:expr, $right:expr, $message:literal, { $($details:tt)* }) => {{}}; + ($left:expr, $right:expr, $message:literal) => {{}}; + } + + /// Asserts at least one of the named conditions always holds, with per-name guidance. + #[macro_export] + macro_rules! always_some { + ({ $($conditions:tt)* }, $message:literal, { $($details:tt)* }) => {{}}; + ({ $($conditions:tt)* }, $message:literal) => {{}}; + } + + /// Asserts every named condition holds at least once, with per-name guidance. + #[macro_export] + macro_rules! sometimes_all { + ({ $($conditions:tt)* }, $message:literal, { $($details:tt)* }) => {{}}; + ({ $($conditions:tt)* }, $message:literal) => {{}}; + } +} + +// Expansion guards. A `macro_rules!` body is only type-checked once expanded, +// and nothing in the workspace calls these macros yet, so without a call site +// the forwarding paths into the SDK are never checked. Each macro is invoked +// here in both arities so the compiler resolves every path under whichever +// feature state is being built. +// +// Literals only, no local bindings: with the feature off the macros elide their +// arguments unevaluated, so a binding passed to one would trip +// `unused_variables`, which the workspace denies. +#[cfg(test)] +mod tests { + #[test] + fn boolean_macros_expand() { + crate::always!(1 > 0, "always holds"); + crate::always!(1 > 0, "always holds with details", { "detail": 1 }); + crate::always_or_unreachable!(1 > 0, "always holds or never runs"); + crate::always_or_unreachable!(1 > 0, "always holds or never runs with details", { "detail": 1 }); + crate::sometimes!(1 > 0, "holds at least once"); + crate::sometimes!(1 > 0, "holds at least once with details", { "detail": 1 }); + } + + #[test] + fn reachability_macros_expand() { + crate::reachable!("this line is reached"); + crate::reachable!("this line is reached with details", { "detail": 1 }); + } + + // `unreachable!` is exercised for expansion only. Reaching it reports a + // failed assertion to the SDK handler, which is inert outside an Antithesis + // run, and these unit tests never execute inside one. + #[test] + fn unreachable_macro_expands() { + if false { + crate::unreachable!("never reached"); + } + if false { + crate::unreachable!("never reached with details", { "detail": 1 }); + } + } + + #[test] + fn numeric_always_macros_expand() { + crate::always_gt!(2, 1, "left exceeds right"); + crate::always_gt!(2, 1, "left exceeds right with details", { "detail": 1 }); + crate::always_ge!(1, 1, "left at least right"); + crate::always_ge!(1, 1, "left at least right with details", { "detail": 1 }); + crate::always_lt!(1, 2, "left below right"); + crate::always_lt!(1, 2, "left below right with details", { "detail": 1 }); + crate::always_le!(1, 1, "left at most right"); + crate::always_le!(1, 1, "left at most right with details", { "detail": 1 }); + } + + #[test] + fn numeric_sometimes_macros_expand() { + crate::sometimes_gt!(2, 1, "left exceeds right at least once"); + crate::sometimes_gt!(2, 1, "left exceeds right at least once with details", { "detail": 1 }); + crate::sometimes_ge!(1, 1, "left at least right at least once"); + crate::sometimes_ge!(1, 1, "left at least right at least once with details", { "detail": 1 }); + crate::sometimes_lt!(1, 2, "left below right at least once"); + crate::sometimes_lt!(1, 2, "left below right at least once with details", { "detail": 1 }); + crate::sometimes_le!(1, 1, "left at most right at least once"); + crate::sometimes_le!(1, 1, "left at most right at least once with details", { "detail": 1 }); + } + + #[test] + fn condition_map_macros_expand() { + crate::always_some!({ lower: 1 > 0, upper: 2 > 1 }, "some named condition holds"); + crate::always_some!( + { lower: 1 > 0, upper: 2 > 1 }, + "some named condition holds with details", + { "detail": 1 } + ); + crate::sometimes_all!({ lower: 1 > 0, upper: 2 > 1 }, "each named condition holds once"); + crate::sometimes_all!( + { lower: 1 > 0, upper: 2 > 1 }, + "each named condition holds once with details", + { "detail": 1 } + ); + } + + #[test] + fn init_is_callable() { + crate::init(); + } +} diff --git a/test/antithesis/scenarios/general/.launch/docker-compose.yaml b/test/antithesis/scenarios/general/.launch/docker-compose.yaml new file mode 100644 index 000000000..91baa1e82 --- /dev/null +++ b/test/antithesis/scenarios/general/.launch/docker-compose.yaml @@ -0,0 +1,90 @@ +name: lading +services: + lading: + build: + context: /home/ssm-user/src/lading + dockerfile: test/antithesis/scenarios/general/Dockerfile + target: lading + container_name: lading + depends_on: + sink: + condition: service_healthy + required: true + environment: + CONFIG_DIR: /shared + NO_COLOR: "1" + RUST_LOG: info + hostname: lading + image: lading:82f962dc-dirty + init: true + networks: + default: null + platform: linux/amd64 + volumes: + - type: volume + source: shared + target: /shared + read_only: true + volume: {} + - type: volume + source: capture + target: /capture + volume: {} + sink: + build: + context: /home/ssm-user/src/lading + dockerfile: test/antithesis/scenarios/general/Dockerfile + target: sink + container_name: sink + environment: + NO_COLOR: "1" + hostname: sink + healthcheck: + test: + - CMD-SHELL + - bash -c 'exec 3<>/dev/tcp/localhost/9000' + timeout: 2s + interval: 2s + retries: 30 + image: sink:82f962dc-dirty + init: true + networks: + default: null + platform: linux/amd64 + workload: + build: + context: /home/ssm-user/src/lading + dockerfile: test/antithesis/scenarios/general/Dockerfile + target: workload + container_name: workload + depends_on: + sink: + condition: service_healthy + required: true + environment: + CONFIG_DIR: /shared + NO_COLOR: "1" + hostname: workload + image: workload:82f962dc-dirty + init: true + networks: + default: null + platform: linux/amd64 + volumes: + - type: volume + source: shared + target: /shared + volume: {} + - type: volume + source: capture + target: /capture + read_only: true + volume: {} +networks: + default: + name: lading_default +volumes: + capture: + name: lading_capture + shared: + name: lading_shared diff --git a/test/antithesis/scratchbook/deployment-topology.md b/test/antithesis/scratchbook/deployment-topology.md new file mode 100644 index 000000000..68d9edc18 --- /dev/null +++ b/test/antithesis/scratchbook/deployment-topology.md @@ -0,0 +1,289 @@ +--- +sut_path: /home/ssm-user/src/lading +commit: 51148899 +updated: 2026-07-24T21:28:13Z +external_references: + - name: the deployment (production runner-orchestrator, generic name) + why: real shutdown/deploy model — RJO launches lading as an ECR container in a + docker/podman pod, uses --target-container observer mode (not Binary target), + never sends lading a signal (self-termination on the experiment timer is the + only graceful stop), tears everything down with `docker rm --force` (SIGKILL), + and enforces bounded shutdown with a (warmup+samples+30)*1.2 watchdog. Defines + what "correct shutdown" means and which faults are live vs moot in production. + - name: Jira (SMPTNG project) + Confluence + why: existing bug tickets and design docs — SMPTNG-725 "RJO alive but not really" + (hang-in-spin after "lading shutdown"), SMPTNG-719/697 (ungraceful-termination + telemetry loss, graceful-termination gap Won't-Fixed), SMPTNG-694 (captures now + zstd-compressed — revisit the "valid jsonl prefix" assumption), SMPTNG-390 + + #1911/#1895 (blackhole traffic recorder crash-consistency), SMPTNG-762/767 + (SUT wall-clock/entropy classes mirroring lading's determinism ADR). + - name: whole lading repo (this checkout) + why: the SUT source spot-verified against current main; every property below is a + checkable invariant whose violation is a real lading defect (file:line evidence + in the catalog). +--- + +# Antithesis deployment topology and scenarios + +Minimal set of Antithesis topologies that cover `catalog.scenarios_needed`. Source +of truth is the property catalog JSON + discovery digest; nothing here is invented. +Each topology lists its containers, its fault profile, and the catalog property +slugs whose violation it would catch. Shutdown/termination safety is the immediate +priority (P0); the rest of the system is catalogued behind it. + +Reused container roles (same code, different wiring per topology): + +- **sink** — TCP byte-counting oracle, SDK-linked, never faulted. Owns + `sink-receives-bytes` (`sometimes! total>0`). Already built (`test/antithesis/sink`). +- **lading** — the instrumented SUT (sancov, `panic=abort`; a panic → SIGABRT caught + as the panic hook's `unreachable! "lading panicked"`). Config chosen per timeline. +- **workload** — driver + checker host. Emits `setup_complete`, writes the sampled + `lading.yaml` via `first_sample_config`, and runs `anytime_capture_consistent` over + the capture volume. Never faulted, so it survives to validate. +- **target** — (Binary/observer topologies only) the process lading supervises or + observes. +- **load-source** — (blackhole topology only) a sender that drives traffic at a + lading blackhole. + +Volumes: `shared` (workload→lading config handoff), `capture` (named volume so the +capture survives a `node_termination` of lading for the checker). + +--- + +## A. `general` — node-fault generator (EXISTS) + +**Containers:** sink + lading(`--no-target`, TCP generator) + workload. +**Faults:** `node_termination` (SIGKILL) on `lading` only; cpu/clock jitter on lading. +**Config:** `--experiment-duration-infinite`, `--capture-path /capture/capture.jsonl`, +`--capture-flush-seconds 1`. + +Justification: SIGKILL of the whole lading container is the exact Antithesis analogue +of the deployment's `docker rm --force` teardown and of a mid-run watchdog kill. It is +the only fault that is *live* in production (SIGTERM never reaches lading there). The +1s flush + named `capture` volume maximise kill-in-flight opportunities and preserve a +fresh file for the checker. + +Proves / guards: +- `node-termination-mid-run` → `jsonl-prefix-valid-after-kill` (torn-record-free, + monotonic prefix), `capture-no-fsync-durability` (open question: does node kill + preserve page cache to the named volume?). +- `no-panic-anywhere` umbrella (panic hook), `sink-receives-bytes` non-vacuity. +- Config variation at the workload seam (`first_sample_config` value menu) rides this + topology with **no new containers** and covers: `small-and-zero-block-size` + (`block-cache-construction-terminates`, `block-cache-zero-max-no-panic`), + `multi-generator-config` (`per-generator-semaphore-no-panic`, + `splunk-hec-response-parse-no-panic`), `malformed-config` + (`generator-addr-uri-validation-no-panic`, `config-numeric-fields-validated`, + `dogstatsd-tag-length-validated`), and the throttle/rate scenarios + (`oversized-block-vs-capacity`, `linear-ramp-parallel`, + `throttle-divide-no-silent-underdelivery`, `grpc-honors-throttle`, + `discarded-blocks-counted`, `error-label-cardinality-bounded`) using the sink as + the delivered-rate oracle. + +Note: the throttle rate-fidelity items (`throttle-divide-*`, `linear-ramp-*`, +`stable-burst-envelope-bounded`) are also provable as **pure proptests** against the +real `Valve` with no Antithesis rig — prefer that where possible; use this topology +only for the end-to-end sink-rate check. + +--- + +## B. `graceful-timer` — lading owns the clock, exits 0 (NEW) + +**Containers:** sink + lading + workload (same as A). +**Faults:** NO `node_termination` on lading. Clock jitter / cpu contention only +(Antithesis controls time to stress the timer and the drain). +**Config:** finite `--experiment-duration-seconds` + `--warmup-duration-seconds`, +`--max-shutdown-delay 30`, `--capture-format parquet` (matches the deployment's +`system.yaml`), `--capture-path /capture/capture.parquet`. + +Justification: this is the *only* graceful stop that exists in the deployment — lading +self-terminates on its experiment timer, drains the maturity window, writes the parquet +footer, exits 0, and RJO copies the capture off before force-killing the pod. Removing +the node fault is deliberate: this scenario exists to prove the happy path is reachable +and bounded, which the SIGKILL scenario (A) can never observe. + +Proves / guards: +- `lading-completes-and-exits-cleanly` (Sometimes / non-vacuity: at least one timeline + reaches exit 0 with a footer-complete capture). +- `parquet-footer-on-graceful-exit` (Always: readable parquet after graceful exit) via + the checker's parquet arm. +- `shutdown-completes-bounded` (Always: exit within `max_shutdown_delay`; the + deployment watchdog makes this operational), `capture-finalize-bounded`. +- `determinism-replay` / `payload-determinism-byte-identical` / + `no-wall-clock-in-payloads`: run two lading instances (or two runs) with the **same + seed+config**; assert byte-identical bytes at the sink, and identical payload + timestamps under a perturbed clock. This is where Antithesis clock control earns its + keep (analogue of SMPTNG-762/767). + +--- + +## C. `unreachable-sink` — connect-loop shutdown responsiveness (NEW) + +**Containers:** sink + lading + workload — but a **network partition** between lading +and sink (Antithesis network fault), or sink held down, so the generator's target +address/socket/uri/container never becomes reachable. +**Faults:** network unreachability on the lading→sink edge; finite experiment duration. +**Variants (config only):** tcp / udp / unix_stream / unix_datagram / grpc generators, +and (with the docker socket present) a misnamed `--target-container`. + +Justification: confirmed-live on main — the TCP connect loop (`tcp.rs:230-251`), the +unix_datagram (`:246-264`) and grpc (`:287-299`) connect loops, and the container +discovery loop (`target.rs:212-244`) all lack a shutdown branch. With an unreachable +peer the worker spins `connect → sleep → continue` forever and never reaches the +`select!` that polls shutdown, so the experiment timer's expiry cannot end the run. The +oracle is external: measure shutdown-signal → process-exit ≤ `max_shutdown_delay`. + +Proves / guards: `shutdown-completes-bounded` (umbrella) and its per-loop children +`tcp-connect-loop-shutdown-responsive`, `unix-datagram-connect-loop-shutdown`, +`grpc-connect-loop-shutdown`, `docker-target-discovery-bounded`. + +--- + +## D. `stalled-receiver` — backpressure/busy-spin under a slow peer (NEW) + +**Containers:** sink + lading + workload, with the **sink modified to accept then +never read** (fills the socket buffer) and to respond slowly on HTTP paths. +**Faults:** none required beyond the slow-receiver behaviour; finite duration; observe +CPU. + +Justification: `unix_stream.rs:281-318` busy-spins on `WouldBlock` via `yield_now` with +no shutdown branch (the source comment admits "if the read side has hung up we will +never know"); http acquires all connection permits on a slow target. Assert bounded +shutdown **and** no core pegged at 100%. + +Proves / guards: `unix-stream-partial-write-shutdown`, `unix-stream-write-error-progress`, +`throttle-capacity-no-zerodelivery-livelock`, and the http backpressure lead. + +--- + +## E. `sigterm-stop` — orchestrator-stop graceful path (NEW, P0) + +**Containers:** sink + lading + workload; lading and workload share a PID namespace +(or workload has a docker exec seam) so the workload can `kill -TERM` lading mid-run. +**Faults:** an explicit `kill -TERM` step from the workload (Antithesis delivers only +SIGKILL via node_termination, so SIGTERM must be an explicit harness action). + +Justification: main has **no `SignalKind::terminate` arm** (`lading.rs:658` only +`ctrl_c`), so today a SIGTERM kills lading non-gracefully (exit 143, no drain, no +footer, no child reap). This is MOOT on the production path (RJO never signals lading) +but is the canonical orchestrator-stop model and is directly tied to SMPTNG-719/697/725. +Only an explicit `kill -TERM` exercises it. + +Proves / guards: `sigterm-graceful-drain` (readable parquet + non-abort exit after +SIGTERM), `parquet-footer-on-graceful-exit`, `orphaned-children-on-signal-death` +(no reparented target/inspector grandchildren afterward), `target-grace-period-honored`. + +--- + +## F. `binary-target` — process supervisor + observer no-panic (NEW, P0) + +**Containers:** sink + lading + **target** + workload. lading runs **without** +`--no-target`: config spawns the `target` process (`target.binary`) so lading is the +supervisor/observer. The `target` container image ships a small program with selectable +behaviours (value-menu): normal, exits-immediately, exits-before-sending-PID, +ignores-SIGTERM, forks-a-grandchild, rapid-exit-then-PID-reuse. +**Faults:** `node_termination` on lading; the target-behaviour selector is the primary +stressor; clock jitter. + +Justification: the observer/Binary-target defects (untimed `target_child.wait()` at +`target.rs:432`, the `expect("catastrophic failure")` on the PID channel at +`observer.rs:114-120`, `assert!(cur_pid==pid)` at `stat.rs:82`, the `panic!` at +`process_descendents.rs:13`, cpu.max parse, kill_on_drop-on-signal orphaning) are only +reachable when lading actually supervises a process. Production uses observer mode +against a sibling container and does NOT exercise these, so this topology is where they +must be hunted. Each malicious target behaviour is one row of the value menu. + +Proves / guards: `target-wait-bounded`, `observer-target-pid-recv-no-panic`, +`observer-pid-reuse-no-panic`, `observer-process-vanish-no-panic`, +`observer-cpu-max-parse-no-panic`, `observer-transient-read-not-fatal`, +`observer-pid-identity-fingerprint`, `orphaned-children-on-signal-death`, +`target-grace-period-honored`. (Covers `target-lifecycle-races` from +`scenarios_needed`.) + +--- + +## G. `blackhole` — lading as sink under fault (NEW) + +**Containers:** **load-source** (a sender — a second lading generator or a minimal +traffic client) + lading configured as a **blackhole** + workload. +**Faults:** fd exhaustion / transient `accept()` errors on the blackhole; a stale +unix_datagram socket left on a persisted volume then a blackhole restart; a throttled +capture-manager drain (slow-capture fault). + +Justification: confirmed-live — the datadog blackhole's fallible `Ok((stream,_)) = +accept()` select arm (`datadog.rs:209`) wedges permanently on the first accept error +(silently backpressuring the target); the unix_datagram blackhole never awaits its +`remove_file` future (`unix_datagram.rs:95`) so a stale socket makes `bind` fail +EADDRINUSE on restart; the datadog record path blocks the HTTP response on a full +bounded capture channel. The oracle is the load-source's request success/latency (the +blackhole must never backpressure it) plus a bind-succeeds assertion on restart. + +Proves / guards: `datadog-blackhole-accept-resilient`, +`unix-datagram-blackhole-removes-stale-socket`, `blackhole-never-backpressures-target`, +`sqs-receive-message-bounded`, `tcp-rr-listener-no-panic` (with `threads>1` and a +pre-bound data port — the `tcp_rr-threads>1-bind-conflict` scenario), +`recorded-traffic-crash-consistency`. + +--- + +## H. `capture-write-fault` — capture IO error is a clean exit, not SIGABRT (NEW) + +**Containers:** sink + lading + workload (as A), with a **disk-full / EIO fault +injected on the `capture` volume** on a flush tick. +**Faults:** IO fault on the capture path; finite duration. + +Justification: `lading.rs:476/501/526` start the capture manager with +`.block_on(start()).expect(...)` under `panic=abort`, so a flush-tick IO error today +SIGABRTs the whole run mid-flush and skips `format.close()`, leaving an unreadable +parquet. Assert: the panic hook stays silent AND the parquet is footer-complete/readable +(clean fatal exit, not abort). + +Proves / guards: `capture-write-failure-not-abort`, `capture-finalize-bounded`, +`multi-format-parquet-not-forfeited`, `capture-histogram-drops-counted`, +`capture-drift-no-silent-gap` (with a >60s scheduling-starvation fault). + +--- + +## I. `split-mode` — two-instance capture merge under partial kill (NEW, P2) + +**Containers:** sink + **lading-sender** (`--no-target`) + **lading-receiver** +(`--target-container` observer) + target + workload (runs the oblivious merge). +**Faults:** `node_termination` on exactly ONE lading instance. + +Justification: mirrors the deployment's split mode where sender and receiver run on +separate pods and their parquet files are merged obliviously; a truncated/footerless +file from a killed side fails the merge. Assert the clean side's captures are not +needlessly discarded and failure attribution is well-defined (open question from the +digest). + +Proves / guards: `split-mode-merge-partial-tolerant`. + +--- + +## Coverage of `scenarios_needed` + +| scenarios_needed item | topology | +| --- | --- | +| unreachable-target | C | +| stalled-receiver | D | +| multi-generator-config | A (config seam) | +| small-and-zero-block-size | A (config seam) | +| oversized-block-vs-capacity | A (config seam) + proptest | +| linear-ramp-parallel | A (config seam) + proptest | +| determinism-replay | B | +| node-termination-mid-run | A | +| sigterm-stop | E | +| capture-write-fault | H | +| target-lifecycle-races | F | +| blackhole-restart-stale-socket | G | +| accept-fault-injection | G | +| slow-capture-drain | G / H | +| malformed-config | A (config seam) | +| split-mode-partial-kill | I | +| tcp_rr-threads>1-bind-conflict | G | + +Minimality note: A and B share an identical container graph and differ only in fault +profile + duration; C/D/E/H are A's container graph plus one fault or one modified +peer. Only F (adds `target`), G (adds `load-source`, lading-as-blackhole), and I (two +lading instances) introduce genuinely new topology. Build order should follow priority: +E and F are P0 shutdown/no-panic and should land right after the existing A. diff --git a/test/antithesis/scratchbook/evaluation/synthesis.md b/test/antithesis/scratchbook/evaluation/synthesis.md new file mode 100644 index 000000000..c3887478d --- /dev/null +++ b/test/antithesis/scratchbook/evaluation/synthesis.md @@ -0,0 +1,246 @@ +--- +sut_path: /home/ssm-user/src/lading +commit: 51148899 +updated: 2026-07-24T21:28:12Z +external_references: + - name: the deployment + why: real shutdown/deploy model (production runner + local dev); how lading is launched, stopped, watchdog-timed, and how captures are collected — establishes which properties are MOOT vs LIVE + - name: Jira/Confluence (datadoghq.atlassian.net, project SMPTNG) + why: existing bug tickets and design docs — SMPTNG-725/719/697 (ungraceful-termination telemetry loss, hang-in-spin), SMPTNG-694 (compressed captures), SMPTNG-762/767 (entropy/wall-clock determinism analogs), SMPTNG-390 (traffic recorder) + - name: whole lading repo + why: source of truth for every property's evidence line/file; spot-verified against current main +--- + +# lading Antithesis portfolio evaluation + +Source of truth: the property catalog (51 properties across no-panic, lifecycle/shutdown, +capture, generators/throttle, observer, blackhole/config, payload/determinism) plus the +discovery digest (deployment shutdown model, Jira/Confluence leads, git fix-history +regressions, per-subsystem source findings). Nothing here is invented; every claim traces +to a catalog `slug` or a digest finding. + +The catalog is a strong bug-hunting portfolio: every property is a checkable invariant whose +violation is a real defect, and many encode LIVE regressions on `main` (the hardening fixes +live only on backup/`blt` branches). The gaps below are not in the *properties* — they are in +the **harness oracles and scenarios** that would actually exercise and observe them, and in a +handful of properties that are structurally vacuous or moot under the Antithesis fault model +as currently configured. + +--- + +## 1. Coverage gaps (properties with no oracle / no scenario today) + +Per the SDK-instrumentation inventory, the SUT binary carries only two real SDK sites (facade +init + panic hook at `antithesis_hooks.rs:27`, and a bootstrap `reachable!` at `lading.rs:792`). +All domain invariants live in `test/antithesis/`, and today that is only: + +- `sink/main.rs:82` — `sometimes! (total>0)` load-arrival oracle. +- `harness/.../anytime_capture_consistent.rs` — jsonl torn/monotonic + parquet-readable-is-consistent (`always!`), plus `sometimes!` non-vacuity floors (MIN_RECORDS=10). +- `harness/.../first_sample_config.rs:36` — per-timeline config-menu anchor. +- The panic hook (covers the entire `no-panic` subsystem indirectly). + +Everything else in the catalog has **no oracle wired**. Concrete gaps, grouped by what is missing: + +### 1a. No shutdown-latency / liveness oracle (highest-value gap) +No external timer measures shutdown-signal -> process-exit, and no `reachable!`/`always!` +marks the bounded-exit branch of any connect/retry loop. Consequently these P0 properties are +**completely unchecked**: +- `shutdown-completes-bounded` (umbrella), `tcp-connect-loop-shutdown-responsive`, + `unix-stream-partial-write-shutdown`, `unix-datagram-connect-loop-shutdown`, + `grpc-connect-loop-shutdown`, `target-wait-bounded`, `docker-target-discovery-bounded`, + `capture-finalize-bounded`. +- Needed scenarios (per catalog `scenarios_needed`): **unreachable-target**, **stalled-receiver**. +- Note: several of these connect-loop hangs are confirmed LIVE on main (tcp.rs:230-283 has no + shutdown branch in the connect leg; unix_datagram.rs:246-264; grpc.rs:287-299; + unix_stream.rs:281-318 busy-yields; target.rs:212-244 docker discovery has no shutdown/timeout arm). + +### 1b. No CPU / livelock observation +`unix-stream-partial-write-shutdown`, `throttle-capacity-no-zerodelivery-livelock`, +`unix-stream-write-error-progress` all hinge on detecting a 100%-CPU busy-spin delivering +~zero bytes. The harness has no CPU/throughput observer. Scenarios **oversized-block-vs-capacity** +and **stalled-receiver** are unimplemented. + +### 1c. No rate-fidelity oracle +The sink counts only "bytes > 0", not *rate*. So every rate invariant is unchecked: +`throttle-divide-no-silent-underdelivery`, `linear-ramp-slope-preserved`, +`grpc-honors-throttle`, `unix-throttle-aggregate-consistent`, `divide-by-zero-startup-error`, +`discarded-blocks-counted`. Catalog flags divide/linear-ramp as **pure proptests** provable +against the real Valve with no rig — cheapest win. Scenarios **oversized-block-vs-capacity**, +**linear-ramp-parallel** unimplemented. + +### 1d. No determinism oracle +No `always!` on a rolling hash of emitted blocks and no two-seed byte-equality replay. +`payload-determinism-byte-identical` and `no-wall-clock-in-payloads` are unchecked. Scenario +**determinism-replay** (incl. perturbed-clock timestamp equality) unimplemented. Jira leads +SMPTNG-762 (CPU-jitter entropy) / SMPTNG-767 (non-monotonic wall clock) are the SUT-analog bug +classes to hunt. + +### 1e. No memory / label-cardinality observation +`error-label-cardinality-bounded` and `sqs-receive-message-bounded` need a memory/key-count +observer + a flapping-target / adversarial-request scenario. None exists. + +### 1f. No blackhole-fault scenarios +`datadog-blackhole-accept-resilient`, `blackhole-never-backpressures-target`, +`unix-datagram-blackhole-removes-stale-socket` need accept-fault-injection, slow-capture-drain, +and blackhole-restart-stale-socket scenarios. None exists. (All three are confirmed-live defects: +datadog.rs:207-212 fallible-select wedge; datadog per-point `send().await` before response; +unix_datagram.rs:95 remove_file future never awaited.) + +### 1g. No target-lifecycle-race scenario +`observer-pid-reuse-no-panic`, `observer-process-vanish-no-panic`, +`observer-cpu-max-parse-no-panic`, `observer-target-pid-recv-no-panic`, +`observer-transient-read-not-fatal`, `observer-pid-identity-fingerprint` need +target-exits/PID-recycle/malformed-cgroup fault injection. The panic hook would catch the +aborts, but the **races are never induced**, so the properties are vacuously green. + +### 1h. No malformed-config scenario +`generator-addr-uri-validation-no-panic`, `block-cache-zero-max-no-panic`, +`config-numeric-fields-validated`, `tcp-rr-listener-no-panic`, `per-generator-semaphore-no-panic`, +`splunk-hec-response-parse-no-panic` need the **malformed-config**, **small-and-zero-block-size**, +**multi-generator-config**, **tcp_rr-threads>1-bind-conflict** scenarios plus a non-JSON blackhole body. + +### 1i. Split-mode and durability blind spots +`split-mode-merge-partial-tolerant`, `recorded-traffic-crash-consistency`, +`capture-no-fsync-durability` all depend on open questions about the deployment (persisted +named-volume page-cache retention; zstd framing of compressed captures per SMPTNG-694; oblivious +merge attribution). No scenario **split-mode-partial-kill** / **node-termination-mid-run** targets these. + +--- + +## 2. Moot under SIGKILL (Antithesis `node_termination` cannot exercise these) + +Antithesis node_termination = SIGKILL of the whole container (untrappable), and the digest +confirms **the deployment never sends lading SIGTERM** — production runs lading in Container +observer mode and tears down via `docker rm --force` (immediate SIGKILL). So lading's +SIGTERM/SIGINT graceful path is effectively dead code in production, and these graceful-only or +Binary-target-only properties **cannot be violated by the Antithesis SIGKILL fault** as configured: + +| Property | Why moot under SIGKILL | Where it is still LIVE | +|---|---|---| +| `sigterm-graceful-drain` | SIGTERM untrappable; SIGKILL skips the graceful arm entirely | Deployment orchestrator-stop path (needs an explicit `kill -TERM` harness step) | +| `target-wait-bounded` | Target is a sibling container force-killed by the runner, not a lading child | Binary-target mode only | +| `target-grace-period-honored` | No SIGTERM delivered to lading; target is disposable | Binary-target mode with a cooperative slow target | +| `orphaned-children-on-signal-death` | Whole-container SIGKILL kills all pids together | Shared-PID-namespace / bare-host deployments | +| `capture-no-fsync-durability` | Only decidable if node_termination preserves page-cache to the persisted volume — **open question**; likely undecidable in-sim | Real VM/instance power-off on a persisted volume | + +Action: to test the graceful contract at all, the harness must add an explicit +**sigterm-stop** step (send `kill -TERM` to lading and validate footer-complete parquet + clean +exit). Without it, `sigterm-graceful-drain`, `parquet-footer-on-graceful-exit` (graceful arm), +`target-grace-period-honored`, and `orphaned-children-on-signal-death` are never exercised. + +--- + +## 3. Vacuous without a graceful exit or a target/sink + +These pass **trivially (vacuously)** unless the precondition is arranged. They are correct +invariants but give false confidence until the trigger is guaranteed on some timeline. + +### 3a. Vacuous without a *graceful* exit +The parquet footer is written *only* in `handle_shutdown -> format.close()` +(state_machine.rs:249-259; parquet.rs:307-324). If no timeline reaches a graceful shutdown, +every parquet-readability assertion is vacuous: +- `parquet-footer-on-graceful-exit` — needs a graceful exit to produce a readable footer at all. +- `capture-finalize-bounded` — the finalize await only runs on the graceful path. +- `lading-completes-and-exits-cleanly` (Sometimes / non-vacuity anchor) — **this is the guard**: + it asserts the good path is reachable on >=1 timeline. If it never fires, the whole + graceful/capture family is silently vacuous. Treat its `sometimes!` as a coverage tripwire. +- `capture-write-failure-not-abort` — vacuous unless a capture-write IO fault is actually injected + (scenario **capture-write-fault**). + +The `anytime_capture_consistent` checker already handles this correctly by design: the parquet +arm is `!readable || invariants_hold` (vacuous on unreadable/killed files, which is intended), +and `sometimes! (readable && records>=10)` guards against parquet *never* being finalized. + +### 3b. Vacuous without a target/sink receiving load +All `needs_target: true` rate/delivery/observability properties are vacuous if no sink receives +bytes or no target is observed: +- `sink-receives-bytes` is the load-arrival non-vacuity guard for the entire generator/throttle family. +- `throttle-divide-no-silent-underdelivery`, `linear-ramp-slope-preserved`, `grpc-honors-throttle`, + `unix-throttle-aggregate-consistent`, `divide-by-zero-startup-error`, `discarded-blocks-counted`, + `throttle-capacity-no-zerodelivery-livelock`, `unix-stream-write-error-progress`, + `error-label-cardinality-bounded` — all require delivered load to be measurable. +- Observer family (`observer-*`) is vacuous unless a target process exists and is sampled. + +Cross-check: `throttle-divide-no-silent-underdelivery` and `linear-ramp-slope-preserved` escape +this vacuity because they are provable as **pure proptests against the real Valve** — no rig, no +target needed. Prioritize those (they need no scenario wiring at all). + +--- + +## 4. Which properties need SUT fixes (live on main) + +The catalog `needs_sut_fix` field is authoritative. The hardening fixes referenced in git +history live on backup/`blt` branches, **not main**, so the following are LIVE regressions the +harness should be able to catch (and which need code changes to *pass*): + +### P0 SUT fixes — shutdown safety +- `sigterm-graceful-drain` — add `SignalKind::terminate` arm to the `lading.rs:658` select. +- `shutdown-completes-bounded` (umbrella) — shutdown branches on every pre-select connect/retry loop + timeout on `target_child.wait()` and capture-finalize await. +- `tcp-connect-loop-shutdown-responsive` — wrap connect in `select!` with `shutdown_wait` (tcp.rs:230-251). +- `unix-stream-partial-write-shutdown` — shutdown branch on the `blk_offset exit, asserting `<= max_shutdown_delay` (30s); plus a `sometimes!`/`reachable!` at every connect-loop bounded-exit branch. Unblocks the entire §1a P0 set. +2. **Add the `unreachable-target` scenario** (generator points at an address/socket/uri/container that never binds). Directly exercises `tcp/udp/unix_stream/unix_datagram/grpc-connect-loop-shutdown` and `docker-target-discovery-bounded` — all confirmed LIVE hangs on main. +3. **Add an explicit `sigterm-stop` harness step** (`kill -TERM lading`) so `sigterm-graceful-drain`, `parquet-footer-on-graceful-exit`, `target-grace-period-honored`, `orphaned-children-on-signal-death` stop being moot (§2). Assert footer-complete parquet + clean exit + no orphans. +4. **Guard non-vacuity**: ensure `lading-completes-and-exits-cleanly` (`sometimes!`) and the parquet `sometimes!(readable && records>=10)` arm actually fire in triage; if not, the whole capture family is silently vacuous. + +### Phase 1 — Shutdown-safety SUT fixes (P0, land on main) +Apply the §4 P0 shutdown fixes: SIGTERM handler; shutdown branches on all connect/retry loops; bounded `target_child.wait()`; bounded docker discovery; bounded capture-finalize; capture-write-error -> graceful exit (not abort). Strong deployment lead: SMPTNG-725 "RJO alive but not really" (hang-in-spin after "lading shutdown") and SMPTNG-719/697 (ungraceful-termination telemetry loss) corroborate these as observed-in-production classes. + +### Phase 2 — No-panic race scenarios (P0) +Add `target-lifecycle-races` (target exits before PID / vanishes mid-listing / PID recycled) and `capture-write-fault` (disk-full/EIO on a flush tick). Panic hook already the oracle; these scenarios *induce* the races so the observer no-panic quartet + `capture-write-failure-not-abort` stop being vacuous. Land the corresponding stat.rs:82 / process_descendents.rs:13 / observer.rs:114 fixes. + +### Phase 3 — Rate fidelity (P0, cheap) +Land `throttle-divide-no-silent-underdelivery` and `linear-ramp-slope-preserved` as **pure proptests against the real Valve** (no rig, no target — no vacuity risk). Then add a sink *rate* oracle + `oversized-block-vs-capacity` and `linear-ramp-parallel` scenarios for `grpc-honors-throttle`, `throttle-capacity-no-zerodelivery-livelock`, `discarded-blocks-counted`. + +### Phase 4 — Determinism (P1) +Add `determinism-replay` (two seeded runs, byte-identical at sink or block-cache hash) and a perturbed-clock timestamp-equality check. Hunt the SMPTNG-762/767 entropy/wall-clock classes inside lading. No SUT fix expected; guards the determinism ADR. + +### Phase 5 — Blackhole & config faults (P1) +`accept-fault-injection` (datadog wedge, fd exhaustion), `slow-capture-drain` (datadog per-point backpressure), `blackhole-restart-stale-socket` (unix_datagram remove_file), `malformed-config` / `small-and-zero-block-size` / `multi-generator-config` / `tcp_rr-threads>1-bind-conflict`. Land the matching §4 P1 fixes. + +### Phase 6 — Durability, split-mode, cardinality (P2, resolve open questions first) +`node-termination-mid-run` (jsonl prefix + parquet consistency — largely covered by the existing checker), `split-mode-partial-kill`, and cardinality/OOM (`error-label-cardinality-bounded`, `sqs-receive-message-bounded`). Blocked on deployment open questions: does node_termination preserve page-cache to the persisted volume? are compressed captures (SMPTNG-694 zstd) a valid prefix when truncated? does a sender-only overrun fail a clean receiver's replicate? + +### Open questions to resolve against the deployment before Phase 6 +- Does Antithesis ever deliver SIGTERM, or only SIGKILL? (Determines whether §2 items are reachable at all in-sim vs only via an explicit harness `kill -TERM`.) +- Does node_termination preserve OS page-cache to the persisted named volume? (P2 vs P1 for `capture-no-fsync-durability`.) +- Are compressed capture streams (post-SMPTNG-694) a valid decodable prefix after truncation, or undecodable past the last flushed frame? (Reframes `jsonl-prefix-valid-after-kill` / `recorded-traffic-crash-consistency`.) +- Is the unix socket path on a persisted volume? (Impact of `unix-datagram-blackhole-removes-stale-socket`.) +- Is per-connection (undivided) throttle intentional for unix_stream? (`unix-throttle-aggregate-consistent`.) +- Is `tcp_rr` ever run with threads>1? (`tcp-rr-listener-no-panic`.) diff --git a/test/antithesis/scratchbook/existing-assertions.md b/test/antithesis/scratchbook/existing-assertions.md new file mode 100644 index 000000000..e8a6c0311 --- /dev/null +++ b/test/antithesis/scratchbook/existing-assertions.md @@ -0,0 +1,100 @@ +--- +sut_path: /home/ssm-user/src/lading +commit: 51148899 +updated: 2026-07-24T21:28:09Z +external_references: + - name: the deployment (production runner + local dev harness) + why: real shutdown/deploy model — how lading is launched, stopped, and how capture completeness is defined operationally; the source of truth for which lading termination paths are actually exercised. + - name: Jira / Confluence (datadoghq.atlassian.net, project SMPTNG) + why: existing bug tickets and design docs (SMPTNG-725 hang-in-spin, SMPTNG-719/697 ungraceful-termination telemetry loss, SMPTNG-694 compressed captures, SMPTNG-390 blackhole recorder, determinism/DST docs). + - name: the whole lading repo (this checkout) + why: the SUT itself — generators, throttle, capture, observer, target/inspector lifecycle, blackholes, and the lading_antithesis facade over antithesis_sdk. +--- + +# Existing Antithesis SDK assertions + +Source of truth: the discovery-digest "Antithesis SDK instrumentation inventory" area +plus the property catalog. This file inventories every Antithesis SDK assertion that +**already exists** in the tree at commit `51148899`. It does not propose new ones. + +## Summary + +- The lading **SUT binary** (`lading/src`) carries **only bootstrap/plumbing probes** — + there are **NO domain-level `always!`/`sometimes!`/`reachable!` invariants** in the + production source beyond one bootstrap `reachable!` and the panic hook. +- All **domain invariants live in the workload** (`test/antithesis/`), not in the SUT: + the sink load-arrival oracle and the capture crash-consistency checker. +- The primary no-panic enforcement is indirect: the panic hook reports any SUT panic as + an `unreachable!` violation before `panic = "abort"` aborts the container. + +## SUT-side sites (`lading/src`) — bootstrap / plumbing only + +| Location | Kind | Message | Notes | +|---|---|---|---| +| `lading/src/antithesis_hooks.rs:27` | `unreachable!` (panic hook) | `"lading panicked"` details `{message, location}` | **The** no-panic ADR bridge. `set_hook` wraps the default hook, downcasts the payload to `&str`/`String`, reports to Antithesis, then forwards to the previous hook. Any reachable panic in lading = an Antithesis-visible `unreachable!` failure. `panic = "abort"` (Cargo.toml:115,120) makes every hit a hard, observable process abort. | +| `lading/src/antithesis_hooks.rs:14` | `init()` call | — | Forwards to `lading_antithesis::init()`, which calls `antithesis_sdk::antithesis_init()` under the `antithesis` feature (no-op otherwise). Installs the panic hook as early as possible. | +| `lading/src/bin/lading.rs:741` | `init()` call site | — | Sole caller of `lading::antithesis_hooks::init()`; runs in `main()` before the tokio runtime is built. | +| `lading/src/bin/lading.rs:792` | `reachable!` | `"lading completed bootstrap"` (no details map) | Bootstrap probe proving the SDK is linked and the instrumentation path is wired. Fires in `main()` after arg parsing, before the runtime is built. | + +### The facade + +- `lading_antithesis` is the single project facade over `antithesis_sdk` (owns the sole + `antithesis` cargo feature). It exports the full macro family: `always!`, + `always_or_unreachable!`, `sometimes!`, `reachable!`, `unreachable!`, + `always_gt/ge/lt/le!`, `sometimes_gt/ge/lt/le!`, `always_some!`, `sometimes_all!` + (`lading_antithesis/src/lib.rs:38-322`). Each macro has an enabled arm forwarding to + the SDK and a disabled no-op arm that elides its args unevaluated. Details maps are + wrapped via `serde_json::json`. + +### NOT Antithesis instrumentation (do not confuse) + +Three `std`-library `unreachable!` sites in `lading/src` are core `unreachable!` macros, +not SDK assertions (they carry no `lading_antithesis::` prefix). They are panic-on-reach +guards subject to the no-panic ADR and would be caught by the panic hook if hit: +- `lading/src/bin/lading.rs:347` (clap one-of guarantee) +- `lading/src/generator/splunk_hec/acknowledgements.rs:110` (`Channel::Ack` arm) +- `lading/src/blackhole/otlp/http.rs:227` and `:258` (path already validated) + +## Workload-side sites (`test/antithesis/`) — all domain invariants + +These are the real oracles. They run in workload/checker containers, not the SUT. + +### Sink load-arrival oracle +| Location | Kind | Message | Notes | +|---|---|---|---| +| `test/antithesis/.../sink/src/main.rs:82` | `sometimes!` `(total > 0)` | `"sink received bytes"` | Load-arrival non-vacuity, fired per-connection read in the never-faulted sink container. Guards against a whole config class delivering nothing (divide stall, capacity livelock, throttle bypass). SDK init at `sink/main.rs:32`. | + +### Capture crash-consistency checker (`anytime_capture_consistent.rs`) +`test/antithesis/harness/src/bin/anytime_capture_consistent.rs` — five SDK sites plus a +reachable anchor. `MIN_RECORDS = 10` non-vacuity floor; runs in the workload container. +| Line | Kind | Condition | Message | +|---|---|---|---| +| 44 | `always!` | `torn_before_final == 0` | `"jsonl capture has no torn record before the final line"` | +| 49 | `always!` | `invariants_hold` | `"jsonl capture fetch_index and per-series time stay monotonic"` | +| 54 | `sometimes!` | `parsed >= 10` | `"jsonl capture accumulated records across the run"` | +| 66 | `always!` | `!readable \|\| invariants_hold` | `"readable parquet capture is internally consistent"` | +| 71 | `sometimes!` | `readable && records >= 10` | `"parquet capture finalized and readable across the run"` | +| 82 | `reachable!` | (only when `checked_any`) | `"capture consistency checker validated a capture file"` | + +Encodes the capture crash-consistency ADR: jsonl torn-final tolerated, parquet +unreadable-after-kill tolerated but a *readable* parquet must be internally consistent. + +### Config-menu anchor +| Location | Kind | Message | Notes | +|---|---|---|---| +| `test/antithesis/harness/src/bin/first_sample_config.rs:36` | `reachable!` | `"first_sample_config sampled a config"` details `{variant}` | Per-timeline config-menu anchor. Draws from `AntithesisRng` (post-`setup_complete`) so Antithesis branches each config pick; counting these in triage shows how many variants were explored. This is the one intentional facade bypass — it names `antithesis_sdk::random::AntithesisRng` directly (`first_sample_config.rs:25`), since the facade covers only assertion macros + `init()`, not the RNG. | + +## Coverage gaps in existing assertions (actionable) + +Nothing beyond the bootstrap probes exists in the SUT source, so these P0/high-value +paths are currently observable **only** indirectly (via the panic hook or the external +capture checker), never asserted directly: +- **Shutdown/termination safety** — no SDK assertion instruments SIGTERM handling, + `target_child.wait()` timeouts, `kill_on_drop`-on-signal, or the connect-loop hangs + (tcp/udp/unix_stream/unix_datagram/grpc). These P0 invariants are visible only as a + panic-hook `unreachable!` or an unreadable capture at the external checker. +- **Determinism** — no `always!` expresses "same seed => byte-identical load"; the + harness relies on the sink byte counter and config sampling, not a direct byte-equality + oracle. +- Whether the `antithesis` feature is compiled into the scenario build (Dockerfile) — so + the enabled-arm macros actually link in — was not verified in the source scan. diff --git a/test/antithesis/scratchbook/plans/2026-07-27-broad-coverage-and-adversarial-receiver.md b/test/antithesis/scratchbook/plans/2026-07-27-broad-coverage-and-adversarial-receiver.md new file mode 100644 index 000000000..e68081d3b --- /dev/null +++ b/test/antithesis/scratchbook/plans/2026-07-27-broad-coverage-and-adversarial-receiver.md @@ -0,0 +1,806 @@ +# Broad Config Coverage + Adversarial Receiver Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the `general` Antithesis scenario randomly cover most of lading's config surface and target-interaction failure modes, so config/startup-crash bugs and target-misbehavior bugs surface without per-bug scenarios. + +**Architecture:** Two evolutions of the existing `general` scenario. (1) The per-timeline config sampler (`harness/src/config.rs`) grows from a single fixed `tcp` generator into a multi-generator sampler whose value menus include boundary values that crash lading at startup; the existing panic hook turns any such crash into an `unreachable!` finding. (2) A new `adversarial-receiver` component (separate from the pure `sink` oracle) speaks the HTTP protocols lading generates and returns a per-timeline sampled adversarial response (2xx / 429-503 / stall / reset / garbage-200), so target-interaction bugs surface. A SUT-side assertion on the `trace_agent` backoff loop catches the retry storm regardless of shutdown masking, and `shutdown-safety` reuses the sampler so the storm's drain impact is measured under the existing bounded-drain oracle. + +**Tech Stack:** Rust 2024, tokio, hyper 1.x (server), the Antithesis SDK via the `lading_antithesis` facade, `serde_yaml`, Docker Compose, snouty. + +## Global Constraints + +- US-ASCII only in all code, comments, and docs. +- No panics in harness/receiver code: return `Result`, no `unwrap`/`expect`/`panic!`. (The sampler deliberately emits configs that crash *lading* at runtime -- that is data the SUT mishandles, not a panic in harness code.) +- No `std::collections::HashMap`; use `IndexMap`/`IndexSet` if a map is needed. +- Workspace clippy is deny-all + pedantic + `unwrap_used`; every crate sets `[lints] workspace = true`. +- TDD: write the failing test first, watch it fail, implement minimally, watch it pass, commit. +- No Jira/SMPTNG keys anywhere in source, including assertion messages. +- Antithesis assertion messages are self-documenting full statements of the property. +- Structured sampler choices draw from `AntithesisRng` (so Antithesis branches them); lading's payload `seed` draws from system entropy, never from `AntithesisRng`. +- Determinism: the sampler is a pure function of its `rng`; no wall-clock or ambient randomness beyond the documented `seed` draw. +- Do not commit or launch; hold for the user. + +## File Structure + +- `test/antithesis/adversarial-receiver/` (NEW crate) -- the adversarial HTTP receiver. `src/mode.rs` holds the pure, unit-tested behavior selection; `src/main.rs` is the hyper server wiring. Split so the decision logic is testable without a socket. +- `test/antithesis/harness/src/config.rs` (MODIFY) -- grows a `Generator` enum, a `TimelinePlan`, `sample_plan`, and a multi-generator `to_yaml`. +- `test/antithesis/harness/src/bin/first_sample_config.rs` (MODIFY) -- writes the receiver mode alongside `lading.yaml`. +- `test/antithesis/scenarios/general/` (MODIFY) -- `Dockerfile` gains an `adversarial-receiver` stage/target; `docker-compose.yaml` gains the service; `lading-entrypoint.sh` unchanged. +- `lading/src/generator/trace_agent.rs` (MODIFY) -- SUT-side assertion on the backoff loop. +- `test/antithesis/scenarios/shutdown-safety/` (MODIFY) -- sample the stressor config instead of hardwiring one. +- `Cargo.toml` (workspace) (MODIFY) -- add the `adversarial-receiver` member. + +--- + +## Part 1 -- Adversarial receiver component + +### Task 1: Receiver mode logic (pure, tested) + +**Files:** +- Create: `test/antithesis/adversarial-receiver/Cargo.toml` +- Create: `test/antithesis/adversarial-receiver/src/mode.rs` +- Create: `test/antithesis/adversarial-receiver/src/lib.rs` +- Modify: `Cargo.toml` (workspace `members`) + +**Interfaces:** +- Produces: `enum Mode { Ok, Retryable, Stall, Reset, GarbageOk }`; `Mode::from_env_or_file(default: Mode) -> Mode`; `Mode::parse(&str) -> Mode` (unknown -> `Ok`); `Mode::status(self) -> u16` (`Ok`/`Stall`/`GarbageOk` -> 200, `Retryable` -> 503); `Mode::body(self) -> &'static [u8]`. + +- [ ] **Step 1: Add the crate to the workspace members** in `Cargo.toml` (workspace), alphabetically near the other `test/antithesis/*` members: + +```toml + "test/antithesis/adversarial-receiver", +``` + +- [ ] **Step 2: Write `Cargo.toml` for the crate** + +```toml +[package] +name = "adversarial-receiver" +version = "0.1.0" +edition = "2024" +license = "MIT" +publish = false +description = "Adversarial HTTP receiver for the lading Antithesis general scenario." + +[[bin]] +name = "adversarial-receiver" +path = "src/main.rs" + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +lading-antithesis = { workspace = true, features = ["antithesis"] } +anyhow = { workspace = true } +tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "time"] } +hyper = { workspace = true, features = ["server", "http1"] } +hyper-util = { workspace = true, features = ["tokio", "server"] } +http-body-util = { workspace = true } +bytes = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } + +[dev-dependencies] +proptest = { workspace = true } +``` + +(If `hyper`/`hyper-util`/`http-body-util`/`bytes` are not yet `[workspace.dependencies]`, they are already used by `lading`; add them to the workspace `[workspace.dependencies]` mirroring lading's versions.) + +- [ ] **Step 3: Write `src/lib.rs`** + +```rust +//! Adversarial HTTP receiver: pure behavior selection plus the server binary. +//! +//! The `mode` module is the decision logic, unit-tested without a socket; the +//! binary (`main.rs`) wires it to hyper. +pub mod mode; +``` + +- [ ] **Step 4: Write the failing test in `src/mode.rs`** + +```rust +//! Per-timeline adversarial response behavior for the receiver. +//! +//! The mode is chosen once per timeline by the workload's config sampler and +//! written to the shared volume; the receiver reads it and applies it to every +//! request. Keeping selection pure makes it testable without a socket. + +/// The adversarial behavior the receiver applies to every request this timeline. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mode { + /// Answer 200 with an empty body. The benign baseline. + Ok, + /// Answer 503 on every request. Drives retry/backoff paths. + Retryable, + /// Accept, then never respond. Drives client stall/timeout paths. + Stall, + /// Drop the connection without any response. Drives reset handling. + Reset, + /// Answer 200 with a body that is not any expected schema. Drives strict + /// response-body parsing paths. + GarbageOk, +} + +impl Mode { + /// Parse a mode token; any unknown token is the benign `Ok`. + #[must_use] + pub fn parse(s: &str) -> Self { + match s.trim() { + "retryable" => Self::Retryable, + "stall" => Self::Stall, + "reset" => Self::Reset, + "garbage_ok" => Self::GarbageOk, + _ => Self::Ok, + } + } + + /// HTTP status for a mode that returns a response. + #[must_use] + pub fn status(self) -> u16 { + match self { + Self::Retryable => 503, + Self::Ok | Self::Stall | Self::GarbageOk => 200, + } + } + + /// Response body for a mode that returns a body. + #[must_use] + pub fn body(self) -> &'static [u8] { + match self { + Self::GarbageOk => b"not-a-known-schema", + _ => b"", + } + } +} + +#[cfg(test)] +mod tests { + use super::Mode; + + #[test] + fn parse_known_and_unknown() { + assert_eq!(Mode::parse("retryable"), Mode::Retryable); + assert_eq!(Mode::parse("stall"), Mode::Stall); + assert_eq!(Mode::parse("reset"), Mode::Reset); + assert_eq!(Mode::parse("garbage_ok"), Mode::GarbageOk); + assert_eq!(Mode::parse("ok"), Mode::Ok); + assert_eq!(Mode::parse("whatever"), Mode::Ok, "unknown -> benign"); + assert_eq!(Mode::parse(" retryable\n"), Mode::Retryable, "trimmed"); + } + + #[test] + fn retryable_is_503_others_200() { + assert_eq!(Mode::Retryable.status(), 503); + assert_eq!(Mode::Ok.status(), 200); + assert_eq!(Mode::GarbageOk.status(), 200); + } + + #[test] + fn garbage_ok_has_nonempty_body() { + assert!(!Mode::GarbageOk.body().is_empty()); + assert!(Mode::Ok.body().is_empty()); + } +} +``` + +- [ ] **Step 5: Run the test, expect FAIL (crate does not build yet / no bin)** + +Run: `cargo test -p adversarial-receiver --lib` +Expected: FAIL (missing `src/main.rs` for the `[[bin]]`, or unresolved deps). + +- [ ] **Step 6: Add a minimal `src/main.rs` stub so the crate builds, then re-run** + +```rust +//! Placeholder; real server wired in Task 2. +fn main() {} +``` + +Run: `cargo test -p adversarial-receiver --lib` +Expected: PASS (3 tests). + +- [ ] **Step 7: Commit** + +```bash +git add Cargo.toml test/antithesis/adversarial-receiver +git commit -m "feat(antithesis): adversarial-receiver mode logic" +``` + +### Task 2: Receiver server (hyper wiring + mode source) + +**Files:** +- Modify: `test/antithesis/adversarial-receiver/src/mode.rs` (add `from_env_or_file`) +- Modify: `test/antithesis/adversarial-receiver/src/main.rs` + +**Interfaces:** +- Consumes: `Mode` from Task 1. +- Produces: a binary listening on `RECEIVER_LISTEN_ADDR` (default `0.0.0.0:8080`) that reads its mode from `RECEIVER_MODE_PATH` (default `/shared/receiver_mode`) at startup, falling back to `Ok` if absent, and applies it to every request. + +- [ ] **Step 1: Add `from_env_or_file` to `mode.rs` with a failing test** + +```rust +// add near the bottom of the impl block: + /// Resolve the timeline's mode from a file path (one token). A missing or + /// unreadable file yields the benign `Ok`, so the receiver serves normally + /// until the sampler has written a mode. + #[must_use] + pub fn from_file(path: &std::path::Path) -> Self { + match std::fs::read_to_string(path) { + Ok(s) => Self::parse(&s), + Err(_) => Self::Ok, + } + } +``` + +```rust +// add to `mod tests`: + #[test] + fn from_file_missing_is_ok() { + let p = std::path::Path::new("/nonexistent/receiver_mode"); + assert_eq!(Mode::from_file(p), Mode::Ok); + } +``` + +- [ ] **Step 2: Run, expect FAIL then PASS after adding the method** + +Run: `cargo test -p adversarial-receiver --lib` +Expected: PASS (4 tests). + +- [ ] **Step 3: Write the server in `src/main.rs`** (mirror the hyper 1.x serving boilerplate in `lading/src/blackhole/http.rs`: `hyper_util::server::conn::auto` or `hyper::server::conn::http1` + `TokioIo`, one task per accepted connection, `tokio::select!` on `ctrl_c`) + +```rust +//! Adversarial HTTP receiver binary for the lading Antithesis general scenario. +//! +//! Reads its timeline mode from the shared volume and answers every request +//! per that mode: 200, 503, stall (never respond), reset (drop), or a 200 with +//! an off-schema body. It is a supporting harness component, not the system +//! under test, so it is built without coverage instrumentation. + +use std::{convert::Infallible, net::SocketAddr, path::PathBuf, sync::Arc}; + +use adversarial_receiver::mode::Mode; +use anyhow::Context as _; +use bytes::Bytes; +use http_body_util::Full; +use hyper::{Request, Response, StatusCode, body::Incoming, service::service_fn}; +use hyper_util::rt::TokioIo; +use tokio::net::TcpListener; +use tracing::{error, info}; +use tracing_subscriber::EnvFilter; + +const DEFAULT_LISTEN_ADDR: &str = "0.0.0.0:8080"; +const DEFAULT_MODE_PATH: &str = "/shared/receiver_mode"; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .init(); + lading_antithesis::init(); + + let addr: SocketAddr = std::env::var("RECEIVER_LISTEN_ADDR") + .unwrap_or_else(|_| DEFAULT_LISTEN_ADDR.to_string()) + .parse() + .context("parse RECEIVER_LISTEN_ADDR")?; + let mode_path: PathBuf = std::env::var_os("RECEIVER_MODE_PATH") + .map_or_else(|| PathBuf::from(DEFAULT_MODE_PATH), PathBuf::from); + let mode = Mode::from_file(&mode_path); + info!("adversarial-receiver listening on {addr} in mode {mode:?}"); + + let listener = TcpListener::bind(addr).await.context("bind receiver")?; + let mode = Arc::new(mode); + + loop { + tokio::select! { + accepted = listener.accept() => { + let (stream, _peer) = match accepted { Ok(v) => v, Err(e) => { error!("accept: {e}"); continue; } }; + let mode = Arc::clone(&mode); + tokio::spawn(async move { + // Reset: drop the freshly accepted connection with no response. + if *mode == Mode::Reset { return; } + let io = TokioIo::new(stream); + let svc = service_fn(move |req| handle(req, *mode)); + if let Err(e) = hyper::server::conn::http1::Builder::new().serve_connection(io, svc).await { + error!("serve_connection: {e}"); + } + }); + } + _ = tokio::signal::ctrl_c() => { info!("shutdown, receiver exiting"); break; } + } + } + Ok(()) +} + +async fn handle(_req: Request, mode: Mode) -> Result>, Infallible> { + // The claim: lading actually reached the receiver with a request. + lading_antithesis::sometimes!(true, "adversarial receiver received a request"); + if mode == Mode::Stall { + // Accept the request and never answer, exercising client stall/timeout. + std::future::pending::<()>().await; + } + let status = StatusCode::from_u16(mode.status()).unwrap_or(StatusCode::OK); + let resp = Response::builder() + .status(status) + .body(Full::new(Bytes::from_static(mode.body()))) + .unwrap_or_else(|_| Response::new(Full::new(Bytes::new()))); + Ok(resp) +} +``` + +(Note: the two `unwrap_or_else`/`unwrap_or` above are total -- they cannot panic because `mode.status()` is always a valid code and the body builder cannot fail on a static body -- but they are written as fallbacks to satisfy `unwrap_used`. If clippy still flags them, replace with explicit `match`.) + +- [ ] **Step 4: Build and smoke-test locally** + +```bash +cargo build -p adversarial-receiver +RECEIVER_MODE_PATH=/dev/null RECEIVER_LISTEN_ADDR=127.0.0.1:18080 target/debug/adversarial-receiver & +sleep 1 +curl -s -o /dev/null -w '%{http_code}\n' http://127.0.0.1:18080/ # expect 200 +printf 'retryable' > /tmp/rmode; kill %1 +RECEIVER_MODE_PATH=/tmp/rmode RECEIVER_LISTEN_ADDR=127.0.0.1:18080 target/debug/adversarial-receiver & +sleep 1 +curl -s -o /dev/null -w '%{http_code}\n' http://127.0.0.1:18080/ # expect 503 +kill %1 +``` + +Expected: `200` then `503`. + +- [ ] **Step 5: clippy + commit** + +```bash +cargo clippy -p adversarial-receiver --all-targets +git add test/antithesis/adversarial-receiver +git commit -m "feat(antithesis): adversarial-receiver hyper server" +``` + +### Task 3: Wire the receiver into the general scenario + +**Files:** +- Modify: `test/antithesis/scenarios/general/Dockerfile` +- Modify: `test/antithesis/scenarios/general/docker-compose.yaml` + +- [ ] **Step 1: Add a build + runtime stage to the Dockerfile.** In `tools-builder`, add the receiver build alongside sink/harness: + +```dockerfile + cargo build --release --package adversarial-receiver --bin adversarial-receiver && \ + cp /tools/target/release/adversarial-receiver /usr/local/bin/adversarial-receiver && \ +``` + +Add a runtime target after the `sink` target: + +```dockerfile +FROM debian:bookworm-slim AS adversarial-receiver +ENV NO_COLOR=1 +RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates && rm -rf /var/lib/apt/lists/* +COPY --from=tools-builder /usr/local/bin/adversarial-receiver /usr/local/bin/adversarial-receiver +EXPOSE 8080 +ENTRYPOINT ["/usr/local/bin/adversarial-receiver"] +``` + +- [ ] **Step 2: Add the service to `docker-compose.yaml`** (reads the shared volume for its mode; healthchecked so lading waits for it): + +```yaml + adversarial-receiver: + container_name: adversarial-receiver + hostname: adversarial-receiver + platform: linux/amd64 + init: true + build: + context: ../../../.. + dockerfile: test/antithesis/scenarios/general/Dockerfile + target: adversarial-receiver + image: adversarial-receiver:${ANTITHESIS_IMAGE_TAG:-latest} + environment: + NO_COLOR: "1" + volumes: + - shared:/shared:ro + healthcheck: + test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/localhost/8080'"] + interval: 2s + timeout: 2s + retries: 30 +``` + +Add `adversarial-receiver: {condition: service_healthy}` to lading's and workload's `depends_on`. + +- [ ] **Step 3: Build the scenario, expect success** + +Run: `docker compose -f test/antithesis/scenarios/general/docker-compose.yaml build` +Expected: all four images build. + +- [ ] **Step 4: Commit** + +```bash +git add test/antithesis/scenarios/general/Dockerfile test/antithesis/scenarios/general/docker-compose.yaml +git commit -m "feat(antithesis): add adversarial-receiver to general scenario" +``` + +--- + +## Part 2 -- Broaden the config sampler + +### Task 4: Multi-generator plan type + `to_yaml` + +**Files:** +- Modify: `test/antithesis/harness/src/config.rs` +- Modify: `test/antithesis/harness/Cargo.toml` (add `hyper` for `Uri`/`Method`, `http` for `HeaderMap` -- both already workspace deps via lading) + +**Interfaces:** +- Produces: `enum Generator { Tcp(tcp::Config), TraceAgent(trace_agent::Config), Http(http::Config) }`; `struct TimelinePlan { generators: Vec, receiver_mode: &'static str, label: String }`; `fn sample_plan(rng: &mut R) -> TimelinePlan`; `fn plan_to_yaml(generators: &[Generator]) -> Result`. + +- [ ] **Step 1: Add deps to `harness/Cargo.toml`** + +```toml +hyper = { workspace = true } +http = { workspace = true } +``` + +- [ ] **Step 2: Write the failing test first** (append to `config.rs` tests) -- the load-bearing invariant is unchanged: everything sampled must parse as a lading config, across many seeds, for every generator kind the menu can produce. + +```rust + #[test] + fn every_sampled_plan_parses_as_lading_config() { + for s in 0..1024_u64 { + let mut rng = StdRng::seed_from_u64(s); + let plan = super::sample_plan(&mut rng); + let yaml = super::plan_to_yaml(&plan.generators).expect("serialize plan"); + let parsed: Result = serde_yaml::from_str(&yaml); + assert!(parsed.is_ok(), "seed {s} rejected:\n{yaml}\n{:?}", parsed.err()); + } + } + + #[test] + fn menu_reaches_every_generator_kind_and_boundary() { + // Prove the crash/interaction boundaries are actually reachable, so the + // scenario is not silently benign. Requires the trace_agent contexts=0 + // crash config, the duplicate-http config, and the retryable mode to all + // appear across a seed sweep. + let mut saw_ctx0 = false; + let mut saw_dup_http = false; + let mut saw_retryable = false; + for s in 0..2048_u64 { + let mut rng = StdRng::seed_from_u64(s); + let plan = super::sample_plan(&mut rng); + if plan.label.contains("trace_agent:contexts0") { saw_ctx0 = true; } + if plan.generators.iter().filter(|g| matches!(g, super::Generator::Http(_))).count() >= 2 { saw_dup_http = true; } + if plan.receiver_mode == "retryable" { saw_retryable = true; } + } + assert!(saw_ctx0, "contexts=0 boundary never sampled"); + assert!(saw_dup_http, "duplicate-http boundary never sampled"); + assert!(saw_retryable, "retryable receiver mode never sampled"); + } +``` + +- [ ] **Step 3: Run, expect FAIL (symbols missing)** + +Run: `cargo test -p harness --lib` +Expected: FAIL (no `sample_plan`, `plan_to_yaml`, `Generator`). + +- [ ] **Step 4: Implement the plan types and `plan_to_yaml`** (replace the tcp-only `to_yaml`; keep `sample` for now or delete once `first_sample_config` switches in Task 6). Add at the top of `config.rs`: + +```rust +use lading::generator::{http, tcp, trace_agent}; +use lading_payload::trace_agent as ta_payload; +use lading_payload::common::config::ConfRange; + +/// One sampled lading generator, typed against lading's own config structs so +/// the value menu cannot drift from the real schema. +pub enum Generator { + Tcp(tcp::Config), + TraceAgent(trace_agent::Config), + Http(http::Config), +} + +impl Generator { + fn key(&self) -> &'static str { + match self { + Generator::Tcp(_) => "tcp", + Generator::TraceAgent(_) => "trace_agent", + Generator::Http(_) => "http", + } + } + fn value(&self) -> Result { + match self { + Generator::Tcp(c) => serde_yaml::to_value(c), + Generator::TraceAgent(c) => serde_yaml::to_value(c), + Generator::Http(c) => serde_yaml::to_value(c), + } + } +} + +/// The full per-timeline plan: the generator set lading boots under, plus the +/// adversarial mode the receiver serves this timeline. +pub struct TimelinePlan { + pub generators: Vec, + pub receiver_mode: &'static str, + pub label: String, +} + +/// Serialize a generator set into the `generator: [ { key: value }, ... ]` YAML +/// lading parses. +/// +/// # Errors +/// Returns an error if a generator config fails to serialize. +pub fn plan_to_yaml(generators: &[Generator]) -> Result { + let mut seq = Vec::with_capacity(generators.len()); + for g in generators { + let mut item = serde_yaml::Mapping::new(); + item.insert(serde_yaml::Value::from(g.key()), g.value()?); + seq.push(serde_yaml::Value::Mapping(item)); + } + let mut top = serde_yaml::Mapping::new(); + top.insert(serde_yaml::Value::from("generator"), serde_yaml::Value::Sequence(seq)); + serde_yaml::to_string(&serde_yaml::Value::Mapping(top)) +} +``` + +(Confirm the `ConfRange` import path with `grep -rn "pub enum ConfRange" lading_payload/src`; adjust the `use` if it differs.) + +- [ ] **Step 5: Implement `sample_plan`** (see Task 5 for the generator builders it calls). Minimal body to compile: + +```rust +const RECEIVER_ADDR: &str = "adversarial-receiver:8080"; + +fn payload_seed() -> [u8; 32] { + let mut seed = [0u8; 32]; + rand::rng().fill_bytes(&mut seed); + seed +} + +#[must_use] +pub fn sample_plan(rng: &mut R) -> TimelinePlan { + // Pick the generator family for this timeline. tcp -> sink (byte oracle); + // trace_agent/http -> adversarial-receiver (interaction oracle). + let kind = rng.random_range(0..4_u8); + let receiver_mode = ["ok", "ok", "retryable", "stall", "garbage_ok", "reset"] + .choose(rng).copied().unwrap_or("ok"); + match kind { + 0 => sample_tcp_plan(rng), + 1 => sample_trace_agent_plan(rng, receiver_mode), + 2 => sample_http_plan(rng, receiver_mode), + _ => sample_dup_http_plan(rng, receiver_mode), + } +} +``` + +- [ ] **Step 6: Run, expect FAIL until Task 5 adds the builders; then the parse + reachability tests pass** + +Run: `cargo test -p harness --lib` +Expected after Task 5: PASS. + +- [ ] **Step 7: Commit** (after Task 5). + +### Task 5: Generator builders with boundary menus + +**Files:** +- Modify: `test/antithesis/harness/src/config.rs` + +**Interfaces:** +- Consumes: `Generator`, `TimelinePlan`, `RECEIVER_ADDR`, `payload_seed()`, the existing `variant_menu()`. +- Produces: `fn sample_tcp_plan`, `fn sample_trace_agent_plan`, `fn sample_http_plan`, `fn sample_dup_http_plan`. + +- [ ] **Step 1: Implement `sample_tcp_plan`** (the existing tcp logic, now returning a plan against the sink; unchanged menus): + +```rust +fn sample_tcp_plan(rng: &mut R) -> TimelinePlan { + let variant = variant_menu().choose(rng).cloned().unwrap_or(payload::Config::Ascii); + let bps_mib = [1_u64, 5, 10, 50, 100].choose(rng).copied().unwrap_or(10); + let bps = bps_mib * 1024 * 1024; + let parallel_connections = rng.random_range(1..=8_u16); + let per_conn = bps / u64::from(parallel_connections); + let cfg = tcp::Config { + seed: payload_seed(), + addr: "sink:9000".to_string(), + variant, + bytes_per_second: Some(byte_unit::Byte::from_u64(bps)), + maximum_block_size: byte_unit::Byte::from_u64(per_conn.clamp(1, 1024 * 1024)), + maximum_prebuild_cache_size_bytes: byte_unit::Byte::from_u64(8 * 1024 * 1024), + parallel_connections, + throttle: None, + }; + TimelinePlan { generators: vec![Generator::Tcp(cfg)], receiver_mode: "ok", + label: format!("tcp:{}", variant_label(&variant)) } +} +``` + +- [ ] **Step 2: Implement `sample_trace_agent_plan`** -- the menu includes the `contexts=0` crash boundary (#4) and the `Ignore` backoff (#2): + +```rust +fn sample_trace_agent_plan(rng: &mut R, receiver_mode: &'static str) -> TimelinePlan { + // contexts=0 is the empty-range crash boundary; the rest are benign. + let (contexts, ctx_label) = match rng.random_range(0..3_u8) { + 0 => (ConfRange::Constant(0), "contexts0"), + 1 => (ConfRange::Constant(1), "contexts1"), + _ => (ConfRange::Inclusive { min: 1, max: 50 }, "contextsN"), + }; + let variant = ta_payload::Config::V04(ta_payload::v04::Config { contexts, ..Default::default() }); + let (backoff, bo_label) = if rng.random_bool_ratio(1, 2) { + (trace_agent::BackoffBehavior::Ignore, "ignore") + } else { + (trace_agent::BackoffBehavior::Obey { max_retries: 3 }, "obey") + }; + let uri = format!("http://{RECEIVER_ADDR}").parse().unwrap_or_else(|_| hyper::Uri::from_static("http://adversarial-receiver:8080")); + let cfg = trace_agent::Config { + seed: payload_seed(), + target_uri: uri, + backoff_behavior: backoff, + variant, + bytes_per_second: Some(byte_unit::Byte::from_u64(1024 * 1024)), + maximum_block_size: byte_unit::Byte::from_u64(1024 * 1024), + maximum_prebuild_cache_size_bytes: byte_unit::Byte::from_u64(8 * 1024 * 1024), + block_cache_method: lading_payload::block::CacheMethod::Fixed, + parallel_connections: rng.random_range(1..=4_u16), + throttle: None, + }; + TimelinePlan { generators: vec![Generator::TraceAgent(cfg)], receiver_mode, + label: format!("trace_agent:{ctx_label}:{bo_label}") } +} +``` + +(`random_bool_ratio` is the rand 0.10 spelling; confirm with `grep -rn "fn random_bool" ~/.cargo` or use `rng.random_range(0..2) == 0`. Confirm `block::CacheMethod::Fixed` variant name with `grep -rn "enum CacheMethod" lading_payload/src`.) + +- [ ] **Step 3: Implement `sample_http_plan` and `sample_dup_http_plan`** -- a single http generator (benign / stall via receiver mode) and the duplicate-http crash boundary (#9): + +```rust +fn one_http(seed: [u8; 32]) -> http::Config { + http::Config { + seed, + target_uri: hyper::Uri::from_static("http://adversarial-receiver:8080/"), + method: http::Method::POST, + headers: http::HeaderMap::new(), + bytes_per_second: Some(byte_unit::Byte::from_u64(1024 * 1024)), + maximum_block_size: byte_unit::Byte::from_u64(1024 * 1024), + parallel_connections: 1, + throttle: None, + } +} +fn sample_http_plan(_rng: &mut R, receiver_mode: &'static str) -> TimelinePlan { + TimelinePlan { generators: vec![Generator::Http(one_http(payload_seed()))], + receiver_mode, label: "http:single".to_string() } +} +fn sample_dup_http_plan(_rng: &mut R, receiver_mode: &'static str) -> TimelinePlan { + // Two http generators trip the process-global CONNECTION_SEMAPHORE double-set. + TimelinePlan { generators: vec![Generator::Http(one_http(payload_seed())), Generator::Http(one_http(payload_seed()))], + receiver_mode, label: "http:dup".to_string() } +} +``` + +(Confirm `http::Method`/`http::HeaderMap` are re-exported at those paths; lading serializes them via `http_serde`, so `serde_yaml::to_value` on the built config must round-trip. The `every_sampled_plan_parses` test proves it; if `HeaderMap`/`Method` fail to serialize under `http_serde`, fall back to building the http item as a raw `serde_yaml::Mapping` in `Generator::value`.) + +- [ ] **Step 4: Run the harness tests, expect PASS** + +Run: `cargo test -p harness --lib` +Expected: PASS, including `every_sampled_plan_parses_as_lading_config` and `menu_reaches_every_generator_kind_and_boundary`. + +- [ ] **Step 5: clippy + commit** + +```bash +cargo clippy -p harness --all-targets +git add test/antithesis/harness +git commit -m "feat(antithesis): sample trace_agent/http config boundaries in general" +``` + +### Task 6: `first_sample_config` writes the receiver mode + +**Files:** +- Modify: `test/antithesis/harness/src/bin/first_sample_config.rs` + +- [ ] **Step 1: Switch to `sample_plan` and write `receiver_mode` before `ready`:** + +```rust + let mut rng = rand::rand_core::UnwrapErr(antithesis_sdk::random::AntithesisRng); + let plan = harness::config::sample_plan(&mut rng); + let yaml = harness::config::plan_to_yaml(&plan.generators).context("serialize sampled plan")?; + + std::fs::write(dir.join("lading.yaml"), yaml.as_bytes()).context("write lading.yaml")?; + std::fs::write(dir.join("receiver_mode"), plan.receiver_mode.as_bytes()).context("write receiver_mode")?; + + lading_antithesis::reachable!("first_sample_config sampled a timeline plan", { "plan": plan.label }); + + std::fs::write(dir.join("ready"), b"ready\n").context("write ready sentinel")?; +``` + +- [ ] **Step 2: Build + clippy, expect success** + +Run: `cargo clippy -p harness --all-targets && cargo build -p harness --bin first_sample_config` +Expected: clean. + +- [ ] **Step 3: Rebuild the general images and `snouty validate`, expect success** + +Run: `docker compose -f test/antithesis/scenarios/general/docker-compose.yaml build && snouty validate test/antithesis/scenarios/general --timeout 120` +Expected: "Setup validation successful." Then tear down: `docker compose -f test/antithesis/scenarios/general/docker-compose.yaml down -v`. + +- [ ] **Step 4: Commit** + +```bash +git add test/antithesis/harness/src/bin/first_sample_config.rs +git commit -m "feat(antithesis): first_sample_config writes receiver mode" +``` + +--- + +## Part 3 -- trace_agent backoff SUT assertion (robust #2) + +### Task 7: Assert the retry loop is bounded / shutdown-aware + +**Files:** +- Modify: `lading/src/generator/trace_agent.rs` +- Test: `lading/src/generator/trace_agent.rs` (existing `#[cfg(test)]`) + +- [ ] **Step 1: Locate the resend loop** in `handle_request`/`spin` where `Backoff::wait` gates a resend on 429/503. Read it with `grep -n "wait()" lading/src/generator/trace_agent.rs`. + +- [ ] **Step 2: Add a self-documenting assertion that the resend count stays bounded** at the top of each resend iteration (choose a generous ceiling, e.g. `MAX_RETRY_MILLIS`-independent `1_000`), so an unbounded `Ignore` storm trips it while a bounded `Obey` run does not: + +```rust +// The retry loop must be bounded; an unbounded resend storm (e.g. Ignore on a +// persistent 429/503) never satisfies this and is caught even if runtime +// teardown happens to mask the exit hang. +lading_antithesis::always!( + resend_attempts < 1_000, + "trace_agent bounded its resend attempts against a rejecting target", + { "resend_attempts": resend_attempts } +); +``` + +(Introduce a local `resend_attempts: u32` incremented each resend. This is instrumentation, not a behavior change.) + +- [ ] **Step 3: Build both feature configs, expect clean** + +Run: `cargo clippy -p lading --all-targets && cargo clippy -p lading --all-targets --features antithesis` +Expected: clean. + +- [ ] **Step 4: Run trace_agent tests, expect PASS** (no behavior change) + +Run: `cargo test -p lading generator::trace_agent` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add lading/src/generator/trace_agent.rs +git commit -m "feat(antithesis): assert trace_agent resend loop stays bounded" +``` + +--- + +## Part 4 -- shutdown-safety samples its stressor + +### Task 8: Sample the stressor config under the bounded-drain oracle + +**Files:** +- Modify: `test/antithesis/scenarios/shutdown-safety/` (add a `first_sample_config` to the workload; replace the hardwired `lading.yaml` with a sampled one) OR add a shutdown-oriented sampler entry. + +- [ ] **Step 1: Decide the seam.** shutdown-safety currently bakes a fixed `lading.yaml`. To sample, give its workload the `first_sample_config` command (as general has) and drop the baked `lading.yaml` COPY; the entrypoint already blocks on `ready`. + +- [ ] **Step 2: Add a `sample_plan` variant biased to drain-stressors** -- unreachable tcp (today), trace_agent+ignore vs a 503 receiver, http+stall. Reuse `sample_plan`; the shutdown-safety scenario simply needs the adversarial-receiver present too (add the service to its compose, or point trace_agent/http at an unreachable addr for the connect-storm subset). + +- [ ] **Step 3: Build + `snouty validate` shutdown-safety, expect success; tear down.** + +- [ ] **Step 4: Commit.** + +(Task 8 is deliberately coarser: it depends on Parts 1-2 landing first and on a decision about whether shutdown-safety hosts its own receiver. Re-plan Task 8 concretely once Parts 1-2 are merged.) + +--- + +## Extension catalog (follow-on, NOT specified as tasks here) + +These reuse the machinery above; each is a menu entry or receiver behavior, added with the same TDD loop. They are listed so coverage gaps are explicit, not silently dropped: + +- **#12 tcp_rr addr** -- add a `Generator::TcpRr(tcp_rr::Config)` menu entry with `addr` a hostname (`"adversarial-receiver"`), which crashes at `spin()`. Crash-only until a `tcp_rr` blackhole peer exists; document that. +- **#6 logrotate max_depth=0** -- `logrotate::Config` has private fields, so it cannot be typed-built from the harness; emit it as a raw `serde_yaml::Mapping` with `max_depth: 0`. Writes to a tmpfs, no receiver. +- **#31 --target-pid > i32::MAX** -- a CLI axis, not a YAML field. Sample it in the lading entrypoint (a value above `2147483647`), not in `config.rs`. +- **#5 splunk_hec strict-body** -- add a `Generator::SplunkHec` entry pointed at the receiver in `garbage_ok` mode (read `splunk_hec::Config` first). +- **#22 blackhole fatal-error** -- add a lading `blackhole` to the sampled config plus a client that stresses it (EMFILE/reset); distinct from the generator path. +- **#8 sqs, #24 datadog OOM, #20 httpd leak** -- long-run resource growth; need a memory/liveness oracle, not just the panic hook. Separate plan. + +--- + +## Self-Review + +- **Spec coverage:** #4 -> Task 5 (contexts=0 menu). #2 -> Task 5 (Ignore + retryable receiver) and Task 7 (bounded-resend assertion). #9 -> Task 5 (dup-http). #15 -> Task 2/3 (stall mode) + Task 5 (http). #5 -> receiver `garbage_ok` (Task 2), generator entry deferred to catalog. #6/#12/#31/#22 -> extension catalog (with the reason each is deferred). shutdown-safety stressor -> Task 8. +- **Placeholder scan:** the only intentionally-coarse task is Task 8 (depends on Parts 1-2) and the extension catalog (explicitly a roadmap, not tasks). All Part 1-3 steps carry real code. +- **Type consistency:** `Generator`, `TimelinePlan`, `sample_plan`, `plan_to_yaml`, `Mode`, `Mode::from_file`, `receiver_mode` string tokens (`ok`/`retryable`/`stall`/`reset`/`garbage_ok`) are used identically across the receiver, sampler, and `first_sample_config`. +- **Unverified API spellings flagged inline** (confirm before relying): `ConfRange` import path, `block::CacheMethod::Fixed`, `rng.random_bool_ratio`, `http::Method`/`http::HeaderMap` round-tripping under `http_serde`. Each has a fallback noted. diff --git a/test/antithesis/scratchbook/properties/arbitrary-block-nonzero-no-panic.md b/test/antithesis/scratchbook/properties/arbitrary-block-nonzero-no-panic.md new file mode 100644 index 000000000..ae65f2dd4 --- /dev/null +++ b/test/antithesis/scratchbook/properties/arbitrary-block-nonzero-no-panic.md @@ -0,0 +1,49 @@ +--- +slug: arbitrary-block-nonzero-no-panic +subsystem: no-panic +assertion_type: Unreachable +priority: P2 +needs_target: false +needs_sut_fix: return Err(arbitrary::Error::IncorrectFormat) instead of .expect at block.rs:119-127 +--- + +# Fuzz Arbitrary Block construction handles zero total_bytes without panic + +## Statement + +The `arbitrary::Arbitrary` impl for `Block` handles a generated `total_bytes` of 0 without +`.expect` panicking, so a fuzz run is not aborted by benign input. + +## Code evidence trail + +- `block.rs:119-127` — `total_bytes = u32::arbitrary(u)?` then + `NonZeroU32::new(total_bytes).expect("total_bytes must be non-zero")`. `u32::arbitrary` can + yield 0, panicking. Only compiled under the `arbitrary` feature (fuzz harness). + +## Assertion-type rationale + +**Unreachable** within the fuzz harness: the `NonZeroU32` `.expect` must never be reached. +Oracle: `cargo fuzz` run. + +## Mode / observability + +A fuzz input yielding `total_bytes==0` is rejected gracefully (`arbitrary::Error`) not a +`NonZeroU32` expect panic. Only compiled under the `arbitrary` feature (fuzz harness), so no +production impact. + +## Mechanism + +Return `Err(arbitrary::Error::IncorrectFormat)` instead of `.expect` at `block.rs:119-127`. + +## needs_sut_fix + +Yes (fuzz-harness-only; no production no-panic impact). + +## Investigation Log + +- [2026-07-24] Confirmed: compiled only under the `arbitrary` feature, so it aborts a fuzz run + rather than affecting production, but it is still a benign-input abort worth fixing. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/blackhole-never-backpressures-target.md b/test/antithesis/scratchbook/properties/blackhole-never-backpressures-target.md new file mode 100644 index 000000000..9bfbb0791 --- /dev/null +++ b/test/antithesis/scratchbook/properties/blackhole-never-backpressures-target.md @@ -0,0 +1,57 @@ +--- +slug: blackhole-never-backpressures-target +subsystem: blackhole/config +assertion_type: Always +priority: P1 +needs_target: true +needs_sut_fix: use try_send + count-on-drop instead of blocking send().await in datadog handle_v2_protobuf +--- + +# Blackhole responds to the target regardless of capture-channel saturation + +## Statement + +A blackhole does not block its HTTP response to the target on a full bounded capture channel; a +slow/saturated capture manager cannot stall the target. + +## Code evidence trail + +- `datadog.rs:296-299` — the HTTP response is built after per-point awaits (status at + `:289-294`). +- `datadog.rs:398,412,416` — `handle_v2_protobuf` awaits `counter_incr`/`gauge_set` per point + BEFORE the response, each calling `lading_capture::send_metric`. +- `lib.rs:53-56` — `send_metric` does `sender.snd.send(metric).await` on a bounded + `mpsc::channel(10_000)` (`manager.rs:302`); a full channel blocks the connection task. +- Contrast: the manager's own histogram path uses `try_send` + drop-on-full (`manager.rs:119-125`). + +## Assertion-type rationale + +**Always** (never-backpressure): on every timeline the target's request latency must stay +bounded regardless of capture-drain speed. + +## Mode / observability + +With a slow capture drain, target request latency stays bounded; the Datadog blackhole does not +await `send().await` per metric point before responding. Oracle: slow-capture-drain fault +scenario measuring target response latency. + +## Mechanism + +Use `try_send` + count-on-drop (consistent with the manager histogram path) instead of a +blocking `send().await` in `datadog.rs` `handle_v2_protobuf` (`:398,412,416`) before the +response is built. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed: the datadog record path blocks on a bounded-channel send per point + before responding — an inconsistent, backpressure-prone choice vs the manager's own + try_send+drop histogram path. Amplified under `RecordPolicy::All` with many series/points. + +## Open Questions + +- Is the capture manager fast enough in practice that the 10k bounded channel never fills under + the deployment's datadog load, or has channel-full been observed? diff --git a/test/antithesis/scratchbook/properties/block-cache-construction-terminates.md b/test/antithesis/scratchbook/properties/block-cache-construction-terminates.md new file mode 100644 index 000000000..d4b02cab6 --- /dev/null +++ b/test/antithesis/scratchbook/properties/block-cache-construction-terminates.md @@ -0,0 +1,63 @@ +--- +slug: block-cache-construction-terminates +subsystem: payload/determinism +assertion_type: Always +priority: P0 +needs_target: false +needs_sut_fix: cap consecutive rejections / add time bound in construct_block_cache_inner; reject max_block_size below serializer floor +--- + +# Block-cache construction always terminates in bounded time + +## Statement + +`construct_block_cache_inner` terminates (with blocks or `InsufficientBlockSizes`) in bounded +time for every config; it never spins forever when `max_block_size` is below the serializer's +minimum viable block. + +## Code evidence trail + +- `block.rs:625-673` — the loop `while bytes_remaining > 0` decrements `bytes_remaining` only on + `Ok(block)` (`:637`). +- `block.rs:651` — on `EmptyBlock` it sets `min_block_size = block_size*0.25`; since + `block_size < max_block_size`, `min_block_size` stays `< max_block_size`, so the only other + exit `if bytes_remaining < min_block_size` (`:670`) never fires when `bytes_remaining` is + large. +- `block.rs:659-668` — the elapsed-time block only logs progress; it never breaks. +- `grpc.rs:212` forwards `maximum_block_size.as_u128()` with no lower-bound validation. + +## Assertion-type rationale + +**Always**/liveness (bounded startup): startup must complete or error within a bounded time on +every config. Oracle: small-`maximum_block_size` scenario + external startup-time bound; SUT +`reachable!` at the bounded-exit. + +## Mode / observability + +Startup completes (or errors) within a bounded time even with a tiny `maximum_block_size`. +Confirmed live: the loop has no time/iteration cap; on repeated `EmptyBlock`, `min_block_size` +stays `< max_block_size` so neither exit fires. + +## Mechanism + +Cap consecutive rejections / add a time bound in `block.rs:625-673` and return +`InsufficientBlockSizes`; reject `max_block_size` below a serializer-reported floor. + +## needs_sut_fix + +Yes. The trace_agent v04 variant is fixed on backup branch `456e85a3`; the general case is +still live on main. + +## Investigation Log + +- [2026-07-24] Verified against source this turn: no iteration or time cap; `min_block_size` + capped at `0.25*block_size < max_block_size`. +- [2026-07-24] Reachable via small `config.maximum_block_size` (e.g. "10 B") with + dogstatsd/otel/trace_agent whose minimum message is far larger. + +## Open Questions + +- Does the deployment/harness ever vary `maximum_block_size` to small values? If config + variation reaches small block sizes, this hang is a live startup failure, not latent. +- Should `fixed_with_max_overhead` reject `maximum_block_bytes` below a serializer-reported + minimum viable size, or impose an iteration/time cap? diff --git a/test/antithesis/scratchbook/properties/block-cache-zero-max-no-panic.md b/test/antithesis/scratchbook/properties/block-cache-zero-max-no-panic.md new file mode 100644 index 000000000..cd692a351 --- /dev/null +++ b/test/antithesis/scratchbook/properties/block-cache-zero-max-no-panic.md @@ -0,0 +1,56 @@ +--- +slug: block-cache-zero-max-no-panic +subsystem: no-panic +assertion_type: Unreachable +priority: P1 +needs_target: false +needs_sut_fix: add a lower-bound (>=1, or >= serializer floor) check on maximum_block_size +--- + +# maximum_block_size of 0 is rejected, not a random_range panic + +## Statement + +A `maximum_block_size` that resolves to 0 is rejected at validation; block-cache construction +never calls `rng.random_range` on an empty `0..0` range. + +## Code evidence trail + +- `block.rs:628` — `rng.random_range(min_block_size..max_block_size)` panics on an empty range + when `max_block_size == 0`. +- `block.rs:214-220` — the guard only errors when `maximum_block_bytes` exceeds `u32::MAX` or + `total_bytes`; a value of 0 passes through as `max_block_size=0`. +- `grpc.rs:205-223` forwards `config.maximum_block_size.as_u128()` unvalidated; a config + `maximum_block_size: '0 B'` parses to 0. + +## Assertion-type rationale + +**Unreachable**: the empty-range `random_range` must never be reached. Oracle: zero-block-size +scenario + panic hook. + +## Mode / observability + +Config with `maximum_block_size '0 B'` produces a clean startup error, not a panic. Currently +reachable via `grpc.rs` forwarding unvalidated `as_u128()`. + +## Mechanism + +Add a lower-bound (`>=1`, or `>=` serializer floor) check on `maximum_block_size` in +`block.rs:214-220` and at generator call sites (e.g. `grpc.rs:205-223`). + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed: `min_block_size` can never independently reach `max_block_size` (it + is capped at `0.25*block_size < max_block_size`), so `max_block_size==0` is the trigger for + the empty-range panic. +- [2026-07-24] Sibling of `block-cache-construction-terminates`: zero is the panic case, small + is the hang case. + +## Open Questions + +- Does config variation ever reach `maximum_block_size: '0 B'`? If so this is a live startup + panic, not latent. diff --git a/test/antithesis/scratchbook/properties/capture-drift-no-silent-gap.md b/test/antithesis/scratchbook/properties/capture-drift-no-silent-gap.md new file mode 100644 index 000000000..166145b98 --- /dev/null +++ b/test/antithesis/scratchbook/properties/capture-drift-no-silent-gap.md @@ -0,0 +1,55 @@ +--- +slug: capture-drift-no-silent-gap +subsystem: capture +assertion_type: Always +priority: P2 +needs_target: false +needs_sut_fix: flush between advance_tick iterations in the drift loop; optionally flag large fetch_index gaps in validate +--- + +# Drift correction does not silently drop unflushed intervals + +## Statement + +When the flush-tick advances multiple ticks after a scheduling stall, the accumulator does not +overwrite unflushed ring slots, so the capture has no invisible `fetch_index` gaps. + +## Code evidence trail + +- `state_machine.rs:219-232` — `handle_flush_tick` advances the accumulator `tick_drift` times + in a loop with no flush between (`:229-231`). +- `accumulator.rs:466-481` — `advance_tick` overwrites ring slots. +- A stall `> ~60s` overwrites unflushed intervals and creates `fetch_index` gaps. +- `validate/jsonl.rs:123-131` — the validator only checks strict monotonicity, NOT contiguity, + so gaps are invisible. + +## Assertion-type rationale + +**Always**: on every timeline the capture must retain all intervals; a silent gap is a defect. +Oracle: induce scheduling starvation (Antithesis) + validator extended to detect gaps. + +## Mode / observability + +Under `>60s` scheduling starvation the capture retains all intervals (no large `fetch_index` +gap). Gaps pass the current strict-monotonic validator, so loss is invisible today. + +## Mechanism + +Flush between `advance_tick` iterations in the `state_machine.rs:229-231` drift loop; +optionally have `validate` flag large `fetch_index` gaps. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed: the drift loop overwrites ring slots without an interleaved flush; the + validator's strict-monotonic (not contiguous) check means the loss passes validation. +- [2026-07-24] Antithesis is well-suited to induce the `>60s` scheduling starvation that + triggers multi-tick drift. + +## Open Questions + +- Under sustained scheduling starvation, does the drift path produce silently-gapped captures + that still pass `validate_lines`, and should the validator flag large gaps as suspicious? diff --git a/test/antithesis/scratchbook/properties/capture-finalize-bounded.md b/test/antithesis/scratchbook/properties/capture-finalize-bounded.md new file mode 100644 index 000000000..5903ba987 --- /dev/null +++ b/test/antithesis/scratchbook/properties/capture-finalize-bounded.md @@ -0,0 +1,52 @@ +--- +slug: capture-finalize-bounded +subsystem: lifecycle/shutdown +assertion_type: Always +priority: P1 +needs_target: false +needs_sut_fix: add a timeout around the capture-manager join await +--- + +# Capture finalize await is time-bounded + +## Statement + +The capture-manager join await during shutdown is bounded; a stalled parquet footer write (slow +volume, disk-full) cannot make lading hang before `runtime.shutdown_timeout` can act. + +## Code evidence trail + +- `lading.rs:713-715` — unbounded `let _ = handle.await;` on the capture-manager join. +- `lading.rs:813` — `runtime.shutdown_timeout` is only reached AFTER `inner_main` returns, so + it cannot backstop a hang inside this await. +- Finalize path: `lading_capture/src/manager.rs:397-412`, `parquet.rs:318-324` (footer write). + +## Assertion-type rationale + +**Always** (bounded latency): on every timeline finalize must complete or be timed out so +`runtime.shutdown_timeout` remains the backstop. + +## Mode / observability + +With a slow/stalled capture volume, lading still exits within `max_shutdown_delay`. Oracle: +slow-disk fault scenario + exit timer. + +## Mechanism + +Add a timeout around `let _ = handle.await;` (`lading.rs:713-715`) so +`runtime.shutdown_timeout` remains the backstop. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed the ordering hazard: `runtime.shutdown_timeout(30s)` at `lading.rs:813` + cannot help because it is only reached after this await completes. +- [2026-07-24] Related to `capture-write-failure-not-abort`: a mid-run write error is currently + a SIGABRT, not a hang; this property covers the stall-without-error case. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/capture-histogram-drops-counted.md b/test/antithesis/scratchbook/properties/capture-histogram-drops-counted.md new file mode 100644 index 000000000..ae1881a06 --- /dev/null +++ b/test/antithesis/scratchbook/properties/capture-histogram-drops-counted.md @@ -0,0 +1,54 @@ +--- +slug: capture-histogram-drops-counted +subsystem: capture +assertion_type: Always +priority: P1 +needs_target: false +needs_sut_fix: add capture_histogram_samples_dropped counter (bounded label, in registry not the sample channel) +--- + +# Dropped histogram samples are counted, not silently lost + +## Statement + +When the capture channel is full or the recorder is uninitialized, dropped latency/histogram +samples increment a bounded-label counter held in the registry, so tail-biased sample loss is +observable. + +## Code evidence trail + +- `manager.rs:302` — the sample channel is `mpsc::channel(10_000)` (bounded). +- `manager.rs:121-133` — `CaptureHistogram::record` uses `try_send` and only `warn!`s on a full + channel, silently dropping the sample. + +## Assertion-type rationale + +**Always** (observability): whenever samples are lost, the loss must be counted so a run +delivering incomplete histograms is distinguishable from a healthy one. + +## Mode / observability + +Under high recording load a run delivering incomplete histograms shows a nonzero +`capture_histogram_samples_dropped` rather than looking healthy. Oracle: saturate the channel, +assert dropped count `> 0` when samples were lost. + +## Mechanism + +Add a `capture_histogram_samples_dropped` counter (bounded reason label, held in the registry +NOT the sample channel) at `manager.rs:121-133`. + +## needs_sut_fix + +Yes. Landed on backup branch (`39d8ae56`), not main. + +## Investigation Log + +- [2026-07-24] Confirmed live on main: samples are dropped with only a `warn!`. In a + latency-measuring tool this is silent tail-biased sample loss with no mark in the capture + file. +- [2026-07-24] The counter must live in the registry, not the sample channel, so the drop count + cannot itself feed back into the full-channel condition. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/capture-no-fsync-durability.md b/test/antithesis/scratchbook/properties/capture-no-fsync-durability.md new file mode 100644 index 000000000..55adcfbdb --- /dev/null +++ b/test/antithesis/scratchbook/properties/capture-no-fsync-durability.md @@ -0,0 +1,52 @@ +--- +slug: capture-no-fsync-durability +subsystem: capture +assertion_type: Always +priority: P2 +needs_target: false +needs_sut_fix: add File::sync_data/sync_all at flush boundaries, or document maturity = handed-to-OS +--- + +# Flushed capture lines survive whole-VM termination + +## Statement + +Capture data reported as flushed reaches the persisted volume durably; a whole-container/VM +`node_termination` does not lose already-flushed jsonl lines. + +## Code evidence trail + +- `jsonl.rs:53` — `self.writer.flush()?` only calls `BufWriter::flush` -> OS page cache. +- Grep of `lading_capture/src` shows no `sync_all`/`sync_data`/`fsync` anywhere. +- `state_machine.rs:313-318` — flush interval; the maturity guarantee is "handed to the OS", not + "durable on the volume". + +## Assertion-type rationale + +**Always** durability invariant: every flushed line must be recoverable after a whole-VM kill. + +## Mode / observability + +After `node_termination`, the surviving prefix includes all lines that were flushed before the +last maturity boundary. If page-cache writes are lost, flushed lines vanish. + +## Mechanism + +Add `File::sync_data`/`sync_all` at flush boundaries (or document that maturity = +handed-to-OS, not durable) in the `lading_capture` jsonl/parquet sinks. + +## needs_sut_fix + +Yes (or an explicit documentation decision). + +## Investigation Log + +- [2026-07-24] Confirmed no fsync in the capture write path. +- [2026-07-24] Priority hinges on the open question: if Antithesis `node_termination` does not + preserve OS page cache to the persisted named volume, even flushed lines can be lost (P1); + otherwise this is closer to theoretical (P2). + +## Open Questions + +- Does `node_termination` preserve OS page cache to the persisted named volume? Determines P2 + vs P1. diff --git a/test/antithesis/scratchbook/properties/capture-write-failure-not-abort.md b/test/antithesis/scratchbook/properties/capture-write-failure-not-abort.md new file mode 100644 index 000000000..709174518 --- /dev/null +++ b/test/antithesis/scratchbook/properties/capture-write-failure-not-abort.md @@ -0,0 +1,64 @@ +--- +slug: capture-write-failure-not-abort +subsystem: capture +assertion_type: Unreachable +priority: P0 +needs_target: false +needs_sut_fix: replace .expect on capture start with Result propagation; plumb mid-run write errors to graceful shutdown +--- + +# A capture write error is a clean fatal exit, not a process abort + +## Statement + +A transient capture write/flush error (disk full, EIO) causes a clean non-zero exit with the +parquet footer flushed, not a `panic=abort` that SIGABRTs the whole run mid-flush and leaves an +unreadable file. + +## Code evidence trail + +- `lading.rs:476`, `:501`, `:526` — `.block_on(capture_manager.start()).expect("failed to + start capture manager")`. +- `manager.rs:409` — `state_machine.next(event)?` propagates format errors up. +- `state_machine.rs:317`/`:343` `flush()?`, `:396` `write_metric()?` — any IO error becomes an + `Err` that hits the `.expect`. +- `Cargo.toml:115,120` — `panic = "abort"` turns the expect into a SIGABRT that skips + `format.close()` (the parquet footer). + +## Assertion-type rationale + +**Unreachable** on the abort site: a capture error must never reach the abort. The oracle is +the IO-fault-injection scenario + the panic hook (must stay silent) + an external +parquet-readability check. + +## Mode / observability + +On an injected capture-write IO error the panic hook must NOT fire and the parquet must be +readable. Currently the `.expect` + `panic=abort` turns any flush-tick error into a SIGABRT +that skips `format.close()`. + +## Mechanism + +Replace the `.expect` on capture start (`lading.rs:476/501/526`) with `Result` propagation to +`Error::CaptureManager` so `BufWriter`s flush on Drop; plumb mid-run write errors to graceful +shutdown. + +## needs_sut_fix + +Yes. Partly landed on backup branch (`b7624af2`), not on main. Note: `b7624af2` is +fatal-on-startup only; the mid-run write error still needs graceful shutdown plumbing. + +## Investigation Log + +- [2026-07-24] Confirmed live on main: the `.expect` + `panic=abort` path means a single mid-run + IO hiccup both kills the experiment AND leaves the parquet footer-less/unreadable, because + `format.close()` (`state_machine.rs:255`) only runs on `Event::ShutdownSignaled`, which the + abort bypasses. +- [2026-07-24] Deployment relevance: production captures are parquet-only; an unreadable parquet + = total capture loss for the replicate (same failure mode as watchdog SIGKILL). + +## Open Questions + +- Is a mid-run capture-write failure intended to be fatal to the whole experiment? If yes it + should be a clean abort with a footer-flushed parquet; if no, capture errors should be logged + and the run continue. Current behavior (SIGABRT + unreadable parquet) is neither. diff --git a/test/antithesis/scratchbook/properties/config-numeric-fields-validated.md b/test/antithesis/scratchbook/properties/config-numeric-fields-validated.md new file mode 100644 index 000000000..506913018 --- /dev/null +++ b/test/antithesis/scratchbook/properties/config-numeric-fields-validated.md @@ -0,0 +1,55 @@ +--- +slug: config-numeric-fields-validated +subsystem: blackhole/config +assertion_type: Always +priority: P2 +needs_target: false +needs_sut_fix: add range checks in config.rs (compression_level 1-22, sample_period_milliseconds > 0) +--- + +# Numeric config fields are validated at load, not deferred to runtime failure + +## Statement + +parquet/zstd `compression_level` (1-22) and `sample_period_milliseconds` (>0) are range-checked +at config load, so an out-of-range value fails early rather than corrupting a capture write or +driving a tight sampling loop. + +## Code evidence trail + +- `config.rs:186-198` — `compression_level: i32` (default 3) with doc "1-22" but no range check; + an out-of-range value surfaces only as a zstd error during capture writing -> footer-less + unreadable parquet on failure. +- `config.rs:106-107` — `sample_period_milliseconds: u64` has no lower-bound check; 0 could + drive a tight observer sampling loop. +- Config validation is centralized here and is the intended crash-early location (module + docstring `:1-3`). + +## Assertion-type rationale + +**Always** (validate-early): on every timeline an out-of-range numeric field must be rejected at +startup, not surfaced as a mid-run failure. Oracle: out-of-range-config scenario asserting a +clean startup error. + +## Mode / observability + +An out-of-range `compression_level` or a zero `sample_period` is rejected at startup (the +documented crash-early location), not surfaced as a mid-run zstd error / busy loop. + +## Mechanism + +Add range checks in `config.rs:186-198` (`compression_level`) and `:106-107` +(`sample_period_milliseconds > 0`). + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed: these numeric fields have no validation; they are validation gaps, not + by-design deferral, since config.rs is the intended crash-early location. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/datadog-blackhole-accept-resilient.md b/test/antithesis/scratchbook/properties/datadog-blackhole-accept-resilient.md new file mode 100644 index 000000000..c84d17ee9 --- /dev/null +++ b/test/antithesis/scratchbook/properties/datadog-blackhole-accept-resilient.md @@ -0,0 +1,58 @@ +--- +slug: datadog-blackhole-accept-resilient +subsystem: blackhole/config +assertion_type: Always +priority: P1 +needs_target: true +needs_sut_fix: match accept() and continue on Err at datadog.rs:209 +--- + +# Datadog blackhole keeps accepting after a transient accept error + +## Statement + +The Datadog blackhole's accept loop logs-and-continues on an `accept()` error and never wedges +(silently ceasing to accept while appearing alive), so it never backpressures the target. + +## Code evidence trail + +- `datadog.rs:207-212` — the `tokio::select!` branch uses a fallible pattern + `Ok((stream, _addr)) = listener.accept()` with only a shutdown branch and no `else`. On an + `accept()` `Err` (EMFILE/ENFILE fd exhaustion, ECONNABORTED) the pattern fails to match, tokio + disables that branch, and `select!` blocks on `shutdown_wait` forever; accept is never re-armed. +- Contrast `common.rs:89-96` which does `match incoming { Ok=>.., Err(e)=>{ error!; continue }}` + (resilient). + +## Assertion-type rationale + +**Always** (blackhole never-backpressure): on every timeline, including fd-exhaustion timelines, +the blackhole must keep serving new connections. + +## Mode / observability + +Under fd exhaustion / transient accept errors the blackhole keeps serving new connections. +Confirmed live: the fallible select arm with no `else` disables the branch on `Err`, then blocks +on shutdown forever. Oracle: fd-exhaustion fault scenario asserting the target's connections +still succeed. + +## Mechanism + +Match `accept()` and `continue` on `Err` (as `common.rs:89-96` does) at `datadog.rs:209`. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Verified against source this turn: datadog is the buggy outlier — it appears + alive but stops serving, so the target's connections stall = backpressure, violating the + blackhole "never backpressure the target" invariant. +- [2026-07-24] Accept-error handling is inconsistent across blackholes (common.rs resilient; + tcp/udp/unix_stream die; datadog wedges) — worth a single policy decision. + +## Open Questions + +- Does lading's supervisor treat a blackhole `run()` returning `Err` (tcp/udp/unix_stream + accept error) as graceful shutdown or hard failure? Determines whether the divergence causes + rig termination vs silent target backpressure. diff --git a/test/antithesis/scratchbook/properties/discarded-blocks-counted.md b/test/antithesis/scratchbook/properties/discarded-blocks-counted.md new file mode 100644 index 000000000..3176414f2 --- /dev/null +++ b/test/antithesis/scratchbook/properties/discarded-blocks-counted.md @@ -0,0 +1,52 @@ +--- +slug: discarded-blocks-counted +subsystem: generators/throttle +assertion_type: Always +priority: P1 +needs_target: true +needs_sut_fix: add blocks_discarded counters in tcp/udp/grpc +--- + +# Under-delivery is observable: discarded blocks are counted + +## Statement + +tcp/udp/grpc generators count throttle-rejected/discarded blocks (`blocks_discarded`) so a run +delivering ~zero bytes is distinguishable from a healthy flat-metrics run. + +## Code evidence trail + +- `tcp.rs:273-276` — discards with only a `debug!` log, no counter. +- Same discard-without-counter pattern in udp and grpc. + +## Assertion-type rationale + +**Always** (observability): whenever blocks are discarded the count must be nonzero, so silent +under-delivery is distinguishable. Oracle: oversized-block scenario asserting +`delivered==0 => blocks_discarded>0`. + +## Mode / observability + +A zero-byte-delivery run surfaces a nonzero `blocks_discarded` rather than only a `debug!` log. +Distinguishes silent under-delivery. + +## Mechanism + +Add `blocks_discarded` counters in `tcp.rs`/`udp.rs` (`73c4805e` on backup branch) and +`grpc.rs`. + +## needs_sut_fix + +Yes. `73c4805e` on backup branch; live on main. + +## Investigation Log + +- [2026-07-24] Confirmed live on main: discards are logged at `debug!` only, so a config that + discards every block (the 0.31.2 busy-discard livelock class) looks like a healthy + flat-metrics run. +- [2026-07-24] Observability companion to `throttle-divide-no-silent-underdelivery` and + `throttle-capacity-no-zerodelivery-livelock`. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/divide-by-zero-startup-error.md b/test/antithesis/scratchbook/properties/divide-by-zero-startup-error.md new file mode 100644 index 000000000..28f841e6b --- /dev/null +++ b/test/antithesis/scratchbook/properties/divide-by-zero-startup-error.md @@ -0,0 +1,53 @@ +--- +slug: divide-by-zero-startup-error +subsystem: generators/throttle +assertion_type: Always +priority: P2 +needs_target: true +needs_sut_fix: distribute the division remainder; surface DivisionByZero as a config validation error +--- + +# bytes_per_second < parallel_connections fails with a clear error, not DivisionByZero surprise + +## Statement + +A config where `bytes_per_second` divided by `parallel_connections` rounds to zero produces a +clear startup validation error, and integer-division truncation does not silently drop the +remainder rate. + +## Code evidence trail + +- `lib.rs` — `divide` returns `DivisionByZero` when `capacity/divisor` rounds to 0. +- `tcp.rs:168-173` — integer division `capacity/divisor` truncates; N workers deliver + `N*floor(bps/N) < bps` whenever `bps` is not divisible by `parallel_connections`; the + remainder (up to `N-1` bytes/interval) is never delivered. + +## Assertion-type rationale + +**Always**: on every timeline a small-bps/high-connection config either errors clearly at +startup or delivers within one interval-quantum of configured. Oracle: startup-error assertion ++ sink rate check. + +## Mode / observability + +Small-bps/high-connection config errors clearly at startup; aggregate delivered rate is within +one interval-quantum of configured (no systematic under-delivery of up to `N-1` bytes/interval). + +## Mechanism + +Distribute the division remainder across workers, and surface `DivisionByZero` as a config +validation error naming `bytes_per_second`/`parallel_connections`. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed: integer-division truncation systematically under-delivers the + remainder, and `bps < parallel_connections` fails to start with a bare `DivisionByZero` + rather than a config-validation error. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/docker-target-discovery-bounded.md b/test/antithesis/scratchbook/properties/docker-target-discovery-bounded.md new file mode 100644 index 000000000..5a1070a9e --- /dev/null +++ b/test/antithesis/scratchbook/properties/docker-target-discovery-bounded.md @@ -0,0 +1,58 @@ +--- +slug: docker-target-discovery-bounded +subsystem: lifecycle/shutdown +assertion_type: Always +priority: P0 +needs_target: true +needs_sut_fix: add shutdown.recv() arm and/or max-attempts timeout to watch_container loop +--- + +# Container-target discovery is bounded and shutdown-aware + +## Statement + +In container/observer mode, the target-container discovery poll loop either finds the +container, times out with an error, or responds to shutdown; it never spins forever so the +experiment timer can start. + +## Code evidence trail + +- `target.rs:212-244` — the `watch_container()` poll loop has only a container-found break plus + `sleep(1s)`; no `shutdown.recv` arm, no max-attempts timeout. +- `lading.rs:632-641` — `experiment_sequence` awaits `target_running` before arming the warmup + and experiment timers, so if the container never appears the timer never starts. +- Contrast PID mode (`target.rs:314-317`) which validates once and errors immediately (bounded). + +## Assertion-type rationale + +**Always** (bounded startup): on every timeline the discovery loop must terminate (found, +error, or shutdown) so lading can reach self-termination. + +## Mode / observability + +With a misnamed/never-started target container, lading either errors out bounded or +self-terminates on its timer; it does not block `experiment_sequence` at +`target_running_watcher.recv()` forever. Oracle: wrong `--target-container` name scenario; +assert lading exits (error or timer) within the watchdog. + +## Mechanism + +Add a `shutdown.recv()` arm and/or a max-attempts timeout to the `target.rs:212-244` +`watch_container` loop. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] This is the PRODUCTION observer path: the deployment launches lading with + `--target-container target` and a bind-mounted docker socket, so this loop runs in real + deployments. High impact. +- [2026-07-24] If discovery never completes, the experiment timer never starts, so lading never + self-terminates — only ctrl_c could end it. Violates the "lading owns the clock / bounded + startup" invariant. Strong deployment lead. + +## Open Questions + +- None specific; strongly deployment-grounded. diff --git a/test/antithesis/scratchbook/properties/dogstatsd-tag-length-validated.md b/test/antithesis/scratchbook/properties/dogstatsd-tag-length-validated.md new file mode 100644 index 000000000..c1a349607 --- /dev/null +++ b/test/antithesis/scratchbook/properties/dogstatsd-tag-length-validated.md @@ -0,0 +1,54 @@ +--- +slug: dogstatsd-tag-length-validated +subsystem: payload/determinism +assertion_type: Always +priority: P2 +needs_target: false +needs_sut_fix: none (merged to main, e98e3052) +--- + +# DogStatsD tag_length.end() <= MIN_TAG_LENGTH is rejected upfront + +## Statement + +A dogstatsd config with `tag_length.end() <= MIN_TAG_LENGTH` is rejected at construction with a +dedicated error naming the value, not swallowed as `Error::StringGenerate` that silently drops +the message. + +## Code evidence trail + +- Merged fix `e98e3052` (#1875) — rejects `tag_length.end() <= MIN_TAG_LENGTH` upfront through + `DogStatsD::new`. +- Root cause: the tags Generator wrapper subtracts 1 from `tag_length.end()` for the ':' + separator; when `end == MIN_TAG_LENGTH` (3) it produced `Inclusive{3,2}`, rejected by the + inner `min<=max` check and mis-surfaced as `Error::StringGenerate` (blaming pool generation, + dropping the message). + +## Assertion-type rationale + +**Always** (config regression): on every timeline such a config must error clearly through +`DogStatsD::new`. Oracle: proptest/fixture over `tag_length` bounds asserting the dedicated +error path. + +## Mode / observability + +Such a config errors clearly through `DogStatsD::new` rather than mis-surfacing as a +pool-generation error. + +## Mechanism + +None (merged to main, `e98e3052`); the property guards against regression. + +## needs_sut_fix + +None (merged to main). Regression-guard. + +## Investigation Log + +- [2026-07-24] Merged to main (`e98e3052`, #1875); also branch + `blt/fix-dogstatsd-tag-length-wrapper-underflow`. This property guards against reintroducing + the underflow. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/error-label-cardinality-bounded.md b/test/antithesis/scratchbook/properties/error-label-cardinality-bounded.md new file mode 100644 index 000000000..9a28363dc --- /dev/null +++ b/test/antithesis/scratchbook/properties/error-label-cardinality-bounded.md @@ -0,0 +1,53 @@ +--- +slug: error-label-cardinality-bounded +subsystem: generators/throttle +assertion_type: Always +priority: P1 +needs_target: true +needs_sut_fix: map errors to io::ErrorKind for labels in tcp/udp (gRPC tonic error still raw, follow-up) +--- + +# Generator error-metric label cardinality is bounded + +## Statement + +`connection_failure`/`request_failure` error labels are drawn from a finite set +(`io::ErrorKind`), not raw `err.to_string()`; a flapping target cannot grow capture memory +without bound. + +## Code evidence trail + +- `tcp.rs`/`udp.rs` — use full `err.to_string()` as the "error" metric label; raw messages + embed addresses/errno text. +- Each distinct label mints a new capture accumulator key (`[u64;61]+[f64;61]+[DDSketch;61]`) + -> unbounded memory growth (ADR-005 OOM class). + +## Assertion-type rationale + +**Always** (bounded cardinality / memory): on every timeline the distinct error-label values +must stay bounded regardless of failure diversity. + +## Mode / observability + +Distinct error-label values stay bounded regardless of failure diversity; capture accumulator +key count does not grow unboundedly. Oracle: flapping-target scenario + capture key-count / +memory observation. + +## Mechanism + +Map errors to `io::ErrorKind` for labels in `tcp.rs`/`udp.rs` (`32dd4cf6` on backup branch); +the gRPC tonic error is still raw (follow-up). + +## needs_sut_fix + +Yes. `32dd4cf6` on backup branch; live on main. gRPC tonic error label still raw (unbounded), +noted as follow-up in the fix. + +## Investigation Log + +- [2026-07-24] Confirmed live on main: raw `err.to_string()` labels embed addresses/errno text, + so a flapping target mints unbounded accumulator keys. + +## Open Questions + +- None specific (gRPC follow-up is tracked in the fix commit). diff --git a/test/antithesis/scratchbook/properties/generator-addr-uri-validation-no-panic.md b/test/antithesis/scratchbook/properties/generator-addr-uri-validation-no-panic.md new file mode 100644 index 000000000..fe9c4ddad --- /dev/null +++ b/test/antithesis/scratchbook/properties/generator-addr-uri-validation-no-panic.md @@ -0,0 +1,49 @@ +--- +slug: generator-addr-uri-validation-no-panic +subsystem: no-panic +assertion_type: Unreachable +priority: P1 +needs_target: false +needs_sut_fix: return Result errors instead of .expect at tcp/udp/grpc construction +--- + +# Malformed addr/target_uri is a Result error, not a construction panic + +## Statement + +An unresolvable/malformed socket address (tcp/udp) or `target_uri` (grpc) yields a clean config +error, not an `.expect` panic at generator construction. + +## Code evidence trail + +- `tcp.rs:154-159` — `to_socket_addrs().expect("could not convert to socket").next().expect(..)`. +- `udp.rs:164-169` — same pattern. +- `grpc.rs:226-231` — `.expect("target_uri must be valid")`; `:245-247` + `.expect("target_uri should have an RPC path")`. + +## Assertion-type rationale + +**Unreachable**: these construction `.expect`s on user-controlled strings must never be reached. +Oracle: malformed-config scenario + panic hook. + +## Mode / observability + +Config with a bad addr/uri fails startup with an error; the panic hook does not fire. + +## Mechanism + +Return `Result` errors instead of `.expect` at `tcp.rs:154-159`, `udp.rs:164-169`, +`grpc.rs:226-231,245-247`. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed: several generators `.expect()` on values derived from user strings, + violating the no-panic ADR (which requires returning the `Result` Error). + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/get-available-memory-cgroup-chain.md b/test/antithesis/scratchbook/properties/get-available-memory-cgroup-chain.md new file mode 100644 index 000000000..a6d39c37f --- /dev/null +++ b/test/antithesis/scratchbook/properties/get-available-memory-cgroup-chain.md @@ -0,0 +1,51 @@ +--- +slug: get-available-memory-cgroup-chain +subsystem: observer +assertion_type: Always +priority: P2 +needs_target: false +needs_sut_fix: none (merged to main, 1085887c) +--- + +# Memory limit reflects the tightest cgroup v2 ancestor limit + +## Statement + +`get_available_memory` walks the cgroup v2 ancestor chain and returns the minimum `memory.max`, +matching kernel hierarchical enforcement, rather than reading "max" from a namespaced root and +believing it has `u64::MAX`. + +## Code evidence trail + +- Merged fix `1085887c` — `get_available_memory` walks the cgroup v2 ancestor chain from the + process-specific path upward and returns the tightest (minimum) `memory.max`. +- Injectable file reader `get_available_memory_with` for deterministic tests. + +## Assertion-type rationale + +**Always** (accuracy): on every timeline the reported available memory must equal the effective +container limit. Oracle: deterministic test with synthetic cgroup files (injectable reader). + +## Mode / observability + +Reported available memory equals the effective container limit, not `u64::MAX`, in a +cgroup-namespaced container. + +## Mechanism + +None (merged to main, `1085887c`); the property guards against regression via the injectable +`get_available_memory_with` reader. + +## needs_sut_fix + +None (merged to main). Regression-guard. + +## Investigation Log + +- [2026-07-24] Merged to main (`1085887c`, #1824). Previously only read the cgroup v2 root + `memory.max`, which reads "max" when the container has its own cgroup namespace, so lading + believed it had `u64::MAX` memory. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/grpc-connect-loop-shutdown.md b/test/antithesis/scratchbook/properties/grpc-connect-loop-shutdown.md new file mode 100644 index 000000000..59e6bc9a1 --- /dev/null +++ b/test/antithesis/scratchbook/properties/grpc-connect-loop-shutdown.md @@ -0,0 +1,49 @@ +--- +slug: grpc-connect-loop-shutdown +subsystem: lifecycle/shutdown +assertion_type: Always +priority: P1 +needs_target: true +needs_sut_fix: add shutdown branch to grpc connect loop +--- + +# gRPC connect loop observes shutdown + +## Statement + +The gRPC generator's initial connect loop polls shutdown; a never-available target does not +wedge the generator. + +## Code evidence trail + +- `grpc.rs:287-299` — the initial connect loop retries `connect()` -> `sleep(100ms)` with no + shutdown check; if the target never comes up the generator spins in the pre-select connect + loop and cannot be shut down gracefully. + +## Assertion-type rationale + +**Always** (bounded shutdown): every timeline that signals shutdown must reach exit within the +bound, including the never-up-target case. + +## Mode / observability + +Never-up target: lading exits within bound rather than spinning `connect()` -> `sleep(100ms)`. +Oracle: external exit-timer. + +## Mechanism + +Add a shutdown branch to the `grpc.rs:287-299` connect loop. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed live on main: gRPC connect loop has no shutdown check. +- [2026-07-24] Related: gRPC also ignores the throttle result (`grpc-honors-throttle`) and + serializes requests, both separate properties. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/grpc-honors-throttle.md b/test/antithesis/scratchbook/properties/grpc-honors-throttle.md new file mode 100644 index 000000000..f325ae4ae --- /dev/null +++ b/test/antithesis/scratchbook/properties/grpc-honors-throttle.md @@ -0,0 +1,52 @@ +--- +slug: grpc-honors-throttle +subsystem: generators/throttle +assertion_type: Always +priority: P0 +needs_target: true +needs_sut_fix: honor the throttle Result in grpc.rs (discard+count on rejection) +--- + +# gRPC generator honors throttle rejections and configured rate + +## Statement + +The gRPC generator discards+counts throttle-rejected blocks (like tcp/udp) and does not send a +block regardless of the throttle result; delivered rate does not exceed configured. + +## Code evidence trail + +- `grpc.rs:307-318` — `let _ = result;` at `:309` ignores the throttle outcome and sends the + block regardless; requests are also awaited sequentially at `:313-318`. + +## Assertion-type rationale + +**Always** (rate fidelity): on every timeline the gRPC delivered rate must not exceed the +configured `bytes_per_second`, and a rejected block must not be sent. + +## Mode / observability + +gRPC delivered rate `<= configured bytes_per_second`; on a Capacity error the block is not +sent. Currently `let _ = result;` ignores the outcome and sends anyway. Oracle: sink byte-rate +vs configured under a gRPC scenario. + +## Mechanism + +Honor the throttle `Result` in `grpc.rs:307-318` (discard + count `blocks_discarded` on +rejection). + +## needs_sut_fix + +Yes. Landed on backup branch (`944d4be4`); live on main. + +## Investigation Log + +- [2026-07-24] Confirmed live regression on main: the throttle result is discarded via + `let _ = result;`. +- [2026-07-24] Separately, gRPC serializes requests (awaits each before the next), so effective + throughput becomes RTT-bound and under-delivers vs configured when the target is slow — a + distinct rate-fidelity concern in the same code region. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/jsonl-prefix-valid-after-kill.md b/test/antithesis/scratchbook/properties/jsonl-prefix-valid-after-kill.md new file mode 100644 index 000000000..ada4f3877 --- /dev/null +++ b/test/antithesis/scratchbook/properties/jsonl-prefix-valid-after-kill.md @@ -0,0 +1,59 @@ +--- +slug: jsonl-prefix-valid-after-kill +subsystem: capture +assertion_type: Always +priority: P1 +needs_target: false +needs_sut_fix: none (holds by construction) +--- + +# A SIGKILLed jsonl capture is always a valid parseable prefix + +## Statement + +Under abrupt kill (`node_termination`/SIGKILL) any surviving jsonl capture parses as a valid +prefix with no torn final record and strictly-increasing per-series `fetch_index`/`time`. + +## Code evidence trail + +- `anytime_capture_consistent.rs:44` — `always! (torn_before_final == 0)`. +- `anytime_capture_consistent.rs:49` — `always! (invariants_hold)`. +- `accumulator.rs:492-496` — `flush_tick = current_tick - INTERVALS`; flush emits ticks in + strictly increasing order (monotonic flush). +- `state_machine.rs:305-307` — `time_ms = start_ms + tick*1000`, `fetch_index == tick`, so + fetch_index->time is a global bijection. + +## Assertion-type rationale + +**Always**: every prefix on every timeline must validate. Oracle already implemented (harness +checker, `MIN_RECORDS=10` non-vacuity floor). + +## Mode / observability + +`anytime_capture_consistent.rs` `always! (torn_before_final==0)` and `always! (invariants_hold)`. +Loss of the last `<=60s` maturity window is tolerated by design; a torn/reordered record is a +defect. + +## Mechanism + +None (holds by construction); property guards against regressions in the accumulator flush +ordering. + +## needs_sut_fix + +None. This is a positive/regression-guard property. + +## Investigation Log + +- [2026-07-24] Confirmed the by-construction argument: `flush()` emits ticks strictly + increasing (`accumulator.rs:496`) and drain continues from `last_flushed_tick+1` upward, so + per-series fetch_index/time are strictly increasing in any truncated jsonl prefix. +- [2026-07-24] CAVEAT (SMPTNG-694): captures are now zstd-compressed. A truncated zstd stream + may be undecodable past the last flushed frame — the "valid parseable prefix" assumption must + be validated against the actual on-disk framing (see `recorded-traffic-crash-consistency` + and `capture-no-fsync-durability`). + +## Open Questions + +- Does the jsonl-on-disk format survive a SIGKILL as a decodable prefix once zstd-framed, or + only up to the last flushed frame boundary? diff --git a/test/antithesis/scratchbook/properties/lading-completes-and-exits-cleanly.md b/test/antithesis/scratchbook/properties/lading-completes-and-exits-cleanly.md new file mode 100644 index 000000000..abdff90e3 --- /dev/null +++ b/test/antithesis/scratchbook/properties/lading-completes-and-exits-cleanly.md @@ -0,0 +1,52 @@ +--- +slug: lading-completes-and-exits-cleanly +subsystem: lifecycle/shutdown +assertion_type: Sometimes +priority: P0 +needs_target: false +needs_sut_fix: none +--- + +# lading self-terminates on its experiment timer and exits 0 (non-vacuity) + +## Statement + +On the happy path lading owns the clock: it reaches experiment end, drains capture, and exits 0 +with a readable capture at least once. + +## Code evidence trail + +- `lading.rs` — `experiment_sequence` + the graceful path (self-termination on the experiment + timer). + +## Assertion-type rationale + +**Sometimes** (liveness non-vacuity): the good shutdown path must be reachable on at least one +timeline, guarding against a regression that makes every timeline hang or abort. + +## Mode / observability + +A run reaches clean self-termination with exit 0 and a footer-complete capture. `reachable!` +anchor for shutdown coverage. Oracle: external exit-code + capture-readable check; SUT +`reachable!` at clean exit. + +## Mechanism + +Sometimes non-vacuity: at least one timeline must reach exit 0 with a readable capture, +otherwise every shutdown-safety Always property would be vacuously true while lading in fact +never shuts down cleanly. + +## needs_sut_fix + +None. + +## Investigation Log + +- [2026-07-24] Deployment context: this is the production success path — lading self-exits 0, + writes the parquet footer, and RJO copies the capture off before force-killing the pod. +- [2026-07-24] Complements the P0 Always shutdown properties by ensuring the clean-exit path is + actually exercised (non-vacuity floor). + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/linear-ramp-slope-preserved.md b/test/antithesis/scratchbook/properties/linear-ramp-slope-preserved.md new file mode 100644 index 000000000..7ef3cfe73 --- /dev/null +++ b/test/antithesis/scratchbook/properties/linear-ramp-slope-preserved.md @@ -0,0 +1,53 @@ +--- +slug: linear-ramp-slope-preserved +subsystem: generators/throttle +assertion_type: Always +priority: P0 +needs_target: true +needs_sut_fix: divide rate_of_change by divisor in the Linear divide arm +--- + +# Linear throttle aggregate ramp slope equals configured rate_of_change + +## Statement + +With a Linear throttle and `parallel_connections>1`, the aggregate ramp slope across workers +equals the single configured `rate_of_change`, not N times it. + +## Code evidence trail + +- `lib.rs:178-193` — the Linear `divide` arm splits capacities but passes `rate_of_change: rate` + unchanged, so N parallel workers each ramp at the full rate -> aggregate `N*rate`. +- The call-site comment claims rate preservation, but the aggregate is `N*rate`. + +## Assertion-type rationale + +**Always** (rate fidelity): on every timeline the aggregate ramp slope must equal the single +configured `rate_of_change`. Pure proptest measuring aggregate slope; or Antithesis oracle +sampling sink rate over the warmup ramp. + +## Mode / observability + +Aggregate delivered-rate ramp reaches max in the configured time, not `1/N` of it. Confirmed +live: `divide()` divides capacities but passes rate unchanged (`rate_of_change: rate`) so N +workers each ramp at full rate. + +## Mechanism + +Divide `rate_of_change` by `divisor` in the `lib.rs` Linear `divide` arm (the call-site comment +claims preservation but the aggregate is `N*rate`). + +## needs_sut_fix + +Yes. Demonstrated on backup branch (`914bb14a`); live on main. + +## Investigation Log + +- [2026-07-24] Verified against source this turn (`lib.rs:178-193`): capacities divided, + `rate_of_change` unchanged. +- [2026-07-24] Contradicts the call-site comment; a subtle rate-fidelity regression the harness + can catch by sampling the warmup ramp slope. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/logrotate-stale-tick-noop.md b/test/antithesis/scratchbook/properties/logrotate-stale-tick-noop.md new file mode 100644 index 000000000..07e5661e7 --- /dev/null +++ b/test/antithesis/scratchbook/properties/logrotate-stale-tick-noop.md @@ -0,0 +1,49 @@ +--- +slug: logrotate-stale-tick-noop +subsystem: no-panic +assertion_type: Unreachable +priority: P1 +needs_target: false +needs_sut_fix: make advance_time early-return on a stale tick in logrotate_fs/model.rs +--- + +# logrotate_fs treats a stale tick as a no-op, never a panic + +## Statement + +`Model::advance_time` treats a tick below current model time as a no-op early return (a benign +FUSE scheduling reorder), never asserting/panicking. + +## Code evidence trail + +- `logrotate_fs/model.rs` — `Model::advance_time` asserted `now >= self.now` and aborted; FUSE + handlers sample the tick before taking the model lock, so two reordered ops present a stale + tick (a benign scheduling race, no clock fault needed). + +## Assertion-type rationale + +**Unreachable**: the stale-tick assert must never fire. Oracle: concurrent FUSE op scenario + +panic hook. + +## Mode / observability + +Reordered FUSE ops presenting a stale tick do not crash the logrotate_fs generator. No clock +fault needed to trigger. + +## Mechanism + +Make `advance_time` early-return on a stale tick in `logrotate_fs/model.rs` (fix `220850e5` on +backup branch; live on main). + +## needs_sut_fix + +Yes. Fix `220850e5` on backup branch; live on main. + +## Investigation Log + +- [2026-07-24] Confirmed: model time only advances; a tick below current is a benign + scheduling-reorder that currently panics rather than being a no-op. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/multi-format-parquet-not-forfeited.md b/test/antithesis/scratchbook/properties/multi-format-parquet-not-forfeited.md new file mode 100644 index 000000000..574f4ec50 --- /dev/null +++ b/test/antithesis/scratchbook/properties/multi-format-parquet-not-forfeited.md @@ -0,0 +1,50 @@ +--- +slug: multi-format-parquet-not-forfeited +subsystem: capture +assertion_type: Always +priority: P1 +needs_target: false +needs_sut_fix: reorder multi close/flush/write_metric to finalize parquet first (or best-effort both) +--- + +# multi format finalizes parquet even if jsonl close errors + +## Statement + +In multi capture mode a trivial jsonl flush/close error does not skip the parquet footer write; +the important format is never sacrificed to the unimportant one. + +## Code evidence trail + +- `multi.rs:69` — `self.jsonl.close()?;` runs BEFORE `:71` `self.parquet.close()?;`. If the + jsonl `close()` errors, the `?` returns before the parquet footer is written. +- `multi.rs:46-48` (`write_metric`) and `:57-58` (`flush`) have the same ordering hazard. + +## Assertion-type rationale + +**Always**: on every timeline where multi mode finalizes, the parquet footer must be written +regardless of a jsonl error. + +## Mode / observability + +Inject a jsonl close error; the parquet footer is still written and the parquet file is +readable. Oracle: fault-inject jsonl path + external parquet readability check. + +## Mechanism + +Reorder `multi.rs` close/flush/write_metric to finalize parquet first (or best-effort both, +aggregating errors) at `multi.rs:46-48,57-58,69-72`. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed the ordering: jsonl (the "unimportant" format) is closed first, so a + trivial jsonl error forfeits the critical parquet footer. + +## Open Questions + +- Should `multi::Format::close()` finalize parquet FIRST, or best-effort both and aggregate + errors? diff --git a/test/antithesis/scratchbook/properties/no-panic-anywhere.md b/test/antithesis/scratchbook/properties/no-panic-anywhere.md new file mode 100644 index 000000000..733c11c5f --- /dev/null +++ b/test/antithesis/scratchbook/properties/no-panic-anywhere.md @@ -0,0 +1,67 @@ +--- +slug: no-panic-anywhere +subsystem: no-panic +assertion_type: Unreachable +priority: P0 +needs_target: false +needs_sut_fix: none (hook already wired) +--- + +# lading never panics (panic hook = Unreachable) + +## Statement + +No panic occurs anywhere in the lading SUT under any config, fault, timing, or shutdown +path; the panic hook must never report "lading panicked". + +## Code evidence trail + +- `lading/src/antithesis_hooks.rs:13-33` — `init()` installs a panic hook that wraps the + previous default hook. +- `lading/src/antithesis_hooks.rs:27` — the hook fires `unreachable! "lading panicked"` with + details `{message, location}` on any panic, then forwards to the previous hook. +- `lading/src/bin/lading.rs:741` — `lading::antithesis_hooks::init()` is called as early as + possible in `main`, before the tokio runtime is built. +- `Cargo.toml:115,120` — both release and dev profiles set `panic = "abort"`, so every panic + is a hard, observable process abort. + +## Assertion-type rationale + +**Unreachable**: a panic is a point in code that must never be reached. The existing panic +hook is the oracle — it is the primary bridge enforcing the no-panic ADR. Because `panic=abort` +turns any hit into a SIGABRT, every reachable panic is an Antithesis-visible `unreachable!` +failure with the message and source location attached. + +## Mode / observability + +The hook fires on any panic before `panic=abort` SIGABRTs the container. Directly +Antithesis-visible via the reported `unreachable!`. This is an umbrella invariant; individual +panic sites are catalogued as their own properties. + +## Mechanism + +Umbrella no-panic invariant. SUT-side probe already present in `antithesis_hooks.rs`. No +additional instrumentation is needed for the umbrella — the individual panic sites listed as +separate properties each need their ADR-compliant `Result` conversions. + +## needs_sut_fix + +None for the hook itself (already wired). The individual panic sites (observer asserts, +`.expect` call sites, OnceCell double-set, etc.) each need their own Result conversions, +tracked as separate properties. + +## Investigation Log + +- [2026-07-24] Confirmed from the SDK inventory: `lading/src` carries only two real SDK sites, + both bootstrap/plumbing — `antithesis_hooks.rs` (init + panic hook) and `lading.rs:792` + (bootstrap `reachable!`). No domain-level assertions live in production source, so the panic + hook is the sole no-panic oracle inside the SUT. +- [2026-07-24] The three std-library `unreachable!` sites (`lading.rs:347`, + `splunk_hec/acknowledgements.rs:110`, `otlp/http.rs:227,258`) are core macros, NOT SDK + instrumentation, but they are still subject to the no-panic ADR and would be caught by the + hook if hit. + +## Open Questions + +- Is the `antithesis` feature actually enabled in the scenario Dockerfile so the enabled-arm + hook compiles in? (Noted unverified in the SDK-instrumentation scan.) diff --git a/test/antithesis/scratchbook/properties/no-wall-clock-in-payloads.md b/test/antithesis/scratchbook/properties/no-wall-clock-in-payloads.md new file mode 100644 index 000000000..67684da49 --- /dev/null +++ b/test/antithesis/scratchbook/properties/no-wall-clock-in-payloads.md @@ -0,0 +1,51 @@ +--- +slug: no-wall-clock-in-payloads +subsystem: payload/determinism +assertion_type: Always +priority: P2 +needs_target: false +needs_sut_fix: none currently (guards against regressions) +--- + +# Payload timestamps are seed-derived and monotonic, never wall-clock + +## Statement + +Timestamps embedded in generated payloads (trace_agent, otel, templated_json) are derived from +the rng/config, never from the system wall clock, and are monotone within a stream. + +## Code evidence trail + +- `trace_agent/v04.rs:326,384` — rng-derived timestamps. +- `templated_json/generator.rs:170-196` — `Timestamp` is rng-derived. +- `block.rs` — `Instant` feeds only progress logging, never bytes. + +## Assertion-type rationale + +**Always**: on every timeline payload timestamps must be reproducible and monotone. Oracle: +replay under a perturbed clock (Antithesis clock control) and compare payload timestamps for +equality/monotonicity. + +## Mode / observability + +Payload timestamps are reproducible across runs and unaffected by a backward clock step. +Violation mirrors SMPTNG-767's out-of-order/duplicate timestamp class. + +## Mechanism + +None currently; guards against regressions. + +## needs_sut_fix + +None (regression-guard). + +## Investigation Log + +- [2026-07-24] Confirmed timestamps are rng-derived, not clock-derived (digest payload + finding). Antithesis clock control makes the perturbed-clock replay a strong oracle. +- [2026-07-24] SMPTNG-767 (ADP aggregate stamps payloads with a non-monotonic wall clock) is + the SUT-analog class this property guards against inside lading. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/observer-cpu-max-parse-no-panic.md b/test/antithesis/scratchbook/properties/observer-cpu-max-parse-no-panic.md new file mode 100644 index 000000000..06bc6e9e4 --- /dev/null +++ b/test/antithesis/scratchbook/properties/observer-cpu-max-parse-no-panic.md @@ -0,0 +1,47 @@ +--- +slug: observer-cpu-max-parse-no-panic +subsystem: observer +assertion_type: Unreachable +priority: P0 +needs_target: true +needs_sut_fix: bounds-check cpu.max parsing / guard zero period +--- + +# Observer never panics on malformed/truncated cpu.max + +## Statement + +`parse_allowed_cores` is bounds-checked (no index panic, guards a zero period) so a +malformed/truncated cgroup `cpu.max` degrades rather than aborting. + +## Code evidence trail + +- `stat.rs` — the `cpu.max` parse path (`parse_allowed_cores`); a truncated/malformed read can + index-panic or divide by a zero period. + +## Assertion-type rationale + +**Unreachable**: the index/divide panic must never be reached. Oracle: malformed-cpu.max fault ++ panic hook. + +## Mode / observability + +A truncated `cpu.max` read (fault-injected) does not SIGABRT the run. + +## Mechanism + +Bounds-check parsing / guard a zero period. + +## needs_sut_fix + +Yes. Fix `6aa1b1ba` on backup branch; live on main. + +## Investigation Log + +- [2026-07-24] Same fix commit (`6aa1b1ba`) covers both the PID-reuse assert and the cpu.max + parse; both are live on main. +- [2026-07-24] Antithesis fault injection can truncate the cgroup `cpu.max` read to trigger this. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/observer-pid-identity-fingerprint.md b/test/antithesis/scratchbook/properties/observer-pid-identity-fingerprint.md new file mode 100644 index 000000000..cfcc83d8f --- /dev/null +++ b/test/antithesis/scratchbook/properties/observer-pid-identity-fingerprint.md @@ -0,0 +1,55 @@ +--- +slug: observer-pid-identity-fingerprint +subsystem: observer +assertion_type: Always +priority: P2 +needs_target: true +needs_sut_fix: capture a start-time/identity fingerprint (proc start_time) and validate it in the pidfd/sampler paths +--- + +# Observer samples the identified target, not a PID-reuse impostor + +## Statement + +lading reports target-exit iff the process identified at startup exits; after a watched PID is +recycled, the observer/pidfd path does not silently attach to an unrelated process and emit +wrong metrics. + +## Code evidence trail + +- `target.rs:302-332` — PID mode watch: `kill(pid,0)` validity check (`:314`) and + `AsyncPidFd::from_pid` (`:327`) are separate steps; the process can vanish and its PID be + recycled in between (TOCTOU). +- `target.rs:246-257` — `watch_container` pidfd path. +- observer/linux sampler reads `/proc/{pid}` and would attach to an impostor after reuse. +- `PID_MAX` is small (`2^22`), so reuse is realistic on busy hosts. + +## Assertion-type rationale + +**Always** (metric integrity): on every timeline metrics attributed to the target must +correspond to the original process identity. + +## Mode / observability + +Metrics attributed to the target correspond to the original process identity; a recycled PID +does not produce plausible-but-wrong metrics with no error. Oracle: PID-reuse scenario checking +metric identity. + +## Mechanism + +Capture a start-time/identity fingerprint (proc `start_time`) and validate it in the +pidfd/sampler paths (`target.rs:302-332`, observer sampler). + +## needs_sut_fix + +Yes (defensive; a lead, TOCTOU + small PID_MAX). + +## Investigation Log + +- [2026-07-24] Distinct from `observer-pid-reuse-no-panic`: that one is the abort on mismatch; + this is the silent-wrong-metrics case where no assert catches an impostor attachment. + +## Open Questions + +- Should lading capture a start-time/identity fingerprint to defend the observer sampler and + pidfd path against PID reuse? diff --git a/test/antithesis/scratchbook/properties/observer-pid-reuse-no-panic.md b/test/antithesis/scratchbook/properties/observer-pid-reuse-no-panic.md new file mode 100644 index 000000000..e3d3d4c86 --- /dev/null +++ b/test/antithesis/scratchbook/properties/observer-pid-reuse-no-panic.md @@ -0,0 +1,48 @@ +--- +slug: observer-pid-reuse-no-panic +subsystem: observer +assertion_type: Unreachable +priority: P0 +needs_target: true +needs_sut_fix: replace the assert! at stat.rs:82 with a skip-and-continue +--- + +# Observer never aborts on PID reuse / mismatch + +## Statement + +The stat sampler degrades (skips the stale sample) instead of asserting `cur_pid==pid` when a +recycled/mismatched PID is read. + +## Code evidence trail + +- `stat.rs:82` — `assert!(cur_pid == pid)`. + +## Assertion-type rationale + +**Unreachable**: the assert must never fire. Oracle: rapid target-exit/PID-reuse scenario + +panic hook. + +## Mode / observability + +Target exit + PID recycle races do not SIGABRT the run. Confirmed live: `assert!(cur_pid == +pid)` at `stat.rs:82`. + +## Mechanism + +Replace the `assert!` at `stat.rs:82` with a skip-and-continue. + +## needs_sut_fix + +Yes. Fix `6aa1b1ba` on backup branch; live on main. + +## Investigation Log + +- [2026-07-24] Verified against source this turn: `stat.rs:82` `assert!(cur_pid == pid)` is live + on main; `panic=abort` makes any hit an observable abort. +- [2026-07-24] Same fix commit (`6aa1b1ba`) also addresses the cpu.max parse (see + `observer-cpu-max-parse-no-panic`). + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/observer-process-vanish-no-panic.md b/test/antithesis/scratchbook/properties/observer-process-vanish-no-panic.md new file mode 100644 index 000000000..89cffc5a5 --- /dev/null +++ b/test/antithesis/scratchbook/properties/observer-process-vanish-no-panic.md @@ -0,0 +1,47 @@ +--- +slug: observer-process-vanish-no-panic +subsystem: observer +assertion_type: Unreachable +priority: P0 +needs_target: true +needs_sut_fix: yield empty iterator instead of panic! at process_descendents.rs:13 +--- + +# Observer never panics when a target vanishes mid-listing + +## Statement + +`ProcessDescendantsIterator` degrades to zero descendants when `Process::new` fails, rather than +panicking, when a target exits before descendant listing. + +## Code evidence trail + +- `process_descendents.rs:13` — `panic!` when `Process::new` fails, instead of yielding an + empty iterator. + +## Assertion-type rationale + +**Unreachable**: the `panic!` must never be reached. Oracle: target-exits-during-sampling +scenario + panic hook. + +## Mode / observability + +A target exiting mid-listing does not crash the run. + +## Mechanism + +Yield an empty iterator (degrade to zero descendants) instead of `panic!` at +`process_descendents.rs:13`. + +## needs_sut_fix + +Yes. Fix `7e1d2968` on backup branch; live on main. + +## Investigation Log + +- [2026-07-24] Confirmed live on main: `ProcessDescendantsIterator::new` panics when + `Process::new` fails, so a target that exits before descendant listing crashes the run. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/observer-target-pid-recv-no-panic.md b/test/antithesis/scratchbook/properties/observer-target-pid-recv-no-panic.md new file mode 100644 index 000000000..db489d3b6 --- /dev/null +++ b/test/antithesis/scratchbook/properties/observer-target-pid-recv-no-panic.md @@ -0,0 +1,53 @@ +--- +slug: observer-target-pid-recv-no-panic +subsystem: no-panic +assertion_type: Unreachable +priority: P0 +needs_target: true +needs_sut_fix: handle recv() Err and None with a returned error at observer.rs:114-120 +--- + +# Observer returns an error, not a panic, when the target PID never arrives + +## Statement + +If a Binary target fails to spawn or exits before sending its PID, the observer returns an error +instead of `.expect("catastrophic failure")` panicking on the closed channel. + +## Code evidence trail + +- `observer.rs:114-120` — two `.expect()` on the `recv()` result and the `Option`. +- Triggered by `target.rs:395` (TargetSpawn error before send) or `:396` (ProcessFinished before + send): the target's `tgt_snd` is dropped, `recv()` returns `Err(Closed)`, and `.expect` + panics. +- The panic is only partly masked in main: `osrv_joinset.join_next()` gets `Err(JoinError)` and + merely logs "Could not join the spawned observer task" (`lading.rs:675`). + +## Assertion-type rationale + +**Unreachable**: the closed-channel `.expect` must never be reached. Oracle: instant-exit/ +bad-path target scenario + panic hook. + +## Mode / observability + +A bad target path / instant-exit target does not panic the observer task. Currently the +`recv()` `Err(Closed)` hits `.expect`. + +## Mechanism + +Handle `recv()` `Err` and `None` with a returned error at `observer.rs:114-120`. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed live on main: two `.expect` on the PID channel recv; a Binary target + that fails to spawn or exits before sending its PID drops the sender and panics the observer. +- [2026-07-24] MOOT on the production observer path (Container mode gets the PID via the docker + socket, not this channel), LIVE for Binary-target mode. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/observer-transient-read-not-fatal.md b/test/antithesis/scratchbook/properties/observer-transient-read-not-fatal.md new file mode 100644 index 000000000..da4445b8c --- /dev/null +++ b/test/antithesis/scratchbook/properties/observer-transient-read-not-fatal.md @@ -0,0 +1,48 @@ +--- +slug: observer-transient-read-not-fatal +subsystem: observer +assertion_type: Always +priority: P1 +needs_target: true +needs_sut_fix: treat component reads as best-effort in observer/linux.rs sample() +--- + +# A transient observer read error does not kill the run + +## Statement + +A single transient procfs/cgroup/wss read error is best-effort (log + skip that component's +sample); it does not `?`-propagate and terminate the whole experiment. + +## Code evidence trail + +- `observer/linux.rs` — `sample()` `?`-propagates a component read error, killing the whole + experiment. + +## Assertion-type rationale + +**Always** (liveness): on every timeline a single transient read error must not terminate the +run. Oracle: transient-read-fault scenario asserting the run continues to self-termination. + +## Mode / observability + +An injected transient read error yields a skipped sample + warning, not a dead run. Only +persistent problems show as repeated warnings + absent metrics. + +## Mechanism + +Treat component reads as best-effort in `observer/linux.rs` `sample()` (fix `30b86a71` on backup +branch; live on main). + +## needs_sut_fix + +Yes. `30b86a71` on backup branch; live on main. + +## Investigation Log + +- [2026-07-24] Confirmed: a single transient procfs/cgroup/wss read error `?`-propagates and + kills the whole experiment on main. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/orphaned-children-on-signal-death.md b/test/antithesis/scratchbook/properties/orphaned-children-on-signal-death.md new file mode 100644 index 000000000..ca43aa88c --- /dev/null +++ b/test/antithesis/scratchbook/properties/orphaned-children-on-signal-death.md @@ -0,0 +1,55 @@ +--- +slug: orphaned-children-on-signal-death +subsystem: lifecycle/shutdown +assertion_type: Always +priority: P1 +needs_target: true +needs_sut_fix: install SIGTERM handler and signal the process group, not just the direct child pid +--- + +# No orphaned target/inspector children on signal-driven death + +## Statement + +lading reaps its Binary-target and inspector children (and their process groups) even when +killed by a signal, rather than relying solely on `kill_on_drop` which cannot fire on +untrapped-signal death. + +## Code evidence trail + +- `target.rs:392` `kill_on_drop(true)`; `:430-431` `kill(target_id, SIGTERM)` to a single pid. +- `inspector.rs:146` `kill_on_drop(true)`; `:175-176` SIGTERM to a single pid. +- `kill_on_drop` cannot fire when lading is killed by an untrapped signal (SIGTERM per + `sigterm-graceful-drain`, or SIGKILL); children are reparented to init. The graceful path only + signals the direct child, not the process group, so grandchildren are never reaped. + +## Assertion-type rationale + +**Always** (no-leak): on every signal-driven death, no orphaned target/inspector grandchildren +remain reparented to init. + +## Mode / observability + +After a SIGTERM (or crash) no orphaned target/inspector grandchildren remain reparented to init. +MOOT under Antithesis whole-container SIGKILL (all pids die together); LIVE in shared-PID- +namespace / bare-host deployments. Oracle: SIGTERM scenario in a shared PID namespace checking +for surviving children. + +## Mechanism + +Install a SIGTERM handler (see `sigterm-graceful-drain`) and send signals to the process group, +not just the direct child pid (`target.rs:430-431`, `inspector.rs:175-176`). + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] MOOT on the production observer path (target is a sibling container SIGKILLed at + teardown; whole container dies together). LIVE in shared-PID-namespace or bare-host + deployments where lading spawns a Binary target/inspector. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/parquet-footer-on-graceful-exit.md b/test/antithesis/scratchbook/properties/parquet-footer-on-graceful-exit.md new file mode 100644 index 000000000..9ff1518f5 --- /dev/null +++ b/test/antithesis/scratchbook/properties/parquet-footer-on-graceful-exit.md @@ -0,0 +1,59 @@ +--- +slug: parquet-footer-on-graceful-exit +subsystem: capture +assertion_type: Always +priority: P0 +needs_target: false +needs_sut_fix: none on the pure graceful path (depends on sigterm-graceful-drain + capture-write-failure-not-abort) +--- + +# Graceful exit always yields a readable parquet capture + +## Statement + +After any graceful termination (experiment timer or SIGTERM) the parquet/multi capture file has +a finalized footer and is fully readable. + +## Code evidence trail + +- `parquet.rs:307-324` — the footer is written only in `close()`; doc: "Without calling close() + the file will be incomplete and unreadable". +- `state_machine.rs:249-259` — `close()` is only reached on `Event::ShutdownSignaled`. + +## Assertion-type rationale + +**Always**: every graceful exit must produce a readable file. Oracle is the existing +`anytime_capture_consistent` checker's parquet arm (readable => internally consistent) plus a +post-graceful-exit readability assertion. + +## Mode / observability + +External reader opens `/captures/captures.parquet` post-exit and parses all row groups. This is +the operational invariant the deployment depends on (watchdog/cancel/kill => unreadable => +capture loss). + +## Mechanism + +None on the pure graceful path; depends on `sigterm-graceful-drain` and +`capture-write-failure-not-abort` holding. + +## needs_sut_fix + +Indirect: this property holds on the pure graceful path as-is, but its practical guarantee +depends on `sigterm-graceful-drain` (so SIGTERM takes the graceful path) and +`capture-write-failure-not-abort` (so a write error does not abort mid-flush). + +## Investigation Log + +- [2026-07-24] Existing harness oracle: `anytime_capture_consistent.rs:66` asserts + `!readable || invariants_hold` and `:71` `sometimes! readable && records>=10`. This property + adds the post-graceful-exit readability assertion. +- [2026-07-24] Deployment: on the success path lading self-exits 0, writes the footer, then RJO + copies the parquet off BEFORE force-killing the pod — so teardown SIGKILL is harmless because + lading already finished. + +## Open Questions + +- Does the parquet writer periodically finalize a readable footer on `flush_seconds:60`, or + only at graceful close? If only at close, EVERY non-graceful stop loses ALL captured data, + not just the last 60s. (Digest orientation: footer only at close.) diff --git a/test/antithesis/scratchbook/properties/payload-determinism-byte-identical.md b/test/antithesis/scratchbook/properties/payload-determinism-byte-identical.md new file mode 100644 index 000000000..7a5f88350 --- /dev/null +++ b/test/antithesis/scratchbook/properties/payload-determinism-byte-identical.md @@ -0,0 +1,61 @@ +--- +slug: payload-determinism-byte-identical +subsystem: payload/determinism +assertion_type: Always +priority: P1 +needs_target: false +needs_sut_fix: none expected (guards against regressions) +--- + +# Same seed + config yields byte-identical load + +## Statement + +For a fixed seed and config, the sequence of bytes lading emits is identical across runs; no +wall-clock, HashMap iteration order, or hidden entropy feeds payload bytes. + +## Code evidence trail + +- Ordered output uses `BTreeMap`/`BTreeSet`: `fluent.rs:108`, `trace_agent/v04.rs:209-222`, + `opentelemetry/trace.rs:1332`. +- `templated_json` uses `FxHashMap` (`resolver.rs:14`) but iteration only assigns internal def + indices resolved by name (`resolver.rs:120-129,178-186`), so output is index-agnostic and + FxHasher is fixed-seed. +- dogstatsd tags use `HashSet` only for uniqueness checks, never iterated for output; selection + is via an indexed `Vec` (`common/tags.rs:34-63`). +- Timestamps are rng-derived: `trace_agent/v04.rs:326,384`, + `templated_json/generator.rs:170-196`. +- `block.rs:614,659` — `tokio::time::Instant` feeds only progress logging, never bytes. + +## Assertion-type rationale + +**Always** (determinism): every pair of same-seed runs must produce identical bytes. Oracle: +byte-equality across two seeded runs, or a SUT `always!` on a rolling hash of emitted blocks. + +## Mode / observability + +Two runs with identical seed/config produce identical byte streams at the sink (or identical +block-cache hashes). Violation breaks the determinism ADR and Antithesis reproducibility. + +## Mechanism + +None expected (BTreeMap/BTreeSet ordering, rng-derived timestamps confirmed); the property +guards against regressions introducing wall-clock/entropy. + +## needs_sut_fix + +None (regression-guard). + +## Investigation Log + +- [2026-07-24] Digest confirms byte output is a pure function of `(seed, config)`; no + wall-clock or HashMap iteration feeds payload bytes. +- [2026-07-24] Lead analog: SMPTNG-762 (AWS-LC CPU-jitter entropy aborts under deterministic + execution) and SMPTNG-767 (non-monotonic wall clock) are SUT bugs whose class must not exist + in lading — hunt lading's own crypto/entropy/time usage for the same pattern. + +## Open Questions + +- No determinism invariant is currently expressed as an SDK `always!` anywhere; the harness + relies on the sink byte counter and config sampling. Should a direct byte-equality oracle be + added? diff --git a/test/antithesis/scratchbook/properties/per-generator-semaphore-no-panic.md b/test/antithesis/scratchbook/properties/per-generator-semaphore-no-panic.md new file mode 100644 index 000000000..f44739bbd --- /dev/null +++ b/test/antithesis/scratchbook/properties/per-generator-semaphore-no-panic.md @@ -0,0 +1,54 @@ +--- +slug: per-generator-semaphore-no-panic +subsystem: no-panic +assertion_type: Unreachable +priority: P1 +needs_target: true +needs_sut_fix: replace static CONNECTION_SEMAPHORE OnceCell with a per-instance Arc; make hot-path expect stop the worker gracefully +--- + +# Two HTTP (or two Splunk-HEC) generators coexist without panic + +## Statement + +Each HTTP/Splunk-HEC generator instance owns its own connection semaphore; configuring two such +generators does not panic on a process-wide `OnceCell::set` and gives independent connection +limits. + +## Code evidence trail + +- `http.rs:37` + `http.rs:187-189` — `static CONNECTION_SEMAPHORE: OnceCell` set via + `.set(..).expect(..)`. +- `splunk_hec.rs:51` + `splunk_hec.rs:243-245` — same pattern. +- `config.rs:101` — `pub generator: Vec`, so two Http generators is a valid + config; the second `new()` panics on the second `OnceCell::set`. + +## Assertion-type rationale + +**Unreachable**: the `OnceCell` double-set panic must never be reached. Oracle: two-http- +generator scenario + panic hook. + +## Mode / observability + +A config with two `Http` generators starts without the second `new()` panicking; per-generator +concurrency limits are independent. Even short of panic, the shared semaphore makes the +per-generator concurrency limit wrong. + +## Mechanism + +Replace the static `CONNECTION_SEMAPHORE` `OnceCell` with a per-instance `Arc` in +`http.rs:37/187-189` and `splunk_hec.rs:51/243-245`; make the hot-path +`.expect("semaphore closed")` stop the worker gracefully. + +## needs_sut_fix + +Yes. Landed on backup branch (`5f8c375e`); live on main. + +## Investigation Log + +- [2026-07-24] Confirmed live regression on main: only duplicate generator IDs are rejected, so + two Http (or two SplunkHec) generators is an allowed config that panics at the second `new()`. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/recorded-traffic-crash-consistency.md b/test/antithesis/scratchbook/properties/recorded-traffic-crash-consistency.md new file mode 100644 index 000000000..1d0c03f3d --- /dev/null +++ b/test/antithesis/scratchbook/properties/recorded-traffic-crash-consistency.md @@ -0,0 +1,50 @@ +--- +slug: recorded-traffic-crash-consistency +subsystem: blackhole/config +assertion_type: Always +priority: P2 +needs_target: true +needs_sut_fix: none identified (LEAD to validate zstd framing / flush boundaries of the recorder) +--- + +# Blackhole-recorded traffic files are crash-consistent + +## Statement + +The blackhole traffic recorder writes files that, when compressed, remain decodable up to the +last flushed frame after an abrupt kill, and are deterministic for a fixed input. + +## Code evidence trail + +- Deployment/history: SMPTNG-390 (blackhole traffic recorder) and the recent record-policy work + (commits #1911, #1895 OpenMetrics scrape bodies). +- Recorded-traffic files are a crash-consistency + determinism surface analogous to jsonl. + +## Assertion-type rationale + +**Always** (crash-consistency): on every timeline a recorder file from a killed blackhole must +decode to a valid prefix. Oracle: `node_termination` scenario + external decode check. + +## Mode / observability + +A recorder file from a SIGKILLed blackhole decodes to a valid prefix (analogous to jsonl), not +undecodable-past-last-frame corruption. + +## Mechanism + +None identified; LEAD to validate zstd framing / flush boundaries of the recorder (SMPTNG-390 / +record-policy work #1911/#1895). + +## needs_sut_fix + +None identified yet (a LEAD to validate). + +## Investigation Log + +- [2026-07-24] Related to SMPTNG-694 (captures now zstd-only): a truncated zstd stream may be + undecodable past the last flushed frame, so the recorder's flush/framing boundaries determine + whether a truncated file is a valid prefix. + +## Open Questions + +- Are recorded-traffic files zstd-framed such that a truncated stream is a valid prefix? diff --git a/test/antithesis/scratchbook/properties/shutdown-completes-bounded.md b/test/antithesis/scratchbook/properties/shutdown-completes-bounded.md new file mode 100644 index 000000000..3994356ce --- /dev/null +++ b/test/antithesis/scratchbook/properties/shutdown-completes-bounded.md @@ -0,0 +1,63 @@ +--- +slug: shutdown-completes-bounded +subsystem: lifecycle/shutdown +assertion_type: Always +priority: P0 +needs_target: true +needs_sut_fix: shutdown branches in every connect/retry loop + timeout on child wait and capture finalize +--- + +# lading exits within max_shutdown_delay after shutdown is signaled + +## Statement + +Once the experiment timer fires (or a shutdown signal is broadcast) lading terminates within +`max_shutdown_delay`; it never hangs in `Server::spin` waiting on a worker that cannot observe +shutdown. + +## Code evidence trail + +- Generator connect/retry loops with no shutdown branch: `tcp.rs:232-251`, `udp.rs`, + `unix_stream.rs:248-268`, `unix_datagram.rs:246-264`, `grpc.rs:287-299`. +- `target.rs:432` and `inspector.rs:176` — bare `.await` on child `wait()` (untimed). +- `lading.rs:713-715` — unbounded `let _ = handle.await;` on capture finalize before + `runtime.shutdown_timeout` at `:813`. +- `lading.rs` — `max_shutdown_delay` (30s) feeds `runtime.shutdown_timeout`. + +## Assertion-type rationale + +**Always** (bounded-latency invariant on every timeline): once shutdown is signaled, exit must +follow within `max_shutdown_delay`. Umbrella liveness for shutdown hangs; per-loop properties +carry the specific mechanisms. + +## Mode / observability + +Wall time from shutdown-broadcast to process exit `<= max_shutdown_delay` (30s). Violation = +overrun -> deployment watchdog SIGKILL -> unreadable parquet -> total capture loss for the +replicate. Oracle: external timer measuring shutdown-signal->exit, or Antithesis liveness +"eventually exits". + +## Mechanism + +Add shutdown branches to every pre-select connect/retry loop (tcp/udp/unix_stream/ +unix_datagram/grpc) and a timeout to `target_child.wait()` and the capture-finalize await; see +the per-loop child properties. + +## needs_sut_fix + +Yes — this umbrella is satisfied by the combined per-loop fixes plus the bounded child-wait +and bounded capture-finalize timeouts. + +## Investigation Log + +- [2026-07-24] Deployment watchdog makes this an operational invariant: RJO arms + `ceil((warmup + total_samples + 30) * 1.2)` seconds; overrunning lading is SIGKILLed + mid-run, leaving a footer-less unreadable parquet = total capture loss for the replicate. +- [2026-07-24] Strongest deployment-confirmed lead: SMPTNG-725 "RJO alive but not really" — + hang-in-spin after "lading shutdown". + +## Open Questions + +- Does the graceful-shutdown driver in `lading.rs` impose a timeout on generator + `Server::spin()` completion, or does it rely solely on `runtime.shutdown_timeout(30s)`? That + determines whether the connect-loop hangs stall shutdown up to 30s or indefinitely. diff --git a/test/antithesis/scratchbook/properties/sigterm-graceful-drain.md b/test/antithesis/scratchbook/properties/sigterm-graceful-drain.md new file mode 100644 index 000000000..16e663258 --- /dev/null +++ b/test/antithesis/scratchbook/properties/sigterm-graceful-drain.md @@ -0,0 +1,63 @@ +--- +slug: sigterm-graceful-drain +subsystem: lifecycle/shutdown +assertion_type: Always +priority: P0 +needs_target: false +needs_sut_fix: add SignalKind::terminate arm to main select +--- + +# SIGTERM finalizes capture like the experiment timer + +## Statement + +When lading receives SIGTERM it runs the same graceful path as experiment-timer +self-termination: broadcast shutdown, drain the capture maturity window, write the +parquet/multi footer, exit non-abnormally. + +## Code evidence trail + +- `lading/src/bin/lading.rs:658` — the main `select!` has only a `signal::ctrl_c()` (SIGINT) + arm; grep finds no `tokio::signal::unix` `SignalKind::terminate` anywhere. +- With no SIGTERM arm, SIGTERM uses default disposition -> lading dies immediately (exit 143) + with NO graceful path: `shutdown_broadcast` never fires, capture is never finalized, child + destructors (`kill_on_drop`) never run. +- Graceful path for reference: `lading_signal` `signal_and_wait` -> capture finalize -> + `runtime.shutdown_timeout`. + +## Assertion-type rationale + +**Always**: the graceful contract must hold every time a SIGTERM is delivered. The best oracle +is external — send SIGTERM mid-run, then validate the capture file is footer-complete and the +process exited without abort. + +## Mode / observability + +After a SIGTERM the on-disk parquet/multi capture is readable (footer present) and lading +exits 143/0, not abort. MOOT under Antithesis `node_termination` (SIGKILL is untrappable) but +LIVE on the deployment's orchestrator-stop path. + +## Mechanism + +Add a `tokio::signal::unix` `SignalKind::terminate` arm in the `lading.rs` main `select` +(alongside `ctrl_c` at :658) that triggers `shutdown_broadcast`. + +## needs_sut_fix + +Yes. Add the `SignalKind::terminate` arm to the main select that triggers the same +`shutdown_broadcast` as the experiment timer and ctrl_c. + +## Investigation Log + +- [2026-07-24] Confirmed live on main: only ctrl_c is trapped at `lading.rs:658`; no SIGTERM + handler exists (grep). +- [2026-07-24] Deployment context: the production runner never sends lading SIGTERM — lading + self-terminates on its timer or is SIGKILLed via `docker rm --force`. So this handler is + effectively dead code on the production path, but the deployment tickets SMPTNG-719/697 + document ungraceful-termination telemetry loss, and a plain orchestrator `docker stop` + (SIGTERM) would corrupt the parquet and orphan children today. + +## Open Questions + +- Does Antithesis ever deliver SIGTERM (vs only SIGKILL `node_termination`)? If not, this is + exercised only by an explicit harness `kill -TERM` step. diff --git a/test/antithesis/scratchbook/properties/sink-receives-bytes.md b/test/antithesis/scratchbook/properties/sink-receives-bytes.md new file mode 100644 index 000000000..41c2bbfb0 --- /dev/null +++ b/test/antithesis/scratchbook/properties/sink-receives-bytes.md @@ -0,0 +1,48 @@ +--- +slug: sink-receives-bytes +subsystem: generators/throttle +assertion_type: Sometimes +priority: P1 +needs_target: true +needs_sut_fix: none (already instrumented) +--- + +# The sink receives load (load-arrival non-vacuity) + +## Statement + +Across a run the sink container receives a nonzero number of bytes from lading generators. + +## Code evidence trail + +- `sink/main.rs:82` — `sometimes! (total>0)` "sink received bytes". +- `sink/main.rs:69-83` — `handle_connection` records bytes and asserts `total>0`; SDK init at + `:32`. + +## Assertion-type rationale + +**Sometimes** (non-vacuity of delivery on at least one timeline): guards against a whole-config +class delivering nothing (divide stall, capacity livelock, throttle bypass). + +## Mode / observability + +`sink/main.rs:82` `sometimes! (total>0)` "sink received bytes". The oracle is already present in +the never-faulted sink container. + +## Mechanism + +Sometimes (non-vacuity of delivery). Oracle already present in the never-faulted sink container. + +## needs_sut_fix + +None (already instrumented). + +## Investigation Log + +- [2026-07-24] Already wired: the sink container owns the "load arrived" assertion. This is the + cross-check that catches the throttle-divide / capacity-livelock / gRPC-throttle-bypass + classes that would otherwise silently deliver zero bytes. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/split-mode-merge-partial-tolerant.md b/test/antithesis/scratchbook/properties/split-mode-merge-partial-tolerant.md new file mode 100644 index 000000000..27c1e9727 --- /dev/null +++ b/test/antithesis/scratchbook/properties/split-mode-merge-partial-tolerant.md @@ -0,0 +1,55 @@ +--- +slug: split-mode-merge-partial-tolerant +subsystem: capture +assertion_type: Always +priority: P2 +needs_target: true +needs_sut_fix: none in lading proper (LEAD to validate against the deployment merge + per-instance finalize) +--- + +# Split-mode capture merge tolerates one clean side when the other overruns + +## Statement + +In split mode, if only the sender (or only the receiver) lading overruns/crashes leaving an +unreadable parquet, the merge outcome and replicate-failure attribution are well-defined (a +clean side's captures are not needlessly discarded by the other's corruption). + +## Code evidence trail + +- Deployment `capture_file_merge` oblivious merge reads row batches and validates schema; a + footerless (killed-lading) file fails to read and errors the replicate. +- Digest deployment findings: merge tolerates one-empty/both-empty inputs but NOT a + corrupt/truncated parquet. + +## Assertion-type rationale + +**Always** (merge robustness): on every timeline the merge/attribution must be well-defined; a +clean side must not be needlessly lost to the other's corruption. + +## Mode / observability + +Merge of a truncated file + a clean file behaves per policy (fail attributed to the corrupt +side), not a silent whole-replicate loss when one side captured cleanly. Oracle: split-mode +scenario killing only one side. + +## Mechanism + +None in lading proper; DEPLOYMENT-DERIVED LEAD to validate against source (the oblivious merge + +lading's per-instance capture finalize). Do not name the deployment. + +## needs_sut_fix + +None in lading proper (this is a merge-policy question in the deployment layer + lading's +per-instance finalize). + +## Investigation Log + +- [2026-07-24] Split mode (SMPTNG-721): the sender lading runs with `--no-target` and writes its + own parquet on a separate pod; the two files are merged. The merge deletes a pre-existing + parquet before glob-reading to avoid stale/merge confusion. +- [2026-07-24] Marked as a LEAD to validate; not a confirmed lading defect. + +## Open Questions + +- Does a sender-only overrun fail the whole replicate when the receiver captured cleanly? diff --git a/test/antithesis/scratchbook/properties/splunk-hec-response-parse-no-panic.md b/test/antithesis/scratchbook/properties/splunk-hec-response-parse-no-panic.md new file mode 100644 index 000000000..ba122b8d7 --- /dev/null +++ b/test/antithesis/scratchbook/properties/splunk-hec-response-parse-no-panic.md @@ -0,0 +1,51 @@ +--- +slug: splunk-hec-response-parse-no-panic +subsystem: no-panic +assertion_type: Unreachable +priority: P1 +needs_target: true +needs_sut_fix: replace serde_json::from_slice::().expect() with error handling; track the detached task's handle +--- + +# Splunk-HEC response parsing never panics on a non-HecResponse body + +## Statement + +The Splunk-HEC generator's spawned request task handles a non-HecResponse body (empty, HTML +error page, "ok") without panicking via `.expect` on serde_json parse. + +## Code evidence trail + +- `splunk_hec.rs:371-374` — `serde_json::from_slice::(&body_bytes).expect("unable + to parse response body")`. Any target/blackhole returning a non-HecResponse body panics the + detached task. + +## Assertion-type rationale + +**Unreachable**: the parse `.expect` must never be reached with a non-conforming body. Oracle: +blackhole returning a plain-text body + panic hook. + +## Mode / observability + +A blackhole/target returning a non-JSON body does not abort the detached task. `panic=abort` +makes this whole-process fatal. + +## Mechanism + +Replace `serde_json::from_slice::(...).expect(...)` at `splunk_hec.rs:371-374` with +error handling; also track the detached task's handle. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed: the response body is parsed with `.expect`; a non-HecResponse body + panics the detached task, and `panic=abort` makes it whole-process fatal. +- [2026-07-24] The detached task is also untracked — a known related defect (splunk_hec + detaches untracked tasks). + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/sqs-receive-message-bounded.md b/test/antithesis/scratchbook/properties/sqs-receive-message-bounded.md new file mode 100644 index 000000000..78c4836e6 --- /dev/null +++ b/test/antithesis/scratchbook/properties/sqs-receive-message-bounded.md @@ -0,0 +1,50 @@ +--- +slug: sqs-receive-message-bounded +subsystem: blackhole/config +assertion_type: Always +priority: P2 +needs_target: false +needs_sut_fix: clamp num_messages to a max (e.g. 10) before the 0..num_messages loop +--- + +# SQS blackhole bounds ReceiveMessage response size + +## Statement + +The SQS blackhole caps `max_number_of_messages` so a single target-controlled request cannot +force an enormous allocation and OOM the blackhole. + +## Code evidence trail + +- `sqs.rs:257-267` — `num_messages = rm.max_number_of_messages` parsed straight from the request + (a `u32`) with no cap. +- `sqs.rs:362-370` — `generate_receive_message_response` loops `for _ in 0..num_messages` + building ~450-byte message strings. Real SQS caps at 10; a value up to `u32::MAX` causes an + enormous String allocation. + +## Assertion-type rationale + +**Always** (amplification/OOM): on every timeline a single request must not force an unbounded +allocation. Oracle: adversarial-request scenario + memory observation. + +## Mode / observability + +A request with a huge `max_number_of_messages` produces a bounded response (real SQS caps at +10), not an unbounded String allocation. + +## Mechanism + +Clamp `num_messages` to a max (e.g. 10) at `sqs.rs:257-267` before the `0..num_messages` loop. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed: the response size is unbounded by a target-controlled `u32`, enabling + memory/CPU amplification from a single request. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/stable-burst-envelope-bounded.md b/test/antithesis/scratchbook/properties/stable-burst-envelope-bounded.md new file mode 100644 index 000000000..84992b1c6 --- /dev/null +++ b/test/antithesis/scratchbook/properties/stable-burst-envelope-bounded.md @@ -0,0 +1,51 @@ +--- +slug: stable-burst-envelope-bounded +subsystem: generators/throttle +assertion_type: Always +priority: P1 +needs_target: false +needs_sut_fix: none if Kani proofs hold; add feature-gated assert_always! in stable.rs to catch regressions +--- + +# Stable throttle never exceeds its per-interval burst envelope + +## Statement + +In the real async `wait_for` path under an adversarial clock, the stable throttle grants at most +`maximum_capacity` at `timeout=0` (no over-delivery) and at most `(MAX_ROLLED_INTERVALS+1)x` +with rolled capacity. + +## Code evidence trail + +- `stable.rs` — the async `wait_for` grant path. +- Digest `3f4a6bd2` — feature-gated SUT assertion + proptests demonstrating the envelope. + +## Assertion-type rationale + +**Always** (rate safety): on every timeline, including adversarial-clock timelines, the granted +capacity per interval must stay within the proven envelope. SUT-side feature-gated +`assert_always!` under an adversarial clock (Antithesis controls time) + proptests. + +## Mode / observability + +Granted capacity per interval stays within the proven envelope; a clock-perturbation-induced +over-grant is a defect. Envelope: `== maximum_capacity` at `timeout=0` (no over-delivery), +`<= (MAX_ROLLED_INTERVALS+1)x` with rolled capacity (up to 11x). + +## Mechanism + +None if the Kani proofs hold; add a feature-gated `assert_always!` in `stable.rs` (landed on +backup branch `3f4a6bd2`) to catch regressions. + +## needs_sut_fix + +None expected (Kani proofs); the feature-gated assertion is a regression guard. + +## Investigation Log + +- [2026-07-24] The throttle has Kani proofs per the ADRs; this property is the runtime + regression guard under Antithesis clock control, complementing the static proofs. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/target-grace-period-honored.md b/test/antithesis/scratchbook/properties/target-grace-period-honored.md new file mode 100644 index 000000000..c5c0ce205 --- /dev/null +++ b/test/antithesis/scratchbook/properties/target-grace-period-honored.md @@ -0,0 +1,61 @@ +--- +slug: target-grace-period-honored +subsystem: lifecycle/shutdown +assertion_type: Always +priority: P1 +needs_target: true +needs_sut_fix: join the target task with a max_shutdown_delay-bounded wait after signaling, instead of dropping the JoinSet immediately +--- + +# A cooperative slow target gets its full post-SIGTERM cleanup window + +## Statement + +On graceful shutdown the target receives SIGTERM and is given up to `max_shutdown_delay` to +clean up/flush before SIGKILL, rather than being SIGKILLed within milliseconds when capture +flush returns. + +## Code evidence trail + +- `lading.rs:690-717` — on graceful/ctrl_c shutdown the main loop breaks, then `inner_main` runs + `signal_and_wait` + capture flush and RETURNS without ever joining `tsrv_joinset`. Returning + drops the JoinSet, aborting the still-running target task at `target_child.wait().await`; + dropping `target_child` triggers `kill_on_drop` -> SIGKILL. +- `target.rs:392` `kill_on_drop(true)`; `:425-433` SIGTERM then wait (comment "give the child a + chance to clean up"). +- `max_shutdown_delay` is only applied later at `runtime.shutdown_timeout`, by which point the + target is already killed. + +## Assertion-type rationale + +**Always** (grace contract): on every graceful shutdown a cooperative target must get its full +cleanup window before SIGKILL. + +## Mode / observability + +A cooperative-but-slow target completes its cleanup/artifact flush before being killed. +Currently `inner_main` returns after capture flush, dropping `tsrv_joinset` and aborting +`target_child.wait` -> `kill_on_drop` SIGKILL. Oracle: SIGTERM-then-slow-cleanup target scenario +checking the target's artifacts are complete. + +## Mechanism + +Join the target task with a `max_shutdown_delay`-bounded wait after signaling, instead of +dropping the JoinSet immediately (`lading.rs:690-717`). + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed the ~0 grace: `inner_main` returns after capture flush (milliseconds), + dropping the JoinSet and triggering `kill_on_drop` SIGKILL long before + `runtime.shutdown_timeout` could gate the cleanup window. +- [2026-07-24] MOOT on the production observer path (target is a sibling container), LIVE for + Binary-target mode. Confirm intended grace semantics before proposing a fix. + +## Open Questions + +- Is the ~0 grace intentional, or is `max_shutdown_delay` meant to gate the target's post-SIGTERM + cleanup window? diff --git a/test/antithesis/scratchbook/properties/target-wait-bounded.md b/test/antithesis/scratchbook/properties/target-wait-bounded.md new file mode 100644 index 000000000..5636f86af --- /dev/null +++ b/test/antithesis/scratchbook/properties/target-wait-bounded.md @@ -0,0 +1,59 @@ +--- +slug: target-wait-bounded +subsystem: lifecycle/shutdown +assertion_type: Always +priority: P0 +needs_target: true +needs_sut_fix: wrap target/inspector child wait() in a timeout that escalates to SIGKILL +--- + +# Post-SIGTERM target/inspector wait is time-bounded + +## Statement + +After lading SIGTERMs a Binary target (or inspector), the `wait()` for the child to exit is +bounded (does not depend solely on external JoinSet-abort/runtime timeout); a SIGTERM-ignoring +target does not hang lading. + +## Code evidence trail + +- `target.rs:432` — bare `target_child.wait().await` after SIGTERM, no timeout of its own. +- `inspector.rs:176` — bare inspector `wait().await`. +- `target.rs:430-431`, `inspector.rs:175` — SIGTERM sent to a single pid (see + `orphaned-children-on-signal-death`). +- Safety currently depends on the external JoinSet-abort / `runtime.shutdown_timeout` backstop. + +## Assertion-type rationale + +**Always** (locally-bounded reap): the reap must be bounded on every timeline, not contingent +on unrelated shutdown wiring holding. + +## Mode / observability + +With a target that ignores SIGTERM, lading still exits within `max_shutdown_delay`. MOOT on the +deployment's container-observer path (target is a sibling container force-killed by the +runner), LIVE for Binary-target mode. Oracle: SIGTERM-ignoring-target scenario + exit timer. + +## Mechanism + +Wrap `target_child.wait()` (`target.rs:432`) and the inspector wait (`inspector.rs:176`) in a +timeout that escalates to SIGKILL before returning. + +## needs_sut_fix + +Yes. The waits are untimed today. + +## Investigation Log + +- [2026-07-24] Deployment context: production uses Container observer mode (`--target-container`), + NOT Binary target mode, so lading does not spawn/SIGTERM/reap the target on the production + path — the target is a sibling container RJO hard-kills. This property is LIVE only for + Binary-target mode (e.g. local dev / other deployments). +- [2026-07-24] Open orientation question flagged: whether tokio's JoinSet-drop abort reliably + runs `kill_on_drop` SIGKILL before `runtime.shutdown_timeout`, or a target can survive the + gap. + +## Open Questions + +- Does JoinSet-drop abort of the target task reliably fire `kill_on_drop` before + `runtime.shutdown_timeout`, or can a SIGTERM-ignoring target survive the gap? diff --git a/test/antithesis/scratchbook/properties/tcp-connect-loop-shutdown-responsive.md b/test/antithesis/scratchbook/properties/tcp-connect-loop-shutdown-responsive.md new file mode 100644 index 000000000..e1ad13602 --- /dev/null +++ b/test/antithesis/scratchbook/properties/tcp-connect-loop-shutdown-responsive.md @@ -0,0 +1,57 @@ +--- +slug: tcp-connect-loop-shutdown-responsive +subsystem: lifecycle/shutdown +assertion_type: Always +priority: P0 +needs_target: true +needs_sut_fix: wrap connect attempt in select! with shutdown_wait +--- + +# TCP worker observes shutdown while (re)connecting + +## Statement + +The TCP generator worker responds to the shutdown signal even when the target is unreachable +and it is stuck in the connect/retry loop. + +## Code evidence trail + +- `tcp.rs:232-251` — the connect branch does `connect().await` -> on error `sleep(1s)` -> + `continue`, forever. +- `tcp.rs:253-283` — the `tokio::select!` containing `shutdown_wait` is only reached once a + connection exists. +- `tcp.rs:206-213` — `Tcp::spin()` blocks on `join_next()` for these workers, so graceful + shutdown hangs (bounded only by the 30s runtime timeout, if reached). + +## Assertion-type rationale + +**Always** (bounded shutdown): every timeline that signals shutdown must reach exit within the +bound, including the unreachable-target case where the worker is in the pre-select connect loop. + +## Mode / observability + +With an unreachable target, after the experiment timer fires lading still exits within +`max_shutdown_delay`. Oracle external (timer to exit) in an unreachable-target scenario; +optionally a SUT `reachable!` at the loop's shutdown exit to prove the branch is taken. + +## Mechanism + +Wrap the connect attempt in `tokio::select!` with `&mut shutdown_wait`, or check shutdown +before the `sleep(1s)`, in the `tcp.rs` connect branch. + +## needs_sut_fix + +Yes. The connect branch has no shutdown branch; only the post-connection `select!` polls +`shutdown_wait`. + +## Investigation Log + +- [2026-07-24] Verified against source this turn: the connect happens before the `select!` and + shutdown is only polled inside it (`tcp.rs:230-283`). Confirmed live on main. +- [2026-07-24] Feeds the `shutdown-completes-bounded` umbrella and the + `unreachable-target` scenario. + +## Open Questions + +- None specific; shares the umbrella's open question about whether `Server::spin` completion + is externally timed out. diff --git a/test/antithesis/scratchbook/properties/tcp-rr-listener-no-panic.md b/test/antithesis/scratchbook/properties/tcp-rr-listener-no-panic.md new file mode 100644 index 000000000..27e54e4eb --- /dev/null +++ b/test/antithesis/scratchbook/properties/tcp-rr-listener-no-panic.md @@ -0,0 +1,52 @@ +--- +slug: tcp-rr-listener-no-panic +subsystem: no-panic +assertion_type: Unreachable +priority: P1 +needs_target: false +needs_sut_fix: return Result from create_listener instead of expect/panic +--- + +# tcp_rr blackhole returns an error, not a panic, on bind failure + +## Statement + +tcp_rr blackhole listener setup failures (address in use, stale bind) return `Error::Bind` +rather than panicking, including the `threads>1` thread-0 prebuild path on the main async task. + +## Code evidence trail + +- `tcp_rr.rs:345` — `.unwrap_or_else(|e| panic!("failed to bind to {binding_addr}: {e}"))`. +- `tcp_rr.rs:346` — `.expect("failed to listen")`; plus setup at `:313-327`. +- `tcp_rr.rs:179` — for `num_threads>1`, `create_listener(0,..)` is invoked directly in async + `run()` (thread-0 pre-build), so a bind failure panics the blackhole task rather than + returning `Error::Bind`. + +## Assertion-type rationale + +**Unreachable**: the `expect`/`panic!` on listener setup must never be reached. Oracle: +pre-bound-port scenario with `threads>1` + panic hook. + +## Mode / observability + +A pre-bound data port yields a clean error, not a SIGABRT. Currently `create_listener` uses +`expect()`/`panic!` (`tcp_rr.rs:345-346`) and thread-0 prebuild runs in async `run()` at `:179`. + +## Mechanism + +Return `Result` from `create_listener` (`tcp_rr.rs:313-346`) instead of `expect`/`panic`. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed: for the worker-thread path a panic is converted to + `Error::ThreadPanicked` via the ready channel, but the thread-0 prebuild path and the + socket-setup expects are hard panics on the main async task. + +## Open Questions + +- Is tcp_rr ever configured with `threads>1` in the harness/deployment? That is the path where + the bind failure panics the main async task rather than degrading to `ThreadPanicked`. diff --git a/test/antithesis/scratchbook/properties/throttle-capacity-no-zerodelivery-livelock.md b/test/antithesis/scratchbook/properties/throttle-capacity-no-zerodelivery-livelock.md new file mode 100644 index 000000000..601a288bc --- /dev/null +++ b/test/antithesis/scratchbook/properties/throttle-capacity-no-zerodelivery-livelock.md @@ -0,0 +1,58 @@ +--- +slug: throttle-capacity-no-zerodelivery-livelock +subsystem: generators/throttle +assertion_type: Always +priority: P0 +needs_target: true +needs_sut_fix: validate maximum_block_size <= bytes_per_second/parallel_connections (post-divide) at construction +--- + +# A block larger than throttle capacity never becomes a zero-delivery busy loop + +## Statement + +No config leaves a generator discarding every block (`block > per-worker capacity`) in a hot +loop at ~100% CPU delivering ~zero bytes; such a config is rejected at startup. + +## Code evidence trail + +- `common.rs:147-157` — `wait_for_block` requests `peek_next_size` tokens (the block's + `total_bytes`). +- `stable.rs:151-156` — a block exceeding `maximum_capacity` returns `Error::Capacity` + immediately with no wait. +- `tcp.rs:273-276` / `udp.rs:284-287` / `unix_stream.rs:320-323` / `unix_datagram.rs:297-300` — + catch Capacity as "Discarding block", advance, and loop -> hot busy loop delivering ~zero + bytes at 100% CPU. + +## Assertion-type rationale + +**Always** (no-livelock): on every timeline a run either delivers a nonzero rate or fails fast +at startup; it never burns a core delivering nothing. + +## Mode / observability + +A run either delivers a nonzero rate or fails fast at startup; it never burns a core delivering +nothing (the 0.31.2 busy-discard livelock class). Oracle: oversized-block scenario + CPU/ +throughput observation; or startup-error assertion. + +## Mechanism + +Validate `maximum_block_size <= bytes_per_second/parallel_connections` (post-divide) at +construction, returning a clear error instead of a runtime discard/spin. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed: when a block's byte size exceeds the (post-divide) throttle capacity, + `wait_for()` returns `Error::Capacity` with no wait, and every generator discards + loops. +- [2026-07-24] Closely related to `throttle-divide-no-silent-underdelivery` (the divide case + that creates this condition) and `discarded-blocks-counted` (observability). + +## Open Questions + +- Should generators treat throttle `Error::Capacity` as a fatal startup validation error rather + than a run-time discard/busy-loop, given `maximum_block_size` and `bytes_per_second` are both + known at construction? diff --git a/test/antithesis/scratchbook/properties/throttle-divide-no-silent-underdelivery.md b/test/antithesis/scratchbook/properties/throttle-divide-no-silent-underdelivery.md new file mode 100644 index 000000000..aebd41efe --- /dev/null +++ b/test/antithesis/scratchbook/properties/throttle-divide-no-silent-underdelivery.md @@ -0,0 +1,56 @@ +--- +slug: throttle-divide-no-silent-underdelivery +subsystem: generators/throttle +assertion_type: Always +priority: P0 +needs_target: true +needs_sut_fix: divide must shrink block sizing consistently with capacity, or validate maximum_block_size <= bps/parallel_connections upfront +--- + +# A config that delivers at N=1 still delivers at N>1 + +## Statement + +`throttle.divide` shrinks per-worker capacity consistently with the block size a worker draws, +so a block accepted at `parallel_connections=1` is not rejected by every worker (Capacity) at +`N>1`, yielding silent zero delivery. + +## Code evidence trail + +- `lib.rs` — `divide` produces per-worker capacity `capacity/divisor` (integer division) but + does NOT shrink the block a worker draws. +- `tcp.rs:273-276` — on a Capacity error the block is discarded (`continue`) with only a + `debug!` log. +- A block sized `R/N < block <= R` is accepted at `parallel_connections=1` but rejected with + `Capacity` by every worker at `N>1` — silent zero delivery. + +## Assertion-type rationale + +**Always** (rate fidelity): on every timeline a config that delivers at `N=1` must not discard +every block at `N>1`. Provable as a pure proptest against the real Valve (no rig); Antithesis +oracle: sink byte counter vs configured rate. + +## Mode / observability + +For `bytes_per_second/N < block <= bytes_per_second` the aggregate delivered bytes at `N>1` are +nonzero and match the `N=1` rate; today every worker discards. Silent (only a `debug!` log). + +## Mechanism + +`divide` must shrink block sizing consistently with capacity, or generators must validate +`maximum_block_size <= bytes_per_second/parallel_connections` upfront. + +## needs_sut_fix + +Yes. Demonstrated on backup branch (`0868e39c`); live on main. + +## Investigation Log + +- [2026-07-24] Confirmed live regression on main. Provable as a pure proptest against the real + Valve without any target rig. +- [2026-07-24] Related to `throttle-capacity-no-zerodelivery-livelock` (the busy-loop / 100% CPU + angle) and `discarded-blocks-counted` (the observability angle). + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/trace-agent-v04-block-terminates.md b/test/antithesis/scratchbook/properties/trace-agent-v04-block-terminates.md new file mode 100644 index 000000000..e9c4b2579 --- /dev/null +++ b/test/antithesis/scratchbook/properties/trace-agent-v04-block-terminates.md @@ -0,0 +1,52 @@ +--- +slug: trace-agent-v04-block-terminates +subsystem: payload/determinism +assertion_type: Always +priority: P0 +needs_target: false +needs_sut_fix: cap consecutive rejections (InsufficientBlockSizes) and emit EmptyBlock not a 1-byte block +--- + +# trace-agent v04 block-cache next_block terminates in bounded time + +## Statement + +trace-agent v04 block-cache construction terminates quickly even when a serialized trace exceeds +`max_block_size`; `to_bytes` never emits a 1-byte empty msgpack array accepted as a valid block, +and construction is not O(n^2) re-serialization. + +## Code evidence trail + +- `block.rs`, `trace_agent/v04.rs` — the v04 `next_block` construction path; observed 31h hang + -> 0.75s after the fix. +- Root cause: `to_bytes` emitted a 1-byte empty msgpack array accepted as a valid block, with + O(n^2) re-serialization when a single trace exceeds `max_block_size`. + +## Assertion-type rationale + +**Always**/liveness: on every config construction completes in sub-second. Fuzz property +`trace_agent_v04_cache_fixed_next_block` + a bounded-time startup assertion. + +## Mode / observability + +Construction completes in sub-second, not hours (observed 31h hang -> 0.75s). An empty result is +`EmptyBlock`, not a 1-byte block. + +## Mechanism + +Cap consecutive rejections (`InsufficientBlockSizes`) and emit `EmptyBlock` not a 1-byte block +(fix `456e85a3` on backup branch; live on main). + +## needs_sut_fix + +Yes. Fix `456e85a3` on backup branch; live on main. + +## Investigation Log + +- [2026-07-24] The general block-cache-hang case is a separate property + (`block-cache-construction-terminates`); this is the trace_agent-v04-specific variant with a + dedicated fuzz property. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/unix-datagram-blackhole-removes-stale-socket.md b/test/antithesis/scratchbook/properties/unix-datagram-blackhole-removes-stale-socket.md new file mode 100644 index 000000000..35b44c00e --- /dev/null +++ b/test/antithesis/scratchbook/properties/unix-datagram-blackhole-removes-stale-socket.md @@ -0,0 +1,52 @@ +--- +slug: unix-datagram-blackhole-removes-stale-socket +subsystem: blackhole/config +assertion_type: Always +priority: P1 +needs_target: true +needs_sut_fix: await the remove_file future in unix_datagram.rs:95 +--- + +# unix_datagram blackhole removes a stale socket before bind + +## Statement + +The unix_datagram blackhole actually removes a leftover socket file before binding, so it starts +cleanly after a hard-kill restart instead of failing bind with EADDRINUSE. + +## Code evidence trail + +- `blackhole/unix_datagram.rs:95` — `let _res = tokio::fs::remove_file(&self.path).map_err( + Error::Io);` uses `futures::TryFutureExt::map_err`, returning a lazy `MapErr` future that is + bound to `_res` and dropped without `.await`, so `remove_file` is never polled/executed. +- The comment at `:93-94` ("Delete the socket, ignore any errors") states the intended behavior, + which is defeated. `bind()` then runs at `:96` against a possibly-existing path. + +## Assertion-type rationale + +**Always** (restart resilience): on every timeline the blackhole must bind cleanly even after a +hard-kill left a stale socket. + +## Mode / observability + +After a `node_termination` leaving a stale socket, the blackhole binds successfully and the +target keeps its sink. Oracle: restart-with-stale-socket scenario asserting bind succeeds. + +## Mechanism + +Await the `remove_file` future in `unix_datagram.rs:95` (drop the lazy `TryFutureExt::map_err` +binding). + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Verified against source this turn: the `remove_file` future is built and dropped, + never awaited; binding to `let _res =` also suppresses the unused-future lint. + +## Open Questions + +- Is the unix socket path on a persisted named volume in the deployment? If so, a never-removed + stale socket guarantees bind failure on every SIGKILL restart, not just occasionally. diff --git a/test/antithesis/scratchbook/properties/unix-datagram-connect-loop-shutdown.md b/test/antithesis/scratchbook/properties/unix-datagram-connect-loop-shutdown.md new file mode 100644 index 000000000..ed7900d08 --- /dev/null +++ b/test/antithesis/scratchbook/properties/unix-datagram-connect-loop-shutdown.md @@ -0,0 +1,50 @@ +--- +slug: unix-datagram-connect-loop-shutdown +subsystem: lifecycle/shutdown +assertion_type: Always +priority: P0 +needs_target: true +needs_sut_fix: add shutdown branch to unix_datagram connect/retry loop +--- + +# unix_datagram connect loop observes shutdown + +## Statement + +The unix_datagram initial connect loop polls shutdown; if the socket path never appears the +worker still shuts down within `max_shutdown_delay`. + +## Code evidence trail + +- `unix_datagram.rs:246-264` — the initial connect loop retries `connect()` -> `sleep(1s)` + with no shutdown check; if the socket path is never available the worker never reaches the + `select!` and shutdown hangs. + +## Assertion-type rationale + +**Always** (bounded shutdown): every timeline that signals shutdown must reach exit within the +bound, including the missing-socket-path case. + +## Mode / observability + +Missing-socket-path scenario: lading exits within bound rather than spinning +`connect()` -> `sleep(1s)` forever. Oracle: external exit-timer in a never-bound-socket +scenario. + +## Mechanism + +Add a shutdown branch to the `unix_datagram.rs:246-264` connect/retry loop. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed live on main: connect loop has no shutdown check. +- [2026-07-24] Feeds the `shutdown-completes-bounded` umbrella and the `unreachable-target` + scenario (socket path that never binds). + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/unix-stream-partial-write-shutdown.md b/test/antithesis/scratchbook/properties/unix-stream-partial-write-shutdown.md new file mode 100644 index 000000000..f6c625f7a --- /dev/null +++ b/test/antithesis/scratchbook/properties/unix-stream-partial-write-shutdown.md @@ -0,0 +1,57 @@ +--- +slug: unix-stream-partial-write-shutdown +subsystem: lifecycle/shutdown +assertion_type: Always +priority: P0 +needs_target: true +needs_sut_fix: shutdown branch in partial-write and connect loops; bounded readiness instead of bare yield_now +--- + +# unix_stream partial-write loop is shutdown-aware and not a busy-spin + +## Statement + +When the receiver's socket buffer is full, the unix_stream inner partial-write loop neither +busy-spins on `WouldBlock` at 100% CPU nor ignores shutdown; it yields to shutdown and bounds +its retry. + +## Code evidence trail + +- `unix_stream.rs:281-318` — the inner `while blk_offset < blk_max` loop has no shutdown branch + and busy-spins via `yield_now()` (at `:302`) when `try_write` returns `WouldBlock`. +- The code comment admits: "If the read side has hung up we will never know and will keep + attempting to write." +- `unix_stream.rs:248-268` — the connect loop also lacks a shutdown check. + +## Assertion-type rationale + +**Always** (bounded shutdown + no livelock): under a stalled/slow receiver every timeline must +still exit within the bound and must not peg a core. + +## Mode / observability + +With a stalled/slow unix receiver, lading exits within `max_shutdown_delay` and does not peg a +core. Oracle: slow-receiver scenario + external exit timer and CPU observation. + +## Mechanism + +Add a shutdown branch to the `while blk_offset < blk_max` loop (`unix_stream.rs:281-318`) and +to the connect loop (`:248-268`); replace the bare `yield_now` spin with a bounded/awaited +readiness. + +## needs_sut_fix + +Yes. Both the partial-write inner loop and the connect loop need shutdown awareness; the +`WouldBlock` busy-yield must become an awaited readiness. + +## Investigation Log + +- [2026-07-24] Confirmed live on main: the inner loop busy-spins on `WouldBlock` and the + comment explicitly acknowledges the read-side-hangup blind spot. +- [2026-07-24] Related but distinct: `unix-stream-write-error-progress` covers the + non-BrokenPipe write-error busy loop; this property is the shutdown + CPU angle. + +## Open Questions + +- Is the undivided per-connection throttle in unix_stream intentional (separate property + `unix-throttle-aggregate-consistent`)? Not relevant to shutdown but shares the file. diff --git a/test/antithesis/scratchbook/properties/unix-stream-write-error-progress.md b/test/antithesis/scratchbook/properties/unix-stream-write-error-progress.md new file mode 100644 index 000000000..408776a61 --- /dev/null +++ b/test/antithesis/scratchbook/properties/unix-stream-write-error-progress.md @@ -0,0 +1,52 @@ +--- +slug: unix-stream-write-error-progress +subsystem: generators/throttle +assertion_type: Always +priority: P2 +needs_target: true +needs_sut_fix: handle non-BrokenPipe write errors (advance/break/reconnect); count packets per block not per partial write +--- + +# unix_stream makes progress on non-BrokenPipe write errors + +## Statement + +On a non-BrokenPipe, non-WouldBlock write error (e.g. ConnectionReset) the unix_stream worker +reconnects or advances rather than busy-looping on the same offset spamming `request_failure`. + +## Code evidence trail + +- `unix_stream.rs:304-315` — the non-BrokenPipe branch falls through with no offset + advance/break/reconnect; the inner while loop re-calls `ready()`/`try_write` on the same + offset, busy-looping and spamming `request_failure`. Only BrokenPipe triggers reconnect. +- `unix_stream.rs:293-295` — `packets_sent` is incremented once per partial write, so one block + can count as many "packets". + +## Assertion-type rationale + +**Always** (no-livelock): on every timeline a ConnectionReset receiver must not pin a core or +emit a runaway `request_failure` count. + +## Mode / observability + +A ConnectionReset receiver does not pin a core or emit a runaway `request_failure` count; +`packets_sent` is not inflated per partial write. Oracle: reset-injecting receiver + CPU/counter +observation. + +## Mechanism + +Handle non-BrokenPipe write errors (advance/break/reconnect) at `unix_stream.rs:304-315`; count +packets per block, not per partial write. + +## needs_sut_fix + +Yes. + +## Investigation Log + +- [2026-07-24] Confirmed: only BrokenPipe triggers reconnect; other write errors fall through + without advancing, causing a busy loop on the same offset. + +## Open Questions + +- None specific. diff --git a/test/antithesis/scratchbook/properties/unix-throttle-aggregate-consistent.md b/test/antithesis/scratchbook/properties/unix-throttle-aggregate-consistent.md new file mode 100644 index 000000000..c7aafc60a --- /dev/null +++ b/test/antithesis/scratchbook/properties/unix-throttle-aggregate-consistent.md @@ -0,0 +1,56 @@ +--- +slug: unix-throttle-aggregate-consistent +subsystem: generators/throttle +assertion_type: Always +priority: P1 +needs_target: true +needs_sut_fix: add .divide(worker_count) in unix_stream/unix_datagram (or document per-connection semantics) +--- + +# unix_stream/unix_datagram aggregate rate matches configured bytes_per_second + +## Statement + +unix_stream and unix_datagram divide the throttle across `parallel_connections` so aggregate +delivery approximates `bytes_per_second`, consistent with tcp/udp, rather than delivering +`parallel_connections x` the rate. + +## Code evidence trail + +- `unix_stream.rs:161-163` — each child gets a full `create_throttle(bytes_per_second)`, no + `.divide`. +- `unix_datagram.rs:186-188` — same, no `.divide`. +- Contrast `tcp.rs:168-173` which divides. Aggregate delivery = `parallel_connections * + bytes_per_second` for unix vs `~= bytes_per_second` for tcp/udp. +- Doc mismatch: `unix_stream.rs:46-47` says "per connection"; `unix_datagram.rs:54` implies + aggregate. + +## Assertion-type rationale + +**Always** (rate fidelity): on every timeline the same `bytes_per_second` key must have a +consistent aggregate meaning across generators. + +## Mode / observability + +Aggregate delivered bytes for unix generators at `N>1` approximate `bytes_per_second`, not +`N x it`. Cross-generator inconsistency: same config key means aggregate for tcp/udp but +per-connection for unix. Oracle: sink byte-rate vs configured. + +## Mechanism + +Add `.divide(worker_count)` in `unix_stream.rs:161-163` and `unix_datagram.rs:186-188` (or +document per-connection semantics deliberately). + +## needs_sut_fix + +Yes (or a deliberate documentation decision). + +## Investigation Log + +- [2026-07-24] Confirmed: unix generators do not divide the throttle, so aggregate is `N x` the + configured rate. + +## Open Questions + +- Is undivided per-connection throttle intentional for unix_stream (its doc says "per + connection") vs unix_datagram (whose doc implies aggregate)? diff --git a/test/antithesis/scratchbook/property-catalog.md b/test/antithesis/scratchbook/property-catalog.md new file mode 100644 index 000000000..495e6c836 --- /dev/null +++ b/test/antithesis/scratchbook/property-catalog.md @@ -0,0 +1,940 @@ +--- +sut_path: /home/ssm-user/src/lading +commit: 51148899 +updated: 2026-07-24T21:28:18Z +external_references: + - name: the deployment (production runner + local dev orchestrator) + why: real shutdown/deploy model — lading "owns the clock", is torn down by SIGKILL (docker rm --force), watchdog=(warmup+samples+30)*1.2, parquet-only captures, container target-observer mode; defines the operational meaning of "correct shutdown" + - name: Jira (project SMPTNG/SMP) + Confluence (datadoghq.atlassian.net) + why: existing bug tickets and design docs — SMPTNG-725 hang-in-spin, SMPTNG-719/697 ungraceful-termination telemetry loss, SMPTNG-694 zstd captures, SMPTNG-390 traffic recorder, SMPTNG-762/767 entropy/wall-clock SUT-analog classes + - name: the whole lading repo (this checkout) + why: source of truth for every property — spot-verified against current main; hardening fixes referenced in git history live only on backup/blt branches, NOT main +--- + +# lading bug-hunting property catalog + +Every property below is a **checkable invariant whose violation is a real lading defect**. The +oracle is either the existing panic hook (`antithesis_hooks.rs:27` reports `unreachable! "lading +panicked"`, then `panic=abort` SIGABRTs the container), an external harness checker, or a +feature-gated SUT-side `assert_always!`/`reachable!`. Prioritize where Antithesis is strongest: +timing, concurrency, partial-failure, no-panic, determinism. + +**SHUTDOWN / TERMINATION SAFETY is the immediate priority (P0).** The full system is catalogued +below it. + +> Regression note: the hardening fixes referenced in git history live only on `backup/20260720/*` +> and `blt/*` branches, **NOT on main**. So the divide stall, linear-ramp compression, gRPC +> throttle bypass, observer PID/vanish/cpu.max/transient panics, logrotate stale-tick panic, +> per-generator semaphore panic, capture abort, silent discards, and unbounded error labels are +> **live regressions in main** — not merely history. + +Assertion types: **Unreachable** for specific panic/abort sites and validation guards; +**Always** for bounded-latency / rate-fidelity / crash-consistency / observability invariants that +must hold on every timeline; **Sometimes** for non-vacuity liveness. + +--- + +## Overview by subsystem and priority + +### lifecycle / shutdown (P0 first — immediate priority) + +| Priority | Property | Assertion | Needs target | One-line invariant | +|---|---|---|---|---| +| P0 | `sigterm-graceful-drain` | Always | no | SIGTERM runs the same graceful drain/finalize path as the experiment timer | +| P0 | `shutdown-completes-bounded` | Always | yes | lading exits within `max_shutdown_delay` once shutdown is signaled | +| P0 | `tcp-connect-loop-shutdown-responsive` | Always | yes | TCP worker observes shutdown while (re)connecting to an unreachable target | +| P0 | `unix-stream-partial-write-shutdown` | Always | yes | unix_stream partial-write loop is shutdown-aware, not a busy-spin | +| P0 | `unix-datagram-connect-loop-shutdown` | Always | yes | unix_datagram connect loop observes shutdown | +| P0 | `target-wait-bounded` | Always | yes | post-SIGTERM target/inspector wait is time-bounded (SIGKILL escalation) | +| P0 | `docker-target-discovery-bounded` | Always | yes | container-target discovery is bounded + shutdown-aware (production observer path) | +| P0 | `lading-completes-and-exits-cleanly` | Sometimes | no | happy-path self-termination reaches exit 0 with readable capture (non-vacuity) | +| P1 | `grpc-connect-loop-shutdown` | Always | yes | gRPC connect loop observes shutdown | +| P1 | `capture-finalize-bounded` | Always | no | capture-finalize await is time-bounded | +| P1 | `orphaned-children-on-signal-death` | Always | yes | no orphaned target/inspector children on signal-driven death | +| P1 | `target-grace-period-honored` | Always | yes | cooperative slow target gets its full post-SIGTERM cleanup window | + +### capture (lading_capture) + +| Priority | Property | Assertion | Needs target | One-line invariant | +|---|---|---|---|---| +| P0 | `capture-write-failure-not-abort` | Unreachable | no | a capture write error is a clean fatal exit, not a `panic=abort` SIGABRT | +| P0 | `parquet-footer-on-graceful-exit` | Always | no | every graceful exit yields a readable footer-complete parquet | +| P1 | `jsonl-prefix-valid-after-kill` | Always | no | a SIGKILLed jsonl capture is always a valid parseable prefix | +| P1 | `multi-format-parquet-not-forfeited` | Always | no | multi mode finalizes parquet even if jsonl close errors | +| P1 | `capture-histogram-drops-counted` | Always | no | dropped histogram samples are counted, not silently lost | +| P2 | `capture-no-fsync-durability` | Always | no | flushed capture lines survive whole-VM termination | +| P2 | `capture-drift-no-silent-gap` | Always | no | drift correction does not silently drop unflushed intervals | +| P2 | `split-mode-merge-partial-tolerant` | Always | yes | split-mode merge tolerates one clean side when the other overruns | + +### payload / determinism (lading_payload) + +| Priority | Property | Assertion | Needs target | One-line invariant | +|---|---|---|---|---| +| P0 | `block-cache-construction-terminates` | Always | no | block-cache construction always terminates in bounded time | +| P0 | `trace-agent-v04-block-terminates` | Always | no | trace-agent v04 `next_block` terminates fast; no 1-byte empty block | +| P1 | `payload-determinism-byte-identical` | Always | no | same seed + config yields byte-identical load | +| P2 | `no-wall-clock-in-payloads` | Always | no | payload timestamps are seed-derived and monotonic, never wall-clock | +| P2 | `dogstatsd-tag-length-validated` | Always | no | `tag_length.end() <= MIN_TAG_LENGTH` is rejected upfront (merged to main) | + +### generators / throttle + +| Priority | Property | Assertion | Needs target | One-line invariant | +|---|---|---|---|---| +| P0 | `throttle-divide-no-silent-underdelivery` | Always | yes | a config delivering at N=1 still delivers at N>1 | +| P0 | `linear-ramp-slope-preserved` | Always | yes | Linear aggregate ramp slope == configured `rate_of_change`, not N× | +| P0 | `throttle-capacity-no-zerodelivery-livelock` | Always | yes | oversized block never becomes a zero-delivery busy loop | +| P0 | `grpc-honors-throttle` | Always | yes | gRPC honors throttle rejections and configured rate | +| P1 | `stable-burst-envelope-bounded` | Always | no | stable throttle never exceeds its per-interval burst envelope | +| P1 | `unix-throttle-aggregate-consistent` | Always | yes | unix_stream/unix_datagram aggregate rate matches configured bps | +| P1 | `discarded-blocks-counted` | Always | yes | under-delivery is observable via a `blocks_discarded` counter | +| P1 | `error-label-cardinality-bounded` | Always | yes | generator error-metric label cardinality is bounded (io::ErrorKind) | +| P1 | `sink-receives-bytes` | Sometimes | yes | the sink receives nonzero load (delivery non-vacuity) | +| P2 | `divide-by-zero-startup-error` | Always | yes | `bps < parallel_connections` fails clearly, no DivisionByZero surprise | +| P2 | `unix-stream-write-error-progress` | Always | yes | unix_stream makes progress on non-BrokenPipe write errors | + +### observer + +| Priority | Property | Assertion | Needs target | One-line invariant | +|---|---|---|---|---| +| P0 | `observer-pid-reuse-no-panic` | Unreachable | yes | never aborts on PID reuse / mismatch (`assert!(cur_pid==pid)`) | +| P0 | `observer-process-vanish-no-panic` | Unreachable | yes | never panics when a target vanishes mid-listing | +| P0 | `observer-cpu-max-parse-no-panic` | Unreachable | yes | never panics on malformed/truncated `cpu.max` | +| P1 | `observer-transient-read-not-fatal` | Always | yes | a transient procfs/cgroup/wss read error does not kill the run | +| P2 | `observer-pid-identity-fingerprint` | Always | yes | samples the identified target, not a PID-reuse impostor | +| P2 | `get-available-memory-cgroup-chain` | Always | no | memory limit reflects tightest cgroup v2 ancestor (merged to main) | + +### no-panic (umbrella + specific sites) + +| Priority | Property | Assertion | Needs target | One-line invariant | +|---|---|---|---|---| +| P0 | `no-panic-anywhere` | Unreachable | no | lading never panics anywhere (panic hook = `Unreachable`) | +| P0 | `observer-target-pid-recv-no-panic` | Unreachable | yes | observer returns error, not `.expect`, when target PID never arrives | +| P1 | `block-cache-zero-max-no-panic` | Unreachable | no | `maximum_block_size` of 0 is rejected, not a `random_range` panic | +| P1 | `per-generator-semaphore-no-panic` | Unreachable | yes | two HTTP (or two Splunk-HEC) generators coexist without OnceCell panic | +| P1 | `splunk-hec-response-parse-no-panic` | Unreachable | yes | Splunk-HEC response parse never panics on a non-HecResponse body | +| P1 | `generator-addr-uri-validation-no-panic` | Unreachable | no | malformed addr/target_uri is a Result error, not a construction panic | +| P1 | `tcp-rr-listener-no-panic` | Unreachable | no | tcp_rr blackhole returns Error::Bind, not a panic, on bind failure | +| P1 | `logrotate-stale-tick-noop` | Unreachable | no | logrotate_fs treats a stale tick as a no-op, never a panic | +| P2 | `arbitrary-block-nonzero-no-panic` | Unreachable | no | fuzz Arbitrary Block handles `total_bytes==0` without `.expect` panic | + +### blackhole / config + +| Priority | Property | Assertion | Needs target | One-line invariant | +|---|---|---|---|---| +| P1 | `unix-datagram-blackhole-removes-stale-socket` | Always | yes | blackhole removes a stale socket before bind (currently lazy future) | +| P1 | `datadog-blackhole-accept-resilient` | Always | yes | Datadog blackhole keeps accepting after a transient accept error | +| P1 | `blackhole-never-backpressures-target` | Always | yes | blackhole responds regardless of capture-channel saturation | +| P2 | `sqs-receive-message-bounded` | Always | no | SQS blackhole bounds `ReceiveMessage` response size | +| P2 | `config-numeric-fields-validated` | Always | no | numeric config fields validated at load, not at runtime failure | +| P2 | `recorded-traffic-crash-consistency` | Always | yes | blackhole-recorded traffic files are crash-consistent | + +--- + +# Properties — shutdown / termination safety (P0 FIRST) + +## sigterm-graceful-drain — SIGTERM finalizes capture like the experiment timer +- **Subsystem:** lifecycle/shutdown · **Priority:** P0 · **Assertion:** Always · **Needs target:** no +- **Statement:** When lading receives SIGTERM it runs the same graceful path as experiment-timer + self-termination: broadcast shutdown, drain the capture maturity window, write the parquet/multi + footer, exit non-abnormally. +- **Observable:** After a SIGTERM the on-disk parquet/multi capture is readable (footer present) and + lading exits 143/0, not abort. MOOT under Antithesis `node_termination` (SIGKILL is untrappable) + but LIVE on the deployment's orchestrator-stop path. +- **Mechanism:** Always invariant on the graceful contract. Best oracle is external: send SIGTERM + mid-run, then validate the capture file is footer-complete and the process exited without abort. + Lead confirmed by deployment tickets SMPTNG-719/697 (ungraceful-termination telemetry loss). +- **Needs SUT fix:** Add a `tokio::signal::unix` `SignalKind::terminate` arm in the `lading.rs` main + `select!` (alongside `ctrl_c` at :658) that triggers `shutdown_broadcast`. +- **Evidence:** `lading.rs:658` only `ctrl_c`; no `SignalKind::terminate` per grep. Digest lifecycle finding 1. +- **Open questions:** Does Antithesis ever deliver SIGTERM (vs only SIGKILL `node_termination`)? If + not, this is exercised only by an explicit harness `kill -TERM` step. + +## shutdown-completes-bounded — lading exits within max_shutdown_delay after shutdown is signaled +- **Subsystem:** lifecycle/shutdown · **Priority:** P0 · **Assertion:** Always · **Needs target:** yes +- **Statement:** Once the experiment timer fires (or a shutdown signal is broadcast) lading + terminates within `max_shutdown_delay`; it never hangs in `Server::spin` waiting on a worker that + cannot observe shutdown. +- **Observable:** Wall time from shutdown-broadcast to process exit `<= max_shutdown_delay` (30s). + Violation = overrun → deployment watchdog SIGKILL → unreadable parquet → total capture loss for + the replicate. +- **Mechanism:** Umbrella liveness for shutdown hangs. Oracle: external timer measuring + shutdown-signal → exit, or Antithesis liveness "eventually exits". Deployment-confirmed lead: + SMPTNG-725 "RJO alive but not really" hang-in-spin after "lading shutdown". +- **Needs SUT fix:** Add shutdown branches to every pre-select connect/retry loop + (tcp/udp/unix_stream/unix_datagram/grpc) and a timeout to `target_child.wait()` and the + capture-finalize await; see per-loop properties. +- **Evidence:** Digest generators findings 1–4; lifecycle findings 2,6,7. + +## tcp-connect-loop-shutdown-responsive — TCP worker observes shutdown while (re)connecting +- **Subsystem:** lifecycle/shutdown · **Priority:** P0 · **Assertion:** Always · **Needs target:** yes +- **Statement:** The TCP generator worker responds to the shutdown signal even when the target is + unreachable and it is stuck in the connect/retry loop. +- **Observable:** With an unreachable target, after the experiment timer fires lading still exits + within `max_shutdown_delay`. Confirmed live: the connect branch (`tcp.rs:230-251`) does + `connect → sleep(1s) → continue` and only the post-connection `select!` polls `shutdown_wait`. +- **Mechanism:** Bounded shutdown. Oracle external (timer to exit) in an unreachable-target + scenario; optionally a SUT `reachable!` at the loop's shutdown exit to prove the branch is taken. +- **Needs SUT fix:** Wrap the connect attempt in `tokio::select!` with `&mut shutdown_wait`, or check + shutdown before the `sleep(1s)`, in the `tcp.rs` connect branch. +- **Evidence:** `tcp.rs:230-283` (connect before select, shutdown only inside select). + +## unix-stream-partial-write-shutdown — unix_stream partial-write loop is shutdown-aware and not a busy-spin +- **Subsystem:** lifecycle/shutdown · **Priority:** P0 · **Assertion:** Always · **Needs target:** yes +- **Statement:** When the receiver's socket buffer is full, the unix_stream inner partial-write loop + neither busy-spins on `WouldBlock` at 100% CPU nor ignores shutdown; it yields to shutdown and + bounds its retry. +- **Observable:** With a stalled/slow unix receiver, lading exits within `max_shutdown_delay` and + does not peg a core. Code comment admits "if the read side has hung up we will never know and will + keep attempting to write." +- **Mechanism:** Bounded shutdown + no livelock. Oracle: slow-receiver scenario + external exit timer + and CPU observation. +- **Needs SUT fix:** Add a shutdown branch to the `while blk_offset < blk_max` loop + (`unix_stream.rs:281-318`) and to the connect loop (`:248-268`); replace the bare `yield_now` spin + with a bounded/awaited readiness. +- **Evidence:** `unix_stream.rs:281-318` inner loop, `yield_now` at `:302`; connect loop `:248-268`. + +## unix-datagram-connect-loop-shutdown — unix_datagram connect loop observes shutdown +- **Subsystem:** lifecycle/shutdown · **Priority:** P0 · **Assertion:** Always · **Needs target:** yes +- **Statement:** The unix_datagram initial connect loop polls shutdown; if the socket path never + appears the worker still shuts down within `max_shutdown_delay`. +- **Observable:** Missing-socket-path scenario: lading exits within bound rather than spinning + `connect → sleep(1s)` forever. +- **Mechanism:** Always. External exit-timer oracle in a never-bound-socket scenario. +- **Needs SUT fix:** Add a shutdown branch to the `unix_datagram.rs:246-264` connect/retry loop. +- **Evidence:** `unix_datagram.rs:246-264`. + +## target-wait-bounded — post-SIGTERM target/inspector wait is time-bounded +- **Subsystem:** lifecycle/shutdown · **Priority:** P0 · **Assertion:** Always · **Needs target:** yes +- **Statement:** After lading SIGTERMs a Binary target (or inspector), the `wait()` for the child to + exit is bounded (does not depend solely on external JoinSet-abort/runtime timeout); a + SIGTERM-ignoring target does not hang lading. +- **Observable:** With a target that ignores SIGTERM, lading still exits within `max_shutdown_delay`. + MOOT on the deployment's container-observer path (target is a sibling container force-killed by the + runner), LIVE for Binary-target mode. +- **Mechanism:** Locally-bounded reap. Oracle: SIGTERM-ignoring-target scenario + exit timer. +- **Needs SUT fix:** Wrap `target_child.wait()` (`target.rs:432`) and inspector wait + (`inspector.rs:176`) in a timeout that escalates to SIGKILL before returning. +- **Evidence:** `target.rs:432`, `inspector.rs:176` bare `.await`. + +## docker-target-discovery-bounded — container-target discovery is bounded and shutdown-aware +- **Subsystem:** lifecycle/shutdown · **Priority:** P0 · **Assertion:** Always · **Needs target:** yes +- **Statement:** In container/observer mode, the target-container discovery poll loop either finds + the container, times out with an error, or responds to shutdown; it never spins forever so the + experiment timer can start. +- **Observable:** With a misnamed/never-started target container, lading either errors out bounded or + self-terminates on its timer; it does not block `experiment_sequence` at + `target_running_watcher.recv()` forever. This is the production observer path, so high impact. +- **Mechanism:** Bounded startup. Oracle: wrong `--target-container` name scenario; assert lading + exits (error or timer) within the watchdog. Strong deployment lead (this is the launched production + mode). +- **Needs SUT fix:** Add a `shutdown.recv()` arm and/or a max-attempts timeout to the + `target.rs:212-244` `watch_container` loop. +- **Evidence:** `target.rs:212-244` (only found-break + `sleep(1s)`, no shutdown/timeout); + `lading.rs:632-641` experiment timer gated on `target_running`. + +## lading-completes-and-exits-cleanly — lading self-terminates on its experiment timer and exits 0 (non-vacuity) +- **Subsystem:** lifecycle/shutdown · **Priority:** P0 · **Assertion:** Sometimes · **Needs target:** no +- **Statement:** On the happy path lading owns the clock: it reaches experiment end, drains capture, + and exits 0 with a readable capture at least once. +- **Observable:** A run reaches clean self-termination with exit 0 and a footer-complete capture. + `reachable!` anchor for shutdown coverage. +- **Mechanism:** Sometimes (liveness non-vacuity): the good shutdown path must be reachable on at + least one timeline, guarding against a regression that makes every timeline hang/abort. Oracle: + external exit-code + capture-readable check; SUT `reachable!` at clean exit. +- **Needs SUT fix:** none. +- **Evidence:** `lading.rs` `experiment_sequence` + graceful path. + +## grpc-connect-loop-shutdown — gRPC connect loop observes shutdown +- **Subsystem:** lifecycle/shutdown · **Priority:** P1 · **Assertion:** Always · **Needs target:** yes +- **Statement:** The gRPC generator's initial connect loop polls shutdown; a never-available target + does not wedge the generator. +- **Observable:** Never-up target: lading exits within bound rather than spinning + `connect → sleep(100ms)`. +- **Mechanism:** Always. External exit-timer oracle. +- **Needs SUT fix:** Add a shutdown branch to the `grpc.rs:287-299` connect loop. +- **Evidence:** `grpc.rs:287-299`. + +## capture-finalize-bounded — capture finalize await is time-bounded +- **Subsystem:** lifecycle/shutdown · **Priority:** P1 · **Assertion:** Always · **Needs target:** no +- **Statement:** The capture-manager join await during shutdown is bounded; a stalled parquet footer + write (slow volume, disk-full) cannot make lading hang before `runtime.shutdown_timeout` can act. +- **Observable:** With a slow/stalled capture volume, lading still exits within `max_shutdown_delay`. +- **Mechanism:** Always. Oracle: slow-disk fault scenario + exit timer. +- **Needs SUT fix:** Add a timeout around `let _ = handle.await;` (`lading.rs:713-715`) so + `runtime.shutdown_timeout` remains the backstop. +- **Evidence:** `lading.rs:713-715` unbounded await before `shutdown_timeout` at `:813`. + +## orphaned-children-on-signal-death — no orphaned target/inspector children on signal-driven death +- **Subsystem:** lifecycle/shutdown · **Priority:** P1 · **Assertion:** Always · **Needs target:** yes +- **Statement:** lading reaps its Binary-target and inspector children (and their process groups) + even when killed by a signal, rather than relying solely on `kill_on_drop` which cannot fire on + untrapped-signal death. +- **Observable:** After a SIGTERM (or crash) no orphaned target/inspector grandchildren remain + reparented to init. MOOT under Antithesis whole-container SIGKILL (all pids die together); LIVE in + shared-PID-namespace / bare-host deployments. +- **Mechanism:** No-leak. Oracle: SIGTERM scenario in a shared PID namespace checking for surviving + children. +- **Needs SUT fix:** Install a SIGTERM handler (see `sigterm-graceful-drain`) and send signals to the + process group, not just the direct child pid (`target.rs:430-431`, `inspector.rs:175-176`). +- **Evidence:** `target.rs:392,430-431`; `inspector.rs:146,175-176`. + +## target-grace-period-honored — a cooperative slow target gets its full post-SIGTERM cleanup window +- **Subsystem:** lifecycle/shutdown · **Priority:** P1 · **Assertion:** Always · **Needs target:** yes +- **Statement:** On graceful shutdown the target receives SIGTERM and is given up to + `max_shutdown_delay` to clean up/flush before SIGKILL, rather than being SIGKILLed within + milliseconds when capture flush returns. +- **Observable:** A cooperative-but-slow target completes its cleanup/artifact flush before being + killed. Currently `inner_main` returns after capture flush, dropping `tsrv_joinset` and aborting + `target_child.wait` → `kill_on_drop` SIGKILL. +- **Mechanism:** Grace contract. Oracle: SIGTERM-then-slow-cleanup target scenario checking the + target's artifacts are complete. +- **Needs SUT fix:** Join the target task with a `max_shutdown_delay`-bounded wait after signaling, + instead of dropping the JoinSet immediately (`lading.rs:690-717`). +- **Evidence:** `lading.rs:690-717`; `target.rs:392,425-433`. +- **Open questions:** Is the ~0 grace intentional, or is `max_shutdown_delay` meant to gate the target + cleanup window? + +--- + +# Properties — capture (lading_capture) + +## capture-write-failure-not-abort — a capture write error is a clean fatal exit, not a process abort +- **Subsystem:** capture · **Priority:** P0 · **Assertion:** Unreachable · **Needs target:** no +- **Statement:** A transient capture write/flush error (disk full, EIO) causes a clean non-zero exit + with the parquet footer flushed, not a `panic=abort` that SIGABRTs the whole run mid-flush and + leaves an unreadable file. +- **Observable:** On an injected capture-write IO error the panic hook must NOT fire and the parquet + must be readable. Currently `.block_on(start()).expect(...)` + `panic=abort` turns any flush-tick + error into a SIGABRT that skips `format.close()`. +- **Mechanism:** Unreachable on the abort site. Oracle: IO-fault-injection scenario + panic hook + + external parquet-readability check. Partly landed on backup branch (`b7624af2`), not on main. +- **Needs SUT fix:** Replace the `.expect` on capture start (`lading.rs:476/501/526`) with Result + propagation to `Error::CaptureManager` so BufWriters flush on Drop; plumb mid-run write errors to + graceful shutdown. +- **Evidence:** `lading.rs:476/501/526` `.expect`; `manager.rs:409` `next(event)?`; `Cargo.toml` + `panic=abort`. + +## parquet-footer-on-graceful-exit — graceful exit always yields a readable parquet capture +- **Subsystem:** capture · **Priority:** P0 · **Assertion:** Always · **Needs target:** no +- **Statement:** After any graceful termination (experiment timer or SIGTERM) the parquet/multi + capture file has a finalized footer and is fully readable. +- **Observable:** An external reader opens `/captures/captures.parquet` post-exit and parses all row + groups. This is the operational invariant the deployment depends on + (watchdog/cancel/kill → unreadable → capture loss). +- **Mechanism:** Always. Oracle is the existing `anytime_capture_consistent` checker's parquet arm + (readable ⇒ internally consistent) plus a post-graceful-exit readability assertion. +- **Needs SUT fix:** none on the pure graceful path; depends on `sigterm-graceful-drain` and + `capture-write-failure-not-abort` holding. +- **Evidence:** `parquet.rs:307-324` footer only in `close()`; `state_machine.rs:249-259` close only + on `ShutdownSignaled`. + +## jsonl-prefix-valid-after-kill — a SIGKILLed jsonl capture is always a valid parseable prefix +- **Subsystem:** capture · **Priority:** P1 · **Assertion:** Always · **Needs target:** no +- **Statement:** Under abrupt kill (`node_termination`/SIGKILL) any surviving jsonl capture parses as + a valid prefix with no torn final record and strictly-increasing per-series `fetch_index`/`time`. +- **Observable:** `anytime_capture_consistent.rs` `always!(torn_before_final==0)` and + `always!(invariants_hold)`. Loss of the last `<=60s` maturity window is tolerated by design; a + torn/reordered record is a defect. +- **Mechanism:** Always. Oracle already implemented (harness checker, `MIN_RECORDS=10` non-vacuity + floor). +- **Needs SUT fix:** none (holds by construction); property guards against regressions in the + accumulator flush ordering. +- **Evidence:** `anytime_capture_consistent.rs:44,49`; `accumulator.rs:492-496` monotonic flush. + +## multi-format-parquet-not-forfeited — multi format finalizes parquet even if jsonl close errors +- **Subsystem:** capture · **Priority:** P1 · **Assertion:** Always · **Needs target:** no +- **Statement:** In multi capture mode a trivial jsonl flush/close error does not skip the parquet + footer write; the important format is never sacrificed to the unimportant one. +- **Observable:** Inject a jsonl close error; the parquet footer is still written and the parquet file + is readable. +- **Mechanism:** Always. Oracle: fault-inject jsonl path + external parquet readability check. +- **Needs SUT fix:** Reorder `multi.rs` close/flush/write_metric to finalize parquet first (or + best-effort both, aggregating errors) at `multi.rs:46-48,57-58,69-72`. +- **Evidence:** `multi.rs:69` `jsonl.close()?` before `:71` `parquet.close()?`. + +## capture-histogram-drops-counted — dropped histogram samples are counted, not silently lost +- **Subsystem:** capture · **Priority:** P1 · **Assertion:** Always · **Needs target:** no +- **Statement:** When the capture channel is full or the recorder is uninitialized, dropped + latency/histogram samples increment a bounded-label counter held in the registry, so tail-biased + sample loss is observable. +- **Observable:** Under high recording load a run delivering incomplete histograms shows a nonzero + `capture_histogram_samples_dropped` rather than looking healthy. +- **Mechanism:** Observability. Oracle: saturate the channel, assert dropped count > 0 when samples + were lost. +- **Needs SUT fix:** Add a `capture_histogram_samples_dropped` counter (bounded reason label, in + registry not the sample channel) at `manager.rs:121-133`. Landed on backup branch (`39d8ae56`), not + main. +- **Evidence:** `manager.rs:302` `channel(10_000)`; `:121-133` `try_send` + warn-drop. + +## capture-no-fsync-durability — flushed capture lines survive whole-VM termination +- **Subsystem:** capture · **Priority:** P2 · **Assertion:** Always · **Needs target:** no +- **Statement:** Capture data reported as flushed reaches the persisted volume durably; a + whole-container/VM `node_termination` does not lose already-flushed jsonl lines. +- **Observable:** After `node_termination`, the surviving prefix includes all lines that were flushed + before the last maturity boundary. If page-cache writes are lost, flushed lines vanish. +- **Mechanism:** Durability invariant. Oracle: `node_termination` scenario comparing pre-kill flush + count to post-kill parsed count. +- **Needs SUT fix:** Add `File::sync_data`/`sync_all` at flush boundaries (or document that maturity = + handed-to-OS, not durable) in the `lading_capture` jsonl/parquet sinks. +- **Evidence:** No `sync_all`/`sync_data` in `lading_capture` per grep; `jsonl.rs:53` + `BufWriter::flush` only. +- **Open questions:** Does `node_termination` preserve OS page cache to the persisted named volume? + Determines P2 vs P1. + +## capture-drift-no-silent-gap — drift correction does not silently drop unflushed intervals +- **Subsystem:** capture · **Priority:** P2 · **Assertion:** Always · **Needs target:** no +- **Statement:** When the flush-tick advances multiple ticks after a scheduling stall, the + accumulator does not overwrite unflushed ring slots, so the capture has no invisible `fetch_index` + gaps. +- **Observable:** Under >60s scheduling starvation the capture retains all intervals (no large + `fetch_index` gap). Gaps pass the current strict-monotonic validator, so loss is invisible today. +- **Mechanism:** Always. Oracle: induce scheduling starvation (Antithesis) + validator extended to + detect gaps. +- **Needs SUT fix:** Flush between `advance_tick` iterations in `state_machine.rs:229-231` drift loop; + optionally have `validate` flag large `fetch_index` gaps. +- **Evidence:** `state_machine.rs:219-232` drift loop; `accumulator.rs:466-481` overwrite. + +## split-mode-merge-partial-tolerant — split-mode capture merge tolerates one clean side when the other overruns +- **Subsystem:** capture · **Priority:** P2 · **Assertion:** Always · **Needs target:** yes +- **Statement:** In split mode, if only the sender (or only the receiver) lading overruns/crashes + leaving an unreadable parquet, the merge outcome and replicate-failure attribution are + well-defined (a clean side's captures are not needlessly discarded by the other's corruption). +- **Observable:** Merge of a truncated file + a clean file behaves per policy (fail attributed to the + corrupt side), not a silent whole-replicate loss when one side captured cleanly. +- **Mechanism:** Merge robustness. Oracle: split-mode scenario killing only one side. + DEPLOYMENT-DERIVED LEAD (validate against source; do not name the deployment). +- **Needs SUT fix:** none in lading proper; LEAD to validate against the deployment's oblivious merge + + lading's per-instance capture finalize. +- **Evidence:** Digest deployment findings; `capture_file_merge` oblivious merge. +- **Open questions:** Does a sender-only overrun fail the whole replicate when the receiver captured + cleanly? + +--- + +# Properties — payload / determinism (lading_payload) + +## block-cache-construction-terminates — block-cache construction always terminates in bounded time +- **Subsystem:** payload/determinism · **Priority:** P0 · **Assertion:** Always · **Needs target:** no +- **Statement:** `construct_block_cache_inner` terminates (with blocks or `InsufficientBlockSizes`) + in bounded time for every config; it never spins forever when `max_block_size` is below the + serializer's minimum viable block. +- **Observable:** Startup completes (or errors) within a bounded time even with a tiny + `maximum_block_size`. Confirmed live: the loop has no time/iteration cap; on repeated `EmptyBlock` + `min_block_size` stays `< max_block_size` so neither exit fires. +- **Mechanism:** Liveness (bounded startup). Oracle: small-`maximum_block_size` scenario + external + startup-time bound; SUT `reachable!` at the bounded-exit. +- **Needs SUT fix:** Cap consecutive rejections / add a time bound in `block.rs:625-673` and return + `InsufficientBlockSizes`; reject `max_block_size` below a serializer-reported floor. (trace_agent + v04 variant fixed on backup branch `456e85a3`; general case still live on main.) +- **Evidence:** `block.rs:625-673` (no cap; `min_block_size=0.25*block_size`). + +## trace-agent-v04-block-terminates — trace-agent v04 block-cache next_block terminates in bounded time +- **Subsystem:** payload/determinism · **Priority:** P0 · **Assertion:** Always · **Needs target:** no +- **Statement:** trace-agent v04 block-cache construction terminates quickly even when a serialized + trace exceeds `max_block_size`; `to_bytes` never emits a 1-byte empty msgpack array accepted as a + valid block, and construction is not O(n²) re-serialization. +- **Observable:** Construction completes in sub-second, not hours (observed 31h hang → 0.75s). An + empty result is `EmptyBlock`, not a 1-byte block. +- **Mechanism:** Liveness. Fuzz property `trace_agent_v04_cache_fixed_next_block` + bounded-time + startup assertion. +- **Needs SUT fix:** Cap consecutive rejections (`InsufficientBlockSizes`) and emit `EmptyBlock` not a + 1-byte block (fix `456e85a3` on backup branch; live on main). +- **Evidence:** `block.rs`, `trace_agent/v04.rs`; digest `456e85a3`. + +## payload-determinism-byte-identical — same seed + config yields byte-identical load +- **Subsystem:** payload/determinism · **Priority:** P1 · **Assertion:** Always · **Needs target:** no +- **Statement:** For a fixed seed and config, the sequence of bytes lading emits is identical across + runs; no wall-clock, HashMap iteration order, or hidden entropy feeds payload bytes. +- **Observable:** Two runs with identical seed/config produce identical byte streams at the sink (or + identical block-cache hashes). Violation breaks the determinism ADR and Antithesis reproducibility. +- **Mechanism:** Determinism. Oracle: byte-equality across two seeded runs, or a SUT `always!` on a + rolling hash of emitted blocks. Lead analog: SMPTNG-762 (CPU-jitter entropy) / SMPTNG-767 + (non-monotonic wall clock) are SUT bugs whose class must not exist in lading. +- **Needs SUT fix:** none expected (BTreeMap/BTreeSet ordering, rng-derived timestamps confirmed); + property guards against regressions introducing wall-clock/entropy. +- **Evidence:** Digest payload finding: BTreeMap/BTreeSet, FxHashMap index-agnostic, rng-derived + timestamps. + +## no-wall-clock-in-payloads — payload timestamps are seed-derived and monotonic, never wall-clock +- **Subsystem:** payload/determinism · **Priority:** P2 · **Assertion:** Always · **Needs target:** no +- **Statement:** Timestamps embedded in generated payloads (trace_agent, otel, templated_json) are + derived from the rng/config, never from the system wall clock, and are monotone within a stream. +- **Observable:** Payload timestamps are reproducible across runs and unaffected by a backward clock + step. Violation mirrors SMPTNG-767's out-of-order/duplicate timestamp class. +- **Mechanism:** Always. Oracle: replay under a perturbed clock (Antithesis clock control) and compare + payload timestamps for equality/monotonicity. +- **Needs SUT fix:** none currently; guards against regressions. +- **Evidence:** `trace_agent/v04.rs:326,384` rng timestamps; `block.rs` Instant feeds only progress + logging. + +## dogstatsd-tag-length-validated — DogStatsD tag_length.end() <= MIN_TAG_LENGTH is rejected upfront +- **Subsystem:** payload/determinism · **Priority:** P2 · **Assertion:** Always · **Needs target:** no +- **Statement:** A dogstatsd config with `tag_length.end() <= MIN_TAG_LENGTH` is rejected at + construction with a dedicated error naming the value, not swallowed as `Error::StringGenerate` + which silently drops the message. +- **Observable:** Such a config errors clearly through `DogStatsD::new` rather than mis-surfacing as a + pool-generation error. +- **Mechanism:** Config regression. Oracle: proptest/fixture over `tag_length` bounds asserting the + dedicated error path. +- **Needs SUT fix:** none (merged to main, `e98e3052`); property guards against regression. +- **Evidence:** `e98e3052` (#1875). + +--- + +# Properties — generators / throttle + +## throttle-divide-no-silent-underdelivery — a config that delivers at N=1 still delivers at N>1 +- **Subsystem:** generators/throttle · **Priority:** P0 · **Assertion:** Always · **Needs target:** yes +- **Statement:** `throttle.divide` shrinks per-worker capacity consistently with the block size a + worker draws, so a block accepted at `parallel_connections=1` is not rejected by every worker + (`Capacity`) at N>1, yielding silent zero delivery. +- **Observable:** For `bytes_per_second/N < block <= bytes_per_second` the aggregate delivered bytes + at N>1 are nonzero and match the N=1 rate; today every worker discards. Silent (only a `debug!` + log). +- **Mechanism:** Rate fidelity. Provable as a pure proptest against the real `Valve` (no rig); + Antithesis oracle: sink byte counter vs configured rate. +- **Needs SUT fix:** `divide` must shrink block sizing consistently with capacity, or generators must + validate `maximum_block_size <= bytes_per_second/parallel_connections` upfront. Demonstrated on + backup branch (`0868e39c`); live on main. +- **Evidence:** `lib.rs` divide `capacity/divisor`; `tcp.rs:273-276` discard+loop; digest throttle + finding. + +## linear-ramp-slope-preserved — Linear throttle aggregate ramp slope equals configured rate_of_change +- **Subsystem:** generators/throttle · **Priority:** P0 · **Assertion:** Always · **Needs target:** yes +- **Statement:** With a Linear throttle and `parallel_connections>1`, the aggregate ramp slope across + workers equals the single configured `rate_of_change`, not N times it. +- **Observable:** Aggregate delivered-rate ramp reaches max in the configured time, not 1/N of it. + Confirmed live: `divide()` divides capacities but passes rate unchanged (`rate_of_change: rate`) so + N workers each ramp at full rate. +- **Mechanism:** Rate fidelity. Pure proptest measuring aggregate slope; or Antithesis oracle + sampling sink rate over the warmup ramp. +- **Needs SUT fix:** Divide `rate_of_change` by divisor in the `lib.rs` Linear divide arm (the + call-site comment claims preservation but aggregate is N*rate). +- **Evidence:** `lib.rs` Linear divide arm: `rate_of_change: rate` unchanged; digest `914bb14a`. + +## throttle-capacity-no-zerodelivery-livelock — a block larger than throttle capacity never becomes a zero-delivery busy loop +- **Subsystem:** generators/throttle · **Priority:** P0 · **Assertion:** Always · **Needs target:** yes +- **Statement:** No config leaves a generator discarding every block (block > per-worker capacity) in + a hot loop at ~100% CPU delivering ~zero bytes; such a config is rejected at startup. +- **Observable:** A run either delivers a nonzero rate or fails fast at startup; it never burns a core + delivering nothing (0.31.2 busy-discard livelock class). +- **Mechanism:** No-livelock. Oracle: oversized-block scenario + CPU/throughput observation; or + startup-error assertion. Related to `throttle-divide-no-silent-underdelivery`. +- **Needs SUT fix:** Validate `maximum_block_size <= bytes_per_second/parallel_connections` + (post-divide) at construction, returning a clear error instead of a runtime discard/spin. +- **Evidence:** `common.rs:147-157`; `stable.rs:151-156` Capacity no-wait; tcp/udp/unix discard+loop. + +## grpc-honors-throttle — gRPC generator honors throttle rejections and configured rate +- **Subsystem:** generators/throttle · **Priority:** P0 · **Assertion:** Always · **Needs target:** yes +- **Statement:** The gRPC generator discards+counts throttle-rejected blocks (like tcp/udp) and does + not send a block regardless of the throttle result; delivered rate does not exceed configured. +- **Observable:** gRPC delivered rate `<= bytes_per_second`; on a `Capacity` error the block is not + sent. Currently `let _ = result;` ignores the outcome and sends anyway. +- **Mechanism:** Rate fidelity. Oracle: sink byte-rate vs configured under a gRPC scenario. +- **Needs SUT fix:** Honor the throttle `Result` in `grpc.rs:307-318` (discard + count + `blocks_discarded` on rejection). Landed on backup branch (`944d4be4`); live on main. +- **Evidence:** `grpc.rs:307-318` (`let _ = result;`). + +## stable-burst-envelope-bounded — stable throttle never exceeds its per-interval burst envelope +- **Subsystem:** generators/throttle · **Priority:** P1 · **Assertion:** Always · **Needs target:** no +- **Statement:** In the real async `wait_for` path under an adversarial clock, the stable throttle + grants at most `maximum_capacity` at `timeout=0` (no over-delivery) and at most + `(MAX_ROLLED_INTERVALS+1)×` with rolled capacity. +- **Observable:** Granted capacity per interval stays within the proven envelope; a + clock-perturbation-induced over-grant is a defect. +- **Mechanism:** Rate safety. SUT-side feature-gated assertion under an adversarial clock (Antithesis + controls time) + proptests. +- **Needs SUT fix:** none if Kani proofs hold; add feature-gated `assert_always!` in `stable.rs` + (landed on backup branch `3f4a6bd2`) to catch regressions. +- **Evidence:** `stable.rs`; digest `3f4a6bd2`. + +## unix-throttle-aggregate-consistent — unix_stream/unix_datagram aggregate rate matches configured bytes_per_second +- **Subsystem:** generators/throttle · **Priority:** P1 · **Assertion:** Always · **Needs target:** yes +- **Statement:** unix_stream and unix_datagram divide the throttle across `parallel_connections` so + aggregate delivery approximates `bytes_per_second`, consistent with tcp/udp, rather than delivering + `parallel_connections ×` the rate. +- **Observable:** Aggregate delivered bytes for unix generators at N>1 approximate `bytes_per_second`, + not N× it. Cross-generator inconsistency: same config key means aggregate for tcp/udp but + per-connection for unix. +- **Mechanism:** Rate fidelity. Oracle: sink byte-rate vs configured. +- **Needs SUT fix:** Add `.divide(worker_count)` in `unix_stream.rs:161-163` and + `unix_datagram.rs:186-188` (or document per-connection semantics deliberately). +- **Evidence:** `unix_stream.rs:161-163`, `unix_datagram.rs:186-188` (no divide) vs `tcp.rs`. +- **Open questions:** Is undivided per-connection throttle intentional for unix_stream? + +## discarded-blocks-counted — under-delivery is observable: discarded blocks are counted +- **Subsystem:** generators/throttle · **Priority:** P1 · **Assertion:** Always · **Needs target:** yes +- **Statement:** tcp/udp/grpc generators count throttle-rejected/discarded blocks (`blocks_discarded`) + so a run delivering ~zero bytes is distinguishable from a healthy flat-metrics run. +- **Observable:** A zero-byte-delivery run surfaces a nonzero `blocks_discarded` rather than only a + `debug!` log. Distinguishes silent under-delivery. +- **Mechanism:** Observability. Oracle: oversized-block scenario asserting + `delivered==0 ⇒ blocks_discarded>0`. +- **Needs SUT fix:** Add `blocks_discarded` counters in `tcp.rs`/`udp.rs` (`73c4805e` on backup + branch) and `grpc.rs`; live on main. +- **Evidence:** `tcp.rs:273-276` `debug!` only; digest `73c4805e`. + +## error-label-cardinality-bounded — generator error-metric label cardinality is bounded +- **Subsystem:** generators/throttle · **Priority:** P1 · **Assertion:** Always · **Needs target:** yes +- **Statement:** `connection_failure`/`request_failure` error labels are drawn from a finite set + (`io::ErrorKind`), not raw `err.to_string()`; a flapping target cannot grow capture memory without + bound. +- **Observable:** Distinct error-label values stay bounded regardless of failure diversity; capture + accumulator key count does not grow unboundedly (ADR-005 OOM class). +- **Mechanism:** Bounded cardinality / memory. Oracle: flapping-target scenario + capture key-count / + memory observation. +- **Needs SUT fix:** Map errors to `io::ErrorKind` for labels in `tcp.rs`/`udp.rs` (`32dd4cf6` on + backup branch); gRPC tonic error still raw (follow-up). Live on main. +- **Evidence:** `tcp.rs`/`udp.rs` raw `err.to_string()` labels; digest `32dd4cf6`. + +## sink-receives-bytes — the sink receives load (load-arrival non-vacuity) +- **Subsystem:** generators/throttle · **Priority:** P1 · **Assertion:** Sometimes · **Needs target:** yes +- **Statement:** Across a run the sink container receives a nonzero number of bytes from lading + generators. +- **Observable:** `sink/main.rs:82` `sometimes!(total>0)` "sink received bytes". Guards against a + whole-config class delivering nothing (divide stall, capacity livelock, throttle bypass). +- **Mechanism:** Non-vacuity of delivery on at least one timeline. Oracle already present in the + never-faulted sink container. +- **Needs SUT fix:** none (already instrumented). +- **Evidence:** `sink/main.rs:69-83`. + +## divide-by-zero-startup-error — bytes_per_second < parallel_connections fails with a clear error +- **Subsystem:** generators/throttle · **Priority:** P2 · **Assertion:** Always · **Needs target:** yes +- **Statement:** A config where `bytes_per_second` divided by `parallel_connections` rounds to zero + produces a clear startup validation error, and integer-division truncation does not silently drop + the remainder rate. +- **Observable:** Small-bps/high-connection config errors clearly at startup; aggregate delivered rate + is within one interval-quantum of configured (no systematic under-delivery of up to N-1 + bytes/interval). +- **Mechanism:** Always. Oracle: startup-error assertion + sink rate check. +- **Needs SUT fix:** Distribute the division remainder across workers, and surface `DivisionByZero` as + a config validation error naming `bytes_per_second`/`parallel_connections`. +- **Evidence:** `lib.rs` divide `DivisionByZero`; `tcp.rs:168-173` integer divide truncation. + +## unix-stream-write-error-progress — unix_stream makes progress on non-BrokenPipe write errors +- **Subsystem:** generators/throttle · **Priority:** P2 · **Assertion:** Always · **Needs target:** yes +- **Statement:** On a non-BrokenPipe, non-WouldBlock write error (e.g. `ConnectionReset`) the + unix_stream worker reconnects or advances rather than busy-looping on the same offset spamming + `request_failure`. +- **Observable:** A `ConnectionReset` receiver does not pin a core or emit a runaway `request_failure` + count; `packets_sent` is not inflated per partial write. +- **Mechanism:** No-livelock. Oracle: reset-injecting receiver + CPU/counter observation. +- **Needs SUT fix:** Handle non-BrokenPipe write errors (advance/break/reconnect) at + `unix_stream.rs:304-315`; count packets per block not per partial write. +- **Evidence:** `unix_stream.rs:304-315,293-295`. + +--- + +# Properties — observer + +## observer-pid-reuse-no-panic — observer never aborts on PID reuse / mismatch +- **Subsystem:** observer · **Priority:** P0 · **Assertion:** Unreachable · **Needs target:** yes +- **Statement:** The stat sampler degrades (skips the stale sample) instead of asserting + `cur_pid==pid` when a recycled/mismatched PID is read. +- **Observable:** Target exit + PID recycle races do not SIGABRT the run. Confirmed live: + `assert!(cur_pid == pid)` at `stat.rs:82`. +- **Mechanism:** Unreachable (the assert must never fire). Oracle: rapid target-exit/PID-reuse + scenario + panic hook. +- **Needs SUT fix:** Replace the `assert!` at `stat.rs:82` with a skip-and-continue (fix `6aa1b1ba` + on backup branch; live on main). +- **Evidence:** `stat.rs:82` `assert!(cur_pid == pid)`. + +## observer-process-vanish-no-panic — observer never panics when a target vanishes mid-listing +- **Subsystem:** observer · **Priority:** P0 · **Assertion:** Unreachable · **Needs target:** yes +- **Statement:** `ProcessDescendantsIterator` degrades to zero descendants when `Process::new` fails, + rather than panicking, when a target exits before descendant listing. +- **Observable:** A target exiting mid-listing does not crash the run. +- **Mechanism:** Unreachable. Oracle: target-exits-during-sampling scenario + panic hook. +- **Needs SUT fix:** Yield an empty iterator instead of `panic!` at `process_descendents.rs:13` (fix + `7e1d2968` on backup branch; live on main). +- **Evidence:** `process_descendents.rs:13` `panic!`. + +## observer-cpu-max-parse-no-panic — observer never panics on malformed/truncated cpu.max +- **Subsystem:** observer · **Priority:** P0 · **Assertion:** Unreachable · **Needs target:** yes +- **Statement:** `parse_allowed_cores` is bounds-checked (no index panic, guards a zero period) so a + malformed/truncated cgroup `cpu.max` degrades rather than aborting. +- **Observable:** A truncated `cpu.max` read (fault-injected) does not SIGABRT the run. +- **Mechanism:** Unreachable. Oracle: malformed-`cpu.max` fault + panic hook. +- **Needs SUT fix:** Bounds-check parsing / guard zero period (fix `6aa1b1ba` on backup branch; live + on main). +- **Evidence:** `stat.rs` `cpu.max` parse; digest `6aa1b1ba`. + +## observer-transient-read-not-fatal — a transient observer read error does not kill the run +- **Subsystem:** observer · **Priority:** P1 · **Assertion:** Always · **Needs target:** yes +- **Statement:** A single transient procfs/cgroup/wss read error is best-effort (log + skip that + component's sample); it does not `?`-propagate and terminate the whole experiment. +- **Observable:** An injected transient read error yields a skipped sample + warning, not a dead run. + Only persistent problems show as repeated warnings + absent metrics. +- **Mechanism:** Liveness. Oracle: transient-read-fault scenario asserting the run continues to + self-termination. +- **Needs SUT fix:** Treat component reads as best-effort in `observer/linux.rs` `sample()` (fix + `30b86a71` on backup branch; live on main). +- **Evidence:** `observer/linux.rs`; digest `30b86a71`. + +## observer-pid-identity-fingerprint — observer samples the identified target, not a PID-reuse impostor +- **Subsystem:** observer · **Priority:** P2 · **Assertion:** Always · **Needs target:** yes +- **Statement:** lading reports target-exit iff the process identified at startup exits; after a + watched PID is recycled, the observer/pidfd path does not silently attach to an unrelated process + and emit wrong metrics. +- **Observable:** Metrics attributed to the target correspond to the original process identity; a + recycled PID does not produce plausible-but-wrong metrics with no error. +- **Mechanism:** Metric integrity. Oracle: PID-reuse scenario checking metric identity. LEAD (TOCTOU, + `PID_MAX` small). +- **Needs SUT fix:** Capture a start-time/identity fingerprint (proc `start_time`) and validate it in + the pidfd/sampler paths (`target.rs:302-332`, observer sampler). +- **Evidence:** `target.rs:302-332,246-257`; observer/linux sampler. + +## get-available-memory-cgroup-chain — memory limit reflects the tightest cgroup v2 ancestor limit +- **Subsystem:** observer · **Priority:** P2 · **Assertion:** Always · **Needs target:** no +- **Statement:** `get_available_memory` walks the cgroup v2 ancestor chain and returns the minimum + `memory.max`, matching kernel hierarchical enforcement, rather than reading `max` from a namespaced + root and believing it has `u64::MAX`. +- **Observable:** Reported available memory equals the effective container limit, not `u64::MAX`, in a + cgroup-namespaced container. +- **Mechanism:** Accuracy. Oracle: deterministic test with synthetic cgroup files (injectable reader). +- **Needs SUT fix:** none (merged to main, `1085887c`); property guards against regression via the + injectable `get_available_memory_with` reader. +- **Evidence:** `1085887c`; digest regression finding. + +--- + +# Properties — no-panic (umbrella + specific sites) + +## no-panic-anywhere — lading never panics (panic hook = Unreachable) +- **Subsystem:** no-panic · **Priority:** P0 · **Assertion:** Unreachable · **Needs target:** no +- **Statement:** No panic occurs anywhere in the lading SUT under any config, fault, timing, or + shutdown path; the panic hook must never report "lading panicked". +- **Observable:** `antithesis_hooks.rs:27` panic hook fires `unreachable!` "lading panicked" + `{message,location}` on any panic before `panic=abort` SIGABRTs the container. Directly + Antithesis-visible. +- **Mechanism:** Umbrella no-panic invariant. Assertion type Unreachable because a panic is a point in + the code that must never be reached; the existing panic hook is the oracle. `panic=abort` makes + every hit a hard, observable process abort. SUT-side probe already present. +- **Needs SUT fix:** none (hook already wired); individual panic sites listed as separate properties + need the ADR-compliant Result conversions. +- **Evidence:** `antithesis_hooks.rs:13-33` panic hook; `Cargo.toml:115,120` `panic=abort`. + +## observer-target-pid-recv-no-panic — observer returns an error, not a panic, when the target PID never arrives +- **Subsystem:** no-panic · **Priority:** P0 · **Assertion:** Unreachable · **Needs target:** yes +- **Statement:** If a Binary target fails to spawn or exits before sending its PID, the observer + returns an error instead of `.expect("catastrophic failure")` panicking on the closed channel. +- **Observable:** A bad target path / instant-exit target does not panic the observer task. Currently + the `recv()` `Err(Closed)` hits `.expect`. +- **Mechanism:** Unreachable. Oracle: instant-exit/bad-path target scenario + panic hook (currently + only partly masked as a join-error log). +- **Needs SUT fix:** Handle `recv()` `Err` and `None` with a returned error at `observer.rs:114-120`. +- **Evidence:** `observer.rs:114-120`; `target.rs:395-396`. + +## block-cache-zero-max-no-panic — maximum_block_size of 0 is rejected, not a random_range panic +- **Subsystem:** no-panic · **Priority:** P1 · **Assertion:** Unreachable · **Needs target:** no +- **Statement:** A `maximum_block_size` that resolves to 0 is rejected at validation; block-cache + construction never calls `rng.random_range` on an empty `0..0` range. +- **Observable:** Config with `maximum_block_size '0 B'` produces a clean startup error, not a panic. + Currently reachable via `grpc.rs` forwarding unvalidated `as_u128()`. +- **Mechanism:** Unreachable (the empty-range `random_range` must never be reached). Oracle: + zero-block-size scenario + panic hook. +- **Needs SUT fix:** Add a lower-bound (`>=1`, or `>=` serializer floor) check on `maximum_block_size` + in `block.rs:214-220` and at generator call sites (e.g. `grpc.rs:205-223`). +- **Evidence:** `block.rs:628` `random_range(min..max)`; `:214-220` guard misses 0. + +## per-generator-semaphore-no-panic — two HTTP (or two Splunk-HEC) generators coexist without panic +- **Subsystem:** no-panic · **Priority:** P1 · **Assertion:** Unreachable · **Needs target:** yes +- **Statement:** Each HTTP/Splunk-HEC generator instance owns its own connection semaphore; + configuring two such generators does not panic on a process-wide `OnceCell::set` and gives + independent connection limits. +- **Observable:** A config with two `Http` generators starts without the second `new()` panicking; + per-generator concurrency limits are independent. `config.generator` is a `Vec` so this config is + allowed. +- **Mechanism:** Unreachable (OnceCell double-set panic must never be reached). Oracle: + two-http-generator scenario + panic hook. +- **Needs SUT fix:** Replace the static `CONNECTION_SEMAPHORE` OnceCell with a per-instance + `Arc` in `http.rs:37/187-189` and `splunk_hec.rs:51/243-245`; make the hot-path + `.expect("semaphore closed")` stop the worker gracefully. Landed on backup branch (`5f8c375e`); + live on main. +- **Evidence:** `http.rs:37,187-189`; `splunk_hec.rs:51,243-245`; `config.rs:101` `Vec`. + +## splunk-hec-response-parse-no-panic — Splunk-HEC response parsing never panics on a non-HecResponse body +- **Subsystem:** no-panic · **Priority:** P1 · **Assertion:** Unreachable · **Needs target:** yes +- **Statement:** The Splunk-HEC generator's spawned request task handles a non-HecResponse body + (empty, HTML error page, "ok") without panicking via `.expect` on serde_json parse. +- **Observable:** A blackhole/target returning a non-JSON body does not abort the detached task. + `panic=abort` makes this whole-process fatal. +- **Mechanism:** Unreachable. Oracle: blackhole returning a plain-text body + panic hook. +- **Needs SUT fix:** Replace `serde_json::from_slice::(...).expect(...)` at + `splunk_hec.rs:371-374` with error handling; also track the detached task's handle. +- **Evidence:** `splunk_hec.rs:371-374`. + +## generator-addr-uri-validation-no-panic — malformed addr/target_uri is a Result error, not a construction panic +- **Subsystem:** no-panic · **Priority:** P1 · **Assertion:** Unreachable · **Needs target:** no +- **Statement:** An unresolvable/malformed socket address (tcp/udp) or `target_uri` (grpc) yields a + clean config error, not an `.expect` panic at generator construction. +- **Observable:** Config with a bad addr/uri fails startup with an error; the panic hook does not + fire. +- **Mechanism:** Unreachable. Oracle: malformed-config scenario + panic hook. +- **Needs SUT fix:** Return `Result` errors instead of `.expect` at `tcp.rs:154-159`, + `udp.rs:164-169`, `grpc.rs:226-231,245-247`. +- **Evidence:** `tcp.rs:154-159`; `udp.rs:164-169`; `grpc.rs:226-247`. + +## tcp-rr-listener-no-panic — tcp_rr blackhole returns an error, not a panic, on bind failure +- **Subsystem:** no-panic · **Priority:** P1 · **Assertion:** Unreachable · **Needs target:** no +- **Statement:** tcp_rr blackhole listener setup failures (address in use, stale bind) return + `Error::Bind` rather than panicking, including the `threads>1` thread-0 prebuild path on the main + async task. +- **Observable:** A pre-bound data port yields a clean error, not a SIGABRT. Currently + `create_listener` uses `expect()`/`panic!` (`tcp_rr.rs:345-346`) and thread-0 prebuild runs in + async `run()` at `:179`. +- **Mechanism:** Unreachable. Oracle: pre-bound-port scenario with `threads>1` + panic hook. +- **Needs SUT fix:** Return `Result` from `create_listener` (`tcp_rr.rs:313-346`) instead of + `expect`/`panic`. +- **Evidence:** `tcp_rr.rs:345-346,179`. +- **Open questions:** Is tcp_rr ever configured with `threads>1`? + +## logrotate-stale-tick-noop — logrotate_fs treats a stale tick as a no-op, never a panic +- **Subsystem:** no-panic · **Priority:** P1 · **Assertion:** Unreachable · **Needs target:** no +- **Statement:** `Model::advance_time` treats a tick below current model time as a no-op early return + (a benign FUSE scheduling reorder), never asserting/panicking. +- **Observable:** Reordered FUSE ops presenting a stale tick do not crash the logrotate_fs generator. + No clock fault needed to trigger. +- **Mechanism:** Unreachable (the stale-tick assert must never fire). Oracle: concurrent FUSE op + scenario + panic hook. +- **Needs SUT fix:** Make `advance_time` early-return on a stale tick in `logrotate_fs/model.rs` (fix + `220850e5` on backup branch; live on main). +- **Evidence:** `logrotate_fs/model.rs`; digest `220850e5`. + +## arbitrary-block-nonzero-no-panic — fuzz Arbitrary Block construction handles zero total_bytes without panic +- **Subsystem:** no-panic · **Priority:** P2 · **Assertion:** Unreachable · **Needs target:** no +- **Statement:** The `arbitrary::Arbitrary` impl for `Block` handles a generated `total_bytes` of 0 + without `.expect` panicking, so a fuzz run is not aborted by benign input. +- **Observable:** A fuzz input yielding `total_bytes==0` is rejected gracefully (`arbitrary::Error`) + not a `NonZeroU32` expect panic. Only compiled under the `arbitrary` feature (fuzz harness), so no + production impact. +- **Mechanism:** Unreachable within the fuzz harness. Oracle: `cargo fuzz run`. +- **Needs SUT fix:** Return `Err(arbitrary::Error::IncorrectFormat)` instead of `.expect` at + `block.rs:119-127`. +- **Evidence:** `block.rs:119-127`. + +--- + +# Properties — blackhole / config + +## unix-datagram-blackhole-removes-stale-socket — unix_datagram blackhole removes a stale socket before bind +- **Subsystem:** blackhole/config · **Priority:** P1 · **Assertion:** Always · **Needs target:** yes +- **Statement:** The unix_datagram blackhole actually removes a leftover socket file before binding, + so it starts cleanly after a hard-kill restart instead of failing bind with `EADDRINUSE`. +- **Observable:** After a `node_termination` leaving a stale socket, the blackhole binds successfully + and the target keeps its sink. Confirmed live: + `let _res = tokio::fs::remove_file(...).map_err(...)` builds a lazy future that is never awaited. +- **Mechanism:** Restart resilience. Oracle: restart-with-stale-socket scenario asserting bind + succeeds. LEAD: impact depends on whether the socket path is on a persisted volume. +- **Needs SUT fix:** Await the `remove_file` future in `unix_datagram.rs:95` (drop the lazy + `TryFutureExt::map_err` binding). +- **Evidence:** `unix_datagram.rs:95` lazy future. +- **Open questions:** Is the unix socket path on a persisted named volume in the deployment? + +## datadog-blackhole-accept-resilient — Datadog blackhole keeps accepting after a transient accept error +- **Subsystem:** blackhole/config · **Priority:** P1 · **Assertion:** Always · **Needs target:** yes +- **Statement:** The Datadog blackhole's accept loop logs-and-continues on an `accept()` error and + never wedges (silently ceasing to accept while appearing alive), so it never backpressures the + target. +- **Observable:** Under fd exhaustion / transient accept errors the blackhole keeps serving new + connections. Confirmed live: `Ok((stream,_addr)) = listener.accept()` fallible select arm with no + else disables the branch on `Err`, then blocks on shutdown forever. +- **Mechanism:** Blackhole never-backpressure. Oracle: fd-exhaustion fault scenario asserting the + target's connections still succeed. +- **Needs SUT fix:** Match `accept()` and continue on `Err` (as `common.rs:89-96` does) at + `datadog.rs:209`. +- **Evidence:** `datadog.rs:207-212` fallible select arm. + +## blackhole-never-backpressures-target — blackhole responds to the target regardless of capture-channel saturation +- **Subsystem:** blackhole/config · **Priority:** P1 · **Assertion:** Always · **Needs target:** yes +- **Statement:** A blackhole does not block its HTTP response to the target on a full bounded capture + channel; a slow/saturated capture manager cannot stall the target. +- **Observable:** With a slow capture drain, target request latency stays bounded; the Datadog + blackhole does not `await send().await` per metric point before responding. +- **Mechanism:** Never-backpressure. Oracle: slow-capture-drain fault scenario measuring target + response latency. +- **Needs SUT fix:** Use `try_send` + count-on-drop (consistent with the manager histogram path) + instead of blocking `send().await` in `datadog.rs` `handle_v2_protobuf` (398,412,416) before the + response is built. +- **Evidence:** `datadog.rs:296-299` response after per-point awaits; `lib.rs:53-56` bounded send. + +## sqs-receive-message-bounded — SQS blackhole bounds ReceiveMessage response size +- **Subsystem:** blackhole/config · **Priority:** P2 · **Assertion:** Always · **Needs target:** no +- **Statement:** The SQS blackhole caps `max_number_of_messages` so a single target-controlled request + cannot force an enormous allocation and OOM the blackhole. +- **Observable:** A request with a huge `max_number_of_messages` produces a bounded response (real SQS + caps at 10), not an unbounded String allocation. +- **Mechanism:** Amplification/OOM. Oracle: adversarial-request scenario + memory observation. +- **Needs SUT fix:** Clamp `num_messages` to a max (e.g. 10) at `sqs.rs:257-267` before the + `0..num_messages` loop. +- **Evidence:** `sqs.rs:362-370,257-267`. + +## config-numeric-fields-validated — numeric config fields are validated at load, not deferred to runtime failure +- **Subsystem:** blackhole/config · **Priority:** P2 · **Assertion:** Always · **Needs target:** no +- **Statement:** parquet/zstd `compression_level` (1-22) and `sample_period_milliseconds` (>0) are + range-checked at config load, so an out-of-range value fails early rather than corrupting a capture + write or driving a tight sampling loop. +- **Observable:** An out-of-range `compression_level` or a zero `sample_period` is rejected at startup + (the documented crash-early location), not surfaced as a mid-run zstd error / busy loop. +- **Mechanism:** Validate-early. Oracle: out-of-range-config scenario asserting clean startup error. +- **Needs SUT fix:** Add range checks in `config.rs:186-198` (`compression_level`) and `:106-107` + (`sample_period_milliseconds > 0`). +- **Evidence:** `config.rs:186-198,106-107`. + +## recorded-traffic-crash-consistency — blackhole-recorded traffic files are crash-consistent +- **Subsystem:** blackhole/config · **Priority:** P2 · **Assertion:** Always · **Needs target:** yes +- **Statement:** The blackhole traffic recorder writes files that, when compressed, remain decodable + up to the last flushed frame after an abrupt kill, and are deterministic for a fixed input. +- **Observable:** A recorder file from a SIGKILLed blackhole decodes to a valid prefix (analogous to + jsonl), not undecodable-past-last-frame corruption. +- **Mechanism:** Crash-consistency. Oracle: `node_termination` scenario + external decode check. LEAD. +- **Needs SUT fix:** none identified; LEAD to validate zstd framing / flush boundaries of the recorder + (SMPTNG-390 / record-policy work #1911/#1895). +- **Evidence:** Digest Atlassian SMPTNG-390; commits #1911,#1895. +- **Open questions:** Are recorded-traffic files zstd-framed such that a truncated stream is a valid + prefix? + +--- + +# Scenarios needed (harness config-variation + fault menu) + +- **unreachable-target:** generator points at an address/socket/uri/container that never + binds/appears; experiment timer fires; assert lading exits within `max_shutdown_delay` (drives + tcp/udp/unix/grpc connect-loop hangs and docker-target-discovery hang). +- **stalled-receiver:** blackhole/sink accepts but never reads (full socket buffer) and slow HTTP + responses; assert bounded shutdown + no 100% CPU busy-spin (unix_stream partial-write, http acquire + backpressure). +- **multi-generator-config:** two Http generators and two SplunkHec generators in one config; assert + no OnceCell double-set panic and independent connection limits. +- **small-and-zero-block-size:** `maximum_block_size` tiny and `'0 B'` with dogstatsd/otel/trace_agent; + assert bounded block-cache construction and no `random_range` panic. +- **oversized-block-vs-capacity:** `maximum_block_size` in `(bps/N, bps]` with `parallel_connections>1`; + assert nonzero delivery, no zero-delivery busy loop, `blocks_discarded>0` on true rejection (divide + stall / capacity livelock). +- **linear-ramp-parallel:** Linear throttle + `parallel_connections>1`; sample aggregate delivered + rate over warmup; assert slope == configured `rate_of_change`. +- **determinism-replay:** same seed+config run twice; assert byte-identical load at the sink (and + under a perturbed clock, identical payload timestamps). +- **node-termination-mid-run:** SIGKILL the whole lading container mid-experiment; assert jsonl + surviving prefix validates and any readable parquet is internally consistent (capture + crash-consistency). +- **sigterm-stop:** send SIGTERM to lading (deployment orchestrator-stop path); assert graceful drain, + parquet footer written, no orphaned children. +- **capture-write-fault:** inject disk-full/EIO on a capture flush tick; assert clean fatal exit with + finalized parquet, no SIGABRT (panic hook silent). +- **target-lifecycle-races:** target that exits before sending its PID, vanishes mid-listing, or whose + PID is recycled; assert observer degrades (no `assert!`/`expect` panic) and does not attribute + impostor metrics. +- **blackhole-restart-stale-socket:** leave a stale unix_datagram socket then restart the blackhole; + assert bind succeeds. +- **accept-fault-injection:** fd exhaustion / transient accept errors at the datadog (and other) + blackholes; assert the blackhole keeps serving the target (no wedge). +- **slow-capture-drain:** throttle the capture manager drain; assert the datadog blackhole responds to + the target within a bounded latency (no channel-full backpressure). +- **malformed-config:** bad addr/target_uri, out-of-range `compression_level`, `sample_period` 0, + dogstatsd `tag_length.end()<=MIN_TAG_LENGTH`; assert clean startup errors, no panic. +- **split-mode-partial-kill:** two-instance split (sender `--no-target` + receiver observer); kill only + one side; assert merge/attribution is well-defined and a clean side's captures are not needlessly + lost. +- **tcp_rr-threads>1-bind-conflict:** pre-bind the data port with `num_threads>1`; assert + `Error::Bind`, no panic on the main async task. diff --git a/test/antithesis/scratchbook/property-relationships.md b/test/antithesis/scratchbook/property-relationships.md new file mode 100644 index 000000000..5dcf163aa --- /dev/null +++ b/test/antithesis/scratchbook/property-relationships.md @@ -0,0 +1,287 @@ +--- +sut_path: /home/ssm-user/src/lading +commit: 51148899 +updated: 2026-07-24T21:28:15Z +external_references: + - name: the deployment (production runner + local dev harness) + why: real shutdown/deploy model — how lading is launched, stopped (self-exit on experiment timer; SIGKILL via docker rm --force; watchdog (warmup+samples+30)*1.2), and how captures are merged/attributed. Grounds which shutdown/termination properties are LIVE vs MOOT on production paths. + - name: Jira (project SMPTNG) + Confluence (datadoghq.atlassian.net) + why: existing bug tickets and design docs — SMPTNG-725 (hang-in-spin after "lading shutdown"), SMPTNG-719/697 (ungraceful-termination telemetry loss), SMPTNG-694 (zstd-only captures / crash-consistency), SMPTNG-390 (blackhole traffic recorder), SMPTNG-762/767 (entropy / wall-clock determinism analogs), SMPTNG-735 Antithesis Triage epic; DST/determinism Confluence pages. + - name: the whole lading repo (this checkout) + why: source of truth for every evidence line-number, ADR (determinism / no-panic / pre-computation / performance-first), the existing panic hook (antithesis_hooks.rs:27), the harness under test/antithesis/, and the backup/blt branches where several fixes live but are NOT on main. +--- + +# Property relationships + +Bug-hunting invariant catalog for lading, clustered by **subsystem**, tagged +**graceful / abrupt / any-path** (which termination timeline exercises it), and +annotated with **dominance / overlap** (which property is the umbrella and which +are its subsumed or dependent members). Every slug below appears in the catalog +JSON; nothing is invented here. + +Legend +- **Path**: `graceful` = experiment-timer or SIGTERM drain; `abrupt` = SIGKILL / + node_termination / hard restart; `any` = config-load / steady-state / rate / + no-panic invariant independent of the termination timeline. +- **Type**: Antithesis assertion type (Unreachable / Always / Sometimes). +- **P**: priority. **main?**: `LIVE` = defect present on `main`; `fix-on-main` = + merged; `guard` = holds, property guards against regression. + +Cross-cutting note used throughout: fixes referenced from git history +(divide stall, linear ramp, gRPC throttle bypass, observer panics, logrotate +stale-tick, per-generator semaphore, capture abort, silent discards, unbounded +error labels, block-cache/trace-agent hangs) live on backup/blt branches, **not +`main`** — they are LIVE regressions here. + +--- + +## 1. no-panic (umbrella over every Unreachable panic site) + +**Dominance:** `no-panic-anywhere` is the umbrella. It is not proved directly; +it is the disjunction "no panic-site property is violated on any timeline", +observed through the existing panic hook (`antithesis_hooks.rs:27`, +`unreachable! "lading panicked"`) plus `panic=abort`. Every `Unreachable` +property in **any** subsystem below is a specific witness that would trip this +umbrella. Adding a new panic-site property = narrowing the umbrella, never +replacing it. + +| slug | Path | Type | P | main? | Notes / evidence | +|---|---|---|---|---|---| +| `no-panic-anywhere` | any | Unreachable | P0 | guard | Umbrella; hook already wired at `antithesis_hooks.rs:13-33`. | +| `block-cache-zero-max-no-panic` | any (startup) | Unreachable | P1 | LIVE | `random_range(0..0)` at `block.rs:628`; grpc.rs forwards unvalidated. | +| `per-generator-semaphore-no-panic` | any (startup) | Unreachable | P1 | LIVE | static `OnceCell::set` double-set for 2×Http/2×SplunkHec. | +| `splunk-hec-response-parse-no-panic` | any | Unreachable | P1 | LIVE | `.expect` on `from_slice::` at splunk_hec.rs:371-374. | +| `generator-addr-uri-validation-no-panic` | any (startup) | Unreachable | P1 | LIVE | `.expect` on addr/uri: tcp.rs:154, udp.rs:164, grpc.rs:226-247. | +| `observer-target-pid-recv-no-panic` | any | Unreachable | P0 | LIVE | `.expect("catastrophic failure")` on closed PID channel, observer.rs:114-120. | +| `tcp-rr-listener-no-panic` | any (startup) | Unreachable | P1 | LIVE | expect/panic! in create_listener, tcp_rr.rs:345-346; threads>1 on async task at :179. | +| `logrotate-stale-tick-noop` | any | Unreachable | P1 | LIVE | stale-tick assert in logrotate_fs/model.rs (fix 220850e5 not on main). | +| `arbitrary-block-nonzero-no-panic` | any (fuzz only) | Unreachable | P2 | LIVE | fuzz-feature only; `NonZeroU32` expect at block.rs:119-127. | + +**Overlap across subsystems:** the observer panic trio +(`observer-pid-reuse-no-panic`, `observer-process-vanish-no-panic`, +`observer-cpu-max-parse-no-panic`) and `capture-write-failure-not-abort` are +*also* `Unreachable` witnesses of `no-panic-anywhere` but are catalogued under +their owning subsystem (observer / capture) because their oracle and fix live +there. Treat them as members of this umbrella when counting panic coverage. + +--- + +## 2. lifecycle/shutdown (IMMEDIATE PRIORITY) + +**Dominance:** `shutdown-completes-bounded` is the P0 liveness **umbrella**: +"once shutdown is signaled, lading exits within `max_shutdown_delay` (30s)". It +is violated by any one of the per-loop hang properties below. `needs_target: +true` on the umbrella; its members are the concrete hang sites to instrument. + +Members subsumed by `shutdown-completes-bounded`: + +| slug | Path | Type | P | main? | Hang site | +|---|---|---|---|---|---| +| `tcp-connect-loop-shutdown-responsive` | graceful | Always | P0 | LIVE | connect before select, tcp.rs:230-283. | +| `unix-stream-partial-write-shutdown` | graceful | Always | P0 | LIVE | inner write loop + yield_now spin, unix_stream.rs:281-318. | +| `unix-datagram-connect-loop-shutdown` | graceful | Always | P0 | LIVE | connect/retry loop, unix_datagram.rs:246-264. | +| `grpc-connect-loop-shutdown` | graceful | Always | P1 | LIVE | connect loop, grpc.rs:287-299. | +| `target-wait-bounded` | graceful | Always | P0 | LIVE | untimed `wait()`, target.rs:432 / inspector.rs:176. | +| `docker-target-discovery-bounded` | graceful (startup) | Always | P0 | LIVE | unbounded discovery poll, target.rs:212-244 — **production observer path**. | +| `capture-finalize-bounded` | graceful | Always | P1 | LIVE | unbounded `handle.await`, lading.rs:713-715. | + +Graceful-contract properties (the entry into that timeline and its cleanup): + +| slug | Path | Type | P | main? | Notes | +|---|---|---|---|---|---| +| `sigterm-graceful-drain` | graceful | Always | P0 | LIVE | no `SignalKind::terminate` arm at lading.rs:658. MOOT under node_termination (SIGKILL), LIVE on orchestrator-stop. Confirmed by SMPTNG-719/697. | +| `target-grace-period-honored` | graceful | Always | P1 | LIVE | ~0 grace: JoinSet dropped -> kill_on_drop SIGKILL after flush, lading.rs:690-717. Open Q: is ~0 intentional? | +| `orphaned-children-on-signal-death` | graceful + abrupt | Always | P1 | LIVE | kill_on_drop can't fire on untrapped signal; single-pid not process-group. MOOT under whole-container SIGKILL, LIVE in shared-PID-ns. | +| `lading-completes-and-exits-cleanly` | graceful | Sometimes | P0 | guard | **Non-vacuity anchor**: at least one timeline reaches exit-0 + footer-complete capture. Guards against "every timeline hangs/aborts". | + +**Overlap / dependency edges (lifecycle ↔ capture):** +- `parquet-footer-on-graceful-exit` (capture) **depends on** both + `sigterm-graceful-drain` and `capture-write-failure-not-abort` holding — it is + the operational payoff of a correct graceful path. +- `capture-finalize-bounded` sits in both clusters: it is a shutdown-liveness + member *and* a capture-finalization concern; catalogued here because its + violation is a hang, not a corrupt file. +- Deployment reality (external_reference #1): production stops lading only via + self-exit or SIGKILL, so `sigterm-graceful-drain` / `orphaned-children` / + `target-grace-period-honored` / `target-wait-bounded` are **MOOT on the + production runner path** but LIVE on the orchestrator-stop / Binary-target / + bare-host paths — keep them, mark the moot-ness, drive them with an explicit + `kill -TERM` harness step. SMPTNG-725 is the strongest deployment match for the + hang class. + +--- + +## 3. capture + +**Dominance:** two independent umbrellas by timeline. +- **graceful:** `parquet-footer-on-graceful-exit` — every graceful exit yields a + readable footer-complete parquet. Depends on lifecycle graceful path (§2) and + on `capture-write-failure-not-abort`. +- **abrupt:** `jsonl-prefix-valid-after-kill` — every SIGKILLed jsonl file is a + valid strictly-monotonic prefix. Already implemented as the harness checker + (`anytime_capture_consistent.rs`), so it is the reference oracle other capture + properties reuse. + +| slug | Path | Type | P | main? | Notes | +|---|---|---|---|---|---| +| `capture-write-failure-not-abort` | any (steady) | Unreachable | P0 | LIVE | `.block_on(start()).expect` + panic=abort SIGABRTs mid-flush, lading.rs:476/501/526. Also a `no-panic-anywhere` witness. | +| `parquet-footer-on-graceful-exit` | graceful | Always | P0 | guard/dep | Footer only in close(), parquet.rs:307-324. | +| `jsonl-prefix-valid-after-kill` | abrupt | Always | P1 | guard | Oracle live: `anytime_capture_consistent.rs:44,49`; MIN_RECORDS=10. | +| `capture-no-fsync-durability` | abrupt | Always | P2 | LIVE | no sync_all/sync_data; jsonl.rs:53 flush-only. Open Q: does node_termination preserve page cache to the named volume? (P2↔P1). | +| `multi-format-parquet-not-forfeited` | graceful | Always | P1 | LIVE | jsonl.close()? before parquet.close()?, multi.rs:69-72. | +| `capture-drift-no-silent-gap` | any (stall) | Always | P2 | LIVE | drift loop advances ticks w/o flush, state_machine.rs:219-232; gaps invisible to validator. | +| `capture-histogram-drops-counted` | any (load) | Always | P1 | LIVE | try_send warn-drop, manager.rs:121-133 (counter fix 39d8ae56 not on main). | +| `split-mode-merge-partial-tolerant` | abrupt | Always | P2 | LIVE(lead) | deployment-derived; one clean side must not be lost to the other's corruption. Open Q per digest. | + +**Overlap:** `capture-write-failure-not-abort` belongs to both this cluster and +the §1 no-panic umbrella. `capture-histogram-drops-counted` and +`capture-drift-no-silent-gap` are **observability** siblings — loss must be +*counted / visible*, not silent — analogous to `discarded-blocks-counted` in §5. +Note SMPTNG-694: captures are zstd-only now; the "valid prefix" assumption for +`jsonl-prefix-valid-after-kill` must be re-checked against zstd framing (also see +`recorded-traffic-crash-consistency` in §7). + +--- + +## 4. payload/determinism + +**Dominance:** `payload-determinism-byte-identical` is the umbrella ADR +invariant (same seed+config -> identical bytes). `no-wall-clock-in-payloads` is +its most likely violation vector (a specific entropy/clock source), so it is a +subsumed member, not a peer. The two block-cache properties are **liveness / +no-panic** members that live here because construction is a payload concern. + +| slug | Path | Type | P | main? | Notes | +|---|---|---|---|---|---| +| `payload-determinism-byte-identical` | any | Always | P1 | guard | BTreeMap/BTreeSet, rng-derived timestamps confirmed. Analog leads: SMPTNG-762/767. | +| `no-wall-clock-in-payloads` | any | Always | P2 | guard | Member of the determinism umbrella; replay under perturbed clock. | +| `block-cache-construction-terminates` | any (startup) | Always | P0 | LIVE | no time/iteration cap, block.rs:625-673 (general case live; v04 fixed off-main). | +| `trace-agent-v04-block-terminates` | any (startup) | Always | P0 | fix-on-main | 31h hang -> 0.75s (456e85a3). Specialization of the line above for v04. | +| `dogstatsd-tag-length-validated` | any (startup) | Always | P2 | fix-on-main | e98e3052 (#1875); guards regression. | + +**Overlap:** `block-cache-zero-max-no-panic` (§1) is the panic-site sibling of +`block-cache-construction-terminates` — same construction path, one is a hang, +the other an empty-range panic; both reachable via unvalidated +`maximum_block_size` (`grpc.rs:205-223`). `trace-agent-v04-block-terminates` is +**dominated by** `block-cache-construction-terminates` (the general bounded- +construction invariant); keep the v04 one as the regression witness. + +--- + +## 5. generators/throttle + +**Dominance:** two umbrellas. +- **rate-fidelity:** aggregate delivered rate == configured `bytes_per_second`. + Members: `throttle-divide-no-silent-underdelivery`, `linear-ramp-slope-preserved`, + `grpc-honors-throttle`, `unix-throttle-aggregate-consistent`, + `divide-by-zero-startup-error`, bounded above by `stable-burst-envelope-bounded`. +- **no-livelock / observability:** a bad config fails fast or delivers nonzero, + never a 100%-CPU zero-delivery spin, and under-delivery is *counted*. Members: + `throttle-capacity-no-zerodelivery-livelock` (umbrella here), + `discarded-blocks-counted`, `unix-stream-write-error-progress`, + `error-label-cardinality-bounded`. +- **non-vacuity:** `sink-receives-bytes` (Sometimes) proves the whole delivery + pipeline is not dead on at least one timeline — it dominates as a floor under + every rate-fidelity member (if it fails, a rate check is vacuous). + +| slug | Path | Type | P | main? | Cluster | +|---|---|---|---|---|---| +| `throttle-divide-no-silent-underdelivery` | any | Always | P0 | LIVE | rate-fidelity; delivers at N=1 must deliver at N>1 (0868e39c off-main). | +| `linear-ramp-slope-preserved` | any | Always | P0 | LIVE | rate-fidelity; Linear divide leaves rate_of_change => N× slope (914bb14a off-main). | +| `grpc-honors-throttle` | any | Always | P0 | LIVE | rate-fidelity; `let _ = result;` grpc.rs:307-318 (944d4be4 off-main). | +| `unix-throttle-aggregate-consistent` | any | Always | P1 | LIVE | rate-fidelity; unix gens miss `.divide()`. Open Q: per-connection intentional? | +| `divide-by-zero-startup-error` | any (startup) | Always | P2 | LIVE | rate-fidelity edge; bps clear error + remainder distributed. | +| `stable-burst-envelope-bounded` | any | Always | P1 | guard | rate-fidelity ceiling; Kani-backed, feature-gated assert (3f4a6bd2 off-main). | +| `throttle-capacity-no-zerodelivery-livelock` | any | Always | P0 | LIVE | no-livelock umbrella; block>capacity busy-discard spin. | +| `discarded-blocks-counted` | any | Always | P1 | LIVE | observability; blocks_discarded counter (73c4805e off-main). | +| `unix-stream-write-error-progress` | any (fault) | Always | P2 | LIVE | no-livelock; non-BrokenPipe reset spins offset, unix_stream.rs:304-315. | +| `error-label-cardinality-bounded` | any (fault) | Always | P1 | LIVE | bounded-memory; io::ErrorKind labels not raw string (32dd4cf6 off-main; gRPC still raw). | +| `sink-receives-bytes` | any | Sometimes | P1 | guard | non-vacuity floor; sink/main.rs:82 already instrumented. | + +**Overlap:** `discarded-blocks-counted` (this cluster, observability) mirrors +`capture-histogram-drops-counted` (§3) — same "loss must be counted" principle +across subsystems. `throttle-capacity-no-zerodelivery-livelock`, +`throttle-divide-no-silent-underdelivery`, and `discarded-blocks-counted` form a +tight triangle around one config class (block > per-worker capacity): fail-fast, +don't-silently-drop, and count-if-you-do respectively. + +--- + +## 6. observer + +**Dominance:** the panic trio are `Unreachable` members of the §1 +`no-panic-anywhere` umbrella but cluster here by fault class = *target lifecycle +race*. `observer-transient-read-not-fatal` is the liveness umbrella (a component +read error must not `?`-propagate to a dead run). `observer-pid-identity- +fingerprint` is the correctness (right-process) invariant that dominates the +PID-reuse pair semantically: `observer-pid-reuse-no-panic` says "don't crash on +reuse", `observer-pid-identity-fingerprint` says "don't silently attach to the +impostor" — the latter is the stronger claim. + +| slug | Path | Type | P | main? | Fault class | +|---|---|---|---|---|---| +| `observer-pid-reuse-no-panic` | any | Unreachable | P0 | LIVE | `assert!(cur_pid==pid)` stat.rs:82 (6aa1b1ba off-main). | +| `observer-process-vanish-no-panic` | any | Unreachable | P0 | LIVE | `panic!` process_descendents.rs:13 (7e1d2968 off-main). | +| `observer-cpu-max-parse-no-panic` | any | Unreachable | P0 | LIVE | cpu.max parse bounds/zero-period (6aa1b1ba off-main). | +| `observer-transient-read-not-fatal` | any (fault) | Always | P1 | LIVE | liveness umbrella; best-effort component reads (30b86a71 off-main). | +| `observer-pid-identity-fingerprint` | any | Always | P2 | LIVE | correctness; start_time fingerprint vs PID-reuse impostor. Dominates the reuse pair. | +| `get-available-memory-cgroup-chain` | any | Always | P2 | fix-on-main | 1085887c; walks cgroup v2 ancestors. guards regression. | + +**Overlap:** `observer-target-pid-recv-no-panic` is catalogued under §1 (its +subsystem field is `no-panic`) but is functionally an observer property — same +target-lifecycle-race fault class. When building the `target-lifecycle-races` +scenario, drive all four observer panic sites plus the identity fingerprint from +one setup (target exits before PID, vanishes mid-listing, PID recycled). + +--- + +## 7. blackhole/config + +**Dominance:** no single umbrella; two overlapping invariant families. +- **never-backpressure-the-target:** `blackhole-never-backpressures-target` is + the umbrella; `datadog-blackhole-accept-resilient` is a specific failure mode + (accept-loop wedge) that violates it, and `sqs-receive-message-bounded` is the + amplification/OOM sibling. +- **restart / crash resilience:** `unix-datagram-blackhole-removes-stale-socket` + (abrupt-restart) and `recorded-traffic-crash-consistency` (abrupt). +- **validate-early:** `config-numeric-fields-validated` (config load). + +| slug | Path | Type | P | main? | Notes | +|---|---|---|---|---|---| +| `blackhole-never-backpressures-target` | any (load) | Always | P1 | LIVE | blocking `send().await` per point before response, datadog.rs:296-299. Umbrella. | +| `datadog-blackhole-accept-resilient` | any (fault) | Always | P1 | LIVE | fallible select arm wedges on accept Err, datadog.rs:207-212. Violates umbrella above. | +| `sqs-receive-message-bounded` | any | Always | P2 | LIVE | uncapped max_number_of_messages, sqs.rs:257-267. Amplification/OOM. | +| `unix-datagram-blackhole-removes-stale-socket` | abrupt (restart) | Always | P1 | LIVE | lazy remove_file future never awaited, unix_datagram.rs:95. Open Q: socket on persisted volume? | +| `recorded-traffic-crash-consistency` | abrupt | Always | P2 | LIVE(lead) | SMPTNG-390 / #1911 / #1895; zstd-framed truncation must be a valid prefix. | +| `config-numeric-fields-validated` | any (startup) | Always | P2 | LIVE | compression_level 1-22, sample_period>0, config.rs:186-198,106-107. | + +**Overlap:** `tcp-rr-listener-no-panic` (§1) is a blackhole-startup property by +location but a no-panic property by class. `config-numeric-fields-validated`, +`dogstatsd-tag-length-validated` (§4), and `divide-by-zero-startup-error` (§5) +are all instances of the same **validate-early** ADR principle across three +subsystems — reject at load, not at runtime. + +--- + +## Cross-subsystem dominance summary + +Four umbrellas dominate the catalog; most other properties are their members or +their regression witnesses: + +1. `no-panic-anywhere` (Unreachable, hook-observed) — dominates **all** panic-site + properties across no-panic, observer, capture (`capture-write-failure-not-abort`). +2. `shutdown-completes-bounded` (Always, liveness) — dominates every per-loop + connect/wait/finalize hang in lifecycle/shutdown. +3. `payload-determinism-byte-identical` (Always) — dominates + `no-wall-clock-in-payloads`; `block-cache-construction-terminates` dominates + `trace-agent-v04-block-terminates`. +4. Rate-fidelity + no-livelock pair (generators/throttle), floored by the + `sink-receives-bytes` non-vacuity Sometimes and the + `lading-completes-and-exits-cleanly` graceful non-vacuity Sometimes. + +Observability siblings (`discarded-blocks-counted`, +`capture-histogram-drops-counted`, `capture-drift-no-silent-gap`) and +validate-early siblings (`config-numeric-fields-validated`, +`dogstatsd-tag-length-validated`, `divide-by-zero-startup-error`, +`generator-addr-uri-validation-no-panic`, `block-cache-zero-max-no-panic`) each +encode one ADR principle recurring across subsystems — instrument them together. diff --git a/test/antithesis/scratchbook/sut-analysis.md b/test/antithesis/scratchbook/sut-analysis.md new file mode 100644 index 000000000..117c4ea2b --- /dev/null +++ b/test/antithesis/scratchbook/sut-analysis.md @@ -0,0 +1,417 @@ +--- +sut_path: /home/ssm-user/src/lading +commit: 51148899 +updated: 2026-07-24T21:28:16Z +external_references: + - name: the deployment (production/local runner + orchestrator) + why: source of truth for the real shutdown/deploy model (how lading is launched, stopped, and torn down; what "correct shutdown" means operationally; the watchdog/cancel/kill paths that determine capture completeness) + - name: Jira / Confluence (datadoghq.atlassian.net, SMPTNG/SMP projects) + why: existing bug tickets and design docs corroborating termination/telemetry-loss classes (SMPTNG-725, -719, -697, -694, -390, -762, -767) and the determinism/DST testing philosophy + - name: the whole lading repo (this checkout) + why: primary code source of truth; all line references below are into this tree and must be re-verified against source, not invented +--- + +# lading SUT analysis (shutdown-safety emphasis) + +lading is a deterministic load generator arranged as **generator -> target -> blackhole**, plus a +passive **observer** and a **capture** subsystem. It is engineered around four ADRs: determinism +(same seed => byte-identical load; no wall-clock in payloads; ordered maps not `HashMap`), +no-panic (`Result`, no `unwrap`/`expect` on reachable paths), pre-computation (payloads prebuilt +into a block cache), and performance-first. `panic = "abort"` is set for both release and dev +profiles (`Cargo.toml:115,120`), so **any reachable panic is a whole-process `SIGABRT`** and the +Antithesis panic hook (`antithesis_hooks.rs:27`, `unreachable!("lading panicked")`) reports it +before the abort. + +This document maps the architecture, state management, concurrency model, and failure-prone areas +across ALL subsystems, prioritising shutdown/termination safety (P0). It is synthesised from the +property catalog + discovery digest and spot-verified against current `main`. Line numbers are +leads; re-verify against source. + +--- + +## 1. Architecture and process topology + +``` + lading process (tokio multi-thread runtime) + ┌───────────────────────────────────────────────────────────────────────┐ + │ main() bin/lading.rs │ + │ ├─ antithesis_hooks::init() :741 (SDK init + panic hook) │ + │ ├─ reachable! "lading completed bootstrap" :792 │ + │ ├─ build config, construct block caches (lading_payload) │ + │ ├─ tokio runtime; inner_main: │ + │ │ ├─ lading_signal broadcast (shutdown) -- register/broadcast │ + │ │ ├─ Target (spawn Binary OR observe Container via docker.sock) │ + │ │ ├─ Observer (per-pid procfs/cgroup sampler) │ + │ │ ├─ Inspector (optional child process) │ + │ │ ├─ Generators[] (Vec) -- tcp/udp/unix_stream/unix_datagram/ │ + │ │ │ grpc/http/splunk_hec/file_gen/... │ + │ │ ├─ Blackholes[] (Vec) -- tcp/udp/unix/http/datadog/sqs/otlp/... │ + │ │ ├─ Capture manager (jsonl | parquet | multi) │ + │ │ └─ experiment_sequence: warmup timer -> experiment timer -> exit │ + │ └─ runtime.shutdown_timeout(max_shutdown_delay=30s) :813 │ + └───────────────────────────────────────────────────────────────────────┘ +``` + +Key config facts: `config.generator` and the blackhole list are **`Vec`s** (`config.rs:101`) — +multiple generators/blackholes of the same kind are allowed (only duplicate IDs are rejected). +This is the root of several latent defects (per-generator process-global statics). + +**Deployment model (the deployment; verify against its source).** Production launches lading as a +container in Container/observer mode (`--target-container target`, docker.sock bind-mounted) — lading +**observes** a sibling target container and does **not** spawn/reap it. lading "owns the clock": +it self-terminates on its own experiment timer; the orchestrator never signals it. Teardown is +`docker rm --force` (immediate SIGKILL). A watchdog of `ceil((warmup + samples + 30) * 1.2)s` +force-ends overruns. **Correct shutdown operationally = lading self-exits 0 within +`max_shutdown_delay` (30s) of experiment end with a footer-complete parquet at +`/captures/captures.parquet`.** Any hard kill before self-exit => footer-less, unreadable parquet => +total capture loss for that replicate (this is why bounded shutdown is P0). Consequences: + - Binary-target defects (untimed `target_child.wait()`, `kill_on_drop` orphaning) are **masked on + the production observer path** but LIVE for Binary-target mode / local dev / bare-host. + - The SIGTERM/ctrl_c graceful path is effectively dead code in this deployment, but a plain + `docker stop`/orchestrator-stop on other paths would hit the missing-SIGTERM-handler defect. + - Jira corroboration: SMPTNG-725 ("RJO alive but not really" — hang-in-spin after "lading + shutdown"), SMPTNG-719/697 (ungraceful-termination telemetry loss), SMPTNG-694 (captures now + zstd-compressed => a truncated stream may be undecodable past the last flushed frame — revisit + the "valid jsonl prefix" assumption against the on-disk format). + +--- + +## 2. State management + +- **Config** (`config.rs`) is the single validate-early gate. Gaps live here: numeric fields + (`compression_level` 1-22 `:186-198`; `sample_period_milliseconds > 0` `:106-107`) are not + range-checked and only fail at runtime. +- **Block cache** (`lading_payload/block.rs`) is the precomputed, immutable payload state. Built + once at startup from `(seed, config)`; a `Cache::Fixed` indexes `blocks[idx % len]`. An empty + cache would panic on modulo, but `InsufficientBlockSizes` (`block.rs:676-680`) prevents an empty + Fixed cache from being constructed. Determinism holds by construction: `BTreeMap`/`BTreeSet` + ordering, `FxHashMap` used index-agnostically, rng-derived (not wall-clock) timestamps. +- **Capture accumulator** (`lading_capture/accumulator.rs`) is a per-series ring of the last + `INTERVALS = 60` ticks. `flush()` emits only tick `current_tick - 60`; the most recent <=60 ticks + live only in memory and reach disk exclusively via `drain_and_write` at graceful shutdown + (`state_machine.rs:326-346`). `fetch_index == tick` and `time_ms = start_ms + tick*1000`, giving a + global bijection and strict per-series monotonicity — this is why a killed jsonl file is a valid + parseable prefix. +- **Throttle valve state** (`lading_throttle`) holds per-worker capacity; `divide(N)` splits + capacity across parallel workers (see §5). + +--- + +## 3. Concurrency model (tokio tasks + lading_signal) + +- **Runtime.** A multi-thread tokio runtime. Every subsystem is a spawned task or a `JoinSet`. +- **Shutdown fan-out = `lading_signal`.** A broadcast primitive: workers register a + `shutdown_watcher`, the driver calls `signal_and_wait` to broadcast and await acknowledgement. + Each worker is expected to poll its watcher inside its main `tokio::select!`. **The central + fragility class:** workers that block *before* reaching their `select!` (connect/retry loops, + partial-write loops, `child.wait()`, capture finalize) never observe the broadcast and cannot be + shut down within the bound. There is a `Lagged` tripwire on the broadcast (`lib.rs:235/268`) worth + watching. +- **Generators** run as a `JoinSet`; `Server::spin()` blocks on `join_next()`. One un-shuttable + worker hangs the whole spin, bounded only by `runtime.shutdown_timeout(30s)` — *if that backstop + is even reached* (see §4 capture-finalize hang, which precedes it). +- **Target/Observer/Inspector** are tasks holding tokio `Child` handles with `kill_on_drop(true)`. + `kill_on_drop` fires only on normal `Drop`, never on untrapped-signal death. +- **Capture manager** runs on `spawn_blocking` (`.block_on(start())`), receiving samples over a + bounded `mpsc::channel(10_000)` (`manager.rs:302`). Histogram record uses `try_send` + + warn-and-drop (`manager.rs:121-133`) — samples silently lost when full. +- **Antithesis coupling.** Antithesis controls scheduling, timing, and faults, and injects + `node_termination` = **SIGKILL of the whole container** (untrappable; only persisted named-volume + artifacts survive). It can also perturb the clock (relevant to throttle burst envelope and any + wall-clock leak). Only two SDK sites live in `lading/src` (panic hook + a bootstrap `reachable!`); + ALL domain invariants live in `test/antithesis/` (sink byte oracle, capture-consistency checker). + +--- + +## 4. Failure-prone areas — SHUTDOWN / TERMINATION SAFETY (P0) + +This is the immediate priority. The umbrella invariant: **once shutdown is signaled (experiment +timer or SIGTERM), lading exits within `max_shutdown_delay` (30s) and never hangs in `Server::spin` +or capture-finalize.** Violation => watchdog SIGKILL => unreadable parquet => total capture loss. + +### 4.1 No SIGTERM handler (P0) +`main` select traps only `signal::ctrl_c()` (SIGINT) at `lading.rs:658`; there is **no +`tokio::signal::unix` `SignalKind::terminate` arm**. A plain SIGTERM => default disposition => exit +143 with NO graceful path (no broadcast, no capture finalize, no child reap). *Fix:* add a +`SignalKind::terminate` arm alongside ctrl_c that triggers `shutdown_broadcast`. Corroborated by +SMPTNG-719/697. + +### 4.2 Pre-`select` connect/retry loops ignore shutdown (P0/P1) +Each of these loops does `connect -> on error sleep -> continue` and only reaches the shutdown-aware +`select!` after a connection exists. With an unreachable target they spin forever: + - **tcp** connect branch `tcp.rs:230-251` (sleep 1s) — P0. + - **unix_datagram** connect loop `unix_datagram.rs:246-264` (sleep 1s) — P0. + - **grpc** connect loop `grpc.rs:287-299` (sleep 100ms) — P1. + - **unix_stream** connect loop `unix_stream.rs:248-268` — P0. +*Fix:* wrap the connect attempt in `select!` with `&mut shutdown_wait`, or check shutdown before the +sleep. *Oracle:* unreachable-target scenario + external exit timer. + +### 4.3 unix_stream partial-write busy-spin (P0) +Inner `while blk_offset < blk_max` loop (`unix_stream.rs:281-318`) has no shutdown branch and +busy-spins on `yield_now()` when `try_write` returns `WouldBlock` (comment admits "if the read side +has hung up we will never know and will keep attempting to write"). A slow/stalled receiver => +un-shuttable + 100% CPU. Related: non-`BrokenPipe`/non-`WouldBlock` write errors (e.g. +`ConnectionReset`) neither advance nor reconnect (`:304-315`) — busy-loop spamming `request_failure`; +`packets_sent` counted per partial write. *Fix:* add a shutdown branch and bounded/awaited readiness; +handle non-BrokenPipe errors (advance/break/reconnect). + +### 4.4 Container-target discovery loop unbounded (P0 — production path) +`watch_container` (`target.rs:212-244`) polls for the named container with only a found-break + +`sleep(1s)`, no shutdown arm, no max-attempts. A misnamed/never-started target => `target_running` +never signals => `experiment_sequence` blocks at `target_running_watcher.recv()` +(`lading.rs:632-641`) => experiment timer never starts => lading never self-terminates. **This is +the launched production mode**, so high impact. *Fix:* add a `shutdown.recv()` arm and/or +max-attempts timeout. + +### 4.5 Untimed `child.wait()` and ~0 target grace (P0/P1 — Binary mode) + - `target_child.wait()` (`target.rs:432`) and inspector wait (`inspector.rs:176`) are bare + `.await` — a SIGTERM-ignoring child hangs lading, relying solely on external JoinSet-abort / + `shutdown_timeout`. *Fix:* wrap in a timeout that escalates to SIGKILL. + - Target grace period is effectively ~0: `inner_main` returns after capture flush without joining + `tsrv_joinset`; dropping it aborts the target task mid-`wait()`, and `kill_on_drop` SIGKILLs the + child within ms of the SIGTERM (`lading.rs:690-717`). The "give the child a chance to clean up" + comment is not honored. *Confirm intended grace semantics before fixing.* + - Orphaned children on signal death: `kill_on_drop` can't fire on untrapped-signal death; graceful + path signals only the direct child pid, not the process group (`target.rs:430-431`, + `inspector.rs:175-176`) => grandchildren leak. Moot under whole-container SIGKILL; live in + shared-PID-namespace / bare-host. + +### 4.6 Capture-finalize await unbounded (P1) +`let _ = handle.await;` (`lading.rs:713-715`) has no timeout and runs **before** +`runtime.shutdown_timeout` (`:813`). A stalled parquet footer write (slow/full disk) hangs lading +with no backstop, because the 30s timeout is only reached after this await returns. *Fix:* wrap in a +timeout so `shutdown_timeout` remains the backstop. + +### 4.7 Non-vacuity anchors + - `lading-completes-and-exits-cleanly` (Sometimes/P0): at least one timeline reaches clean self-exit + 0 with a footer-complete capture — guards against a regression that makes EVERY timeline hang/abort. + - Stale comment at `lading.rs:785-787` ("divide by two") is wrong — no divide-by-two exists; the + full `max_shutdown_delay` goes only to `shutdown_timeout`. + +--- + +## 5. Failure-prone areas — Generators & throttle (rate fidelity, no-panic, livelock) + +### 5.1 throttle `divide()` breaks aggregate rate (P0) +`divide(N)` shrinks per-worker capacity to `R/N` but NOT the block a worker draws. A block sized +`R/N < block <= R` is accepted at `parallel_connections=1` but rejected with `Capacity` by every +worker at N>1 => **silent zero delivery** (only a `debug!` log). *Fix:* shrink block sizing +consistently, or validate `maximum_block_size <= bytes_per_second/parallel_connections` upfront. +Pure-proptest-provable against the real Valve (demonstrated on backup branch `0868e39c`, live on main). + +### 5.2 Linear throttle ramp compression (P0) +`divide` splits capacities but passes `rate_of_change` unchanged (`lib.rs` Linear arm), so N workers +ramp at aggregate `N*rate` — the ramp reaches max in `1/N` the intended time. *Fix:* divide +`rate_of_change` by the divisor. Provable as a pure proptest measuring aggregate slope. + +### 5.3 Capacity-livelock / zero-delivery busy loop (P0) +When block byte size exceeds per-worker capacity, `wait_for` returns `Capacity` immediately (no +wait); tcp/udp/unix discard + loop => hot 100% CPU delivering ~zero bytes (0.31.2 busy-discard +class). *Fix:* validate `maximum_block_size <= bytes_per_second/parallel_connections` post-divide at +construction. + +### 5.4 gRPC ignores the throttle result (P0) +`grpc.rs:307-318` does `let _ = result;` and sends regardless => neither honors discards nor the +configured rate (over-delivers). Also serializes requests (RTT-bound). *Fix:* honor the Result +(discard + count), matching tcp/udp. + +### 5.5 unix generators don't `.divide()` (P1) +`unix_stream.rs:161-163` and `unix_datagram.rs:186-188` give each worker a full-rate throttle => +aggregate `N * bytes_per_second` (vs tcp/udp aggregate ~=bps). Cross-generator inconsistency for the +same config key. *Confirm intended per-connection semantics (unix_stream doc says "per connection").* + +### 5.6 Integer-division truncation / DivisionByZero (P2) +`bps` not divisible by `N` drops up to `N-1` bytes/interval; `bps < N` returns `DivisionByZero` and +fails to start with a surprising error. *Fix:* distribute remainder; surface a clear validation error. + +### 5.7 Observability of under-delivery (P1) +tcp/udp/grpc must count `blocks_discarded` (currently `debug!` only) and bound error-label +cardinality to `io::ErrorKind` (raw `err.to_string()` embeds addresses => unbounded capture-key +growth, ADR-005 OOM class; gRPC tonic error still raw). Both landed on backup branches, live on main. + +### 5.8 Generator no-panic sites (P1) + - Two `Http` (or two `SplunkHec`) generators panic on the second `new()` via a process-global + `static CONNECTION_SEMAPHORE: OnceCell` double-`set` (`http.rs:37,187-189`; + `splunk_hec.rs:51,243-245`). Also shares one process-wide limit. *Fix:* per-instance + `Arc`; make hot-path `.expect("semaphore closed")` stop the worker gracefully. + - Splunk-HEC response parse: `serde_json::from_slice::(...).expect(...)` + (`splunk_hec.rs:371-374`) panics a **detached, untracked** task on any non-JSON body. + - Malformed addr/uri: `.expect` at construction (`tcp.rs:154-159`, `udp.rs:164-169`, + `grpc.rs:226-247`) instead of a `Result` error. + - http backpressure: `CONNECTION_SEMAPHORE.acquire().await` sits outside the `select!` and there is + no per-request timeout; shutdown does `acquire_many(all)` => a stalled target hangs shutdown + (`http.rs:229-297`). Splunk-HEC wraps requests in a 1s timeout and is more resilient. + +### 5.9 sink non-vacuity (P1) +`sink/main.rs:82` `sometimes! (total>0)` guards against a whole config class delivering nothing +(divide stall, capacity livelock, throttle bypass). + +--- + +## 6. Failure-prone areas — Payload / determinism (no-panic, termination, reproducibility) + +- **Block-cache construction can hang forever (P0):** `construct_block_cache_inner` + (`block.rs:625-673`) has no time/iteration cap. If `max_block_size < serializer minimum viable + block`, every attempt is `EmptyBlock`, `bytes_remaining` never decreases, `min_block_size` stays + `< max_block_size`, so neither exit fires. *Fix:* cap consecutive rejections + time bound => return + `InsufficientBlockSizes`; reject `max_block_size` below a serializer floor. trace_agent v04 variant + fixed on backup `456e85a3` (31h hang -> 0.75s); general case live on main. +- **`maximum_block_size == 0` => `random_range(0..0)` panic (P1):** `block.rs:628`; the guard + `:214-220` misses 0. Reachable via `grpc.rs:205-223` forwarding unvalidated `as_u128()`. *Fix:* + lower-bound check. +- **Determinism holds (P1 regression guard):** byte output is a pure function of `(seed, config)`; + `BTreeMap`/`BTreeSet`, index-agnostic `FxHashMap`, rng-derived timestamps + (`trace_agent/v04.rs:326,384`); `Instant` feeds only progress logging. Property guards against a + regression introducing wall-clock/entropy (analog to SMPTNG-762 CPU-jitter entropy / SMPTNG-767 + non-monotonic wall clock). +- **No wall-clock in payloads (P2):** trace_agent/otel/templated_json timestamps are seed-derived and + monotone; must survive a backward clock step under Antithesis clock control. +- **Fuzz `Arbitrary for Block` (P2):** `total_bytes == 0` panics on `NonZeroU32::new().expect` + (`block.rs:119-127`); only under the `arbitrary` feature. *Fix:* return + `Err(arbitrary::Error::IncorrectFormat)`. +- **dogstatsd tag_length (P2, merged e98e3052):** `tag_length.end() <= MIN_TAG_LENGTH` now rejected + upfront through `DogStatsD::new`; property guards regression. + +--- + +## 7. Failure-prone areas — Capture (crash-consistency, durability, no-abort) + +- **Capture write error aborts the whole run (P0):** `start()` propagates via `next(event)?` + (`manager.rs:409`) and the caller does `.block_on(start()).expect(...)` (`lading.rs:476/501/526`) + under `panic=abort` => a transient flush IO error (disk full/EIO) SIGABRTs the run, skips + `format.close()`, leaves a footer-less unreadable parquet. *Fix:* propagate to + `Error::CaptureManager` so BufWriters flush on Drop; plumb mid-run errors to graceful shutdown. + (Startup fix on backup `b7624af2`; mid-run still a follow-up.) +- **Parquet footer only at graceful close (P0):** `state_machine.rs:249-259` -> `parquet.rs:307-324`. + With no SIGTERM handler, SIGTERM/SIGKILL skip `close()` => unreadable by design. The graceful-exit + invariant (`parquet-footer-on-graceful-exit`) depends on §4.1 + capture-write-not-abort. +- **jsonl valid prefix (P1, holds by construction):** the harness checker + (`anytime_capture_consistent.rs:44,49`) asserts `torn_before_final==0` and monotonic + fetch_index/time on any truncated prefix; `MIN_RECORDS=10` non-vacuity floor. NOTE SMPTNG-694: + captures are now zstd-compressed — re-verify a truncated stream is still a valid prefix. +- **60s maturity loss + no fsync (P2):** the last <=60 ticks live only in memory until graceful + drain; no `sync_all`/`sync_data` anywhere in `lading_capture` (`jsonl.rs:53` BufWriter flush only) + => flushed-but-unsynced lines can vanish on whole-VM kill if page cache isn't preserved to the + named volume. Open question: does node_termination preserve page cache? +- **multi close order forfeits parquet (P1):** `multi.rs:69` `jsonl.close()?` runs before `:71` + `parquet.close()?`; a trivial jsonl error skips the important parquet footer. *Fix:* finalize + parquet first / best-effort both. +- **Drift correction silent gaps (P2):** `state_machine.rs:219-232` advances multiple ticks without + interleaved flush; `advance_tick` overwrites unflushed ring slots (`accumulator.rs:466-481`) => + invisible fetch_index gaps under >60s starvation (passes the strict-monotonic validator). +- **Histogram drops silent (P1):** full channel => `try_send` warn-and-drop (`manager.rs:121-133`); + add a bounded-label `capture_histogram_samples_dropped` counter in the registry (backup `39d8ae56`). +- **Capture no-panic sites (P0 via abort):** `accumulator.rs:527,231`, `manager.rs:236`, repeated + `state_machine.rs:315/342/395/411/428` `.expect("format must be present...")` — each aborts the run. +- **Split-mode merge (P2, deployment lead):** the oblivious merge tolerates empty but not a + truncated/footerless parquet — a corrupt side errors the whole replicate. Confirm a clean side's + captures aren't needlessly lost when only one side overruns. + +--- + +## 8. Failure-prone areas — Observer (no-panic under target races) + +`panic=abort` makes each of these a hard SIGABRT (all live on main; fixes only on backup): +- **PID reuse assert (P0):** `stat.rs:82` `assert!(cur_pid == pid)` fires on a recycled/mismatched + PID. *Fix:* skip-and-continue (backup `6aa1b1ba`). +- **Process vanish mid-listing (P0):** `process_descendents.rs:13` `panic!` when `Process::new` + fails. *Fix:* yield empty iterator (backup `7e1d2968`). +- **Malformed cpu.max (P0):** unchecked index / zero-period divide in `parse_allowed_cores`. *Fix:* + bounds-check + guard (backup `6aa1b1ba`). +- **Target PID never arrives (P0):** `observer.rs:114-120` `.expect("catastrophic failure")` on + `recv()` `Err(Closed)` when a Binary target fails to spawn / instant-exits (`target.rs:395-396`). + Only partly masked as a join-error log. *Fix:* return an error. +- **Transient read kills the run (P1):** a single procfs/cgroup/wss read error `?`-propagates and + terminates the experiment. *Fix:* best-effort log+skip (backup `30b86a71`). +- **PID-reuse impostor metrics (P2):** `kill(pid,0)` check then `AsyncPidFd::from_pid` is a TOCTOU; + a recycled PID silently yields plausible-but-wrong metrics. *Fix:* capture + validate a + proc start_time identity fingerprint (`target.rs:302-332`). +- **cgroup memory limit (P2, merged 1085887c):** `get_available_memory` now walks the v2 ancestor + chain returning the tightest `memory.max`; property guards regression via the injectable reader. + +--- + +## 9. Failure-prone areas — Blackholes & config (never-backpressure, no-panic, amplification) + +- **unix_datagram stale-socket bind fail (P1):** `unix_datagram.rs:95` + `let _res = tokio::fs::remove_file(...).map_err(...)` builds a lazy `MapErr` future that is + **never awaited** — the stale socket is never removed, so bind fails `EADDRINUSE` after a + hard-kill restart. *Fix:* `.await` the remove. (Impact depends on whether the socket path is on a + persisted volume.) +- **datadog accept-loop wedge (P1):** `datadog.rs:207-212` uses a fallible `Ok((stream,_)) = + listener.accept()` select arm with no `else` — on an accept `Err` (EMFILE/ECONNABORTED under fault) + the branch is disabled and select blocks on shutdown forever; the loop stops accepting while + appearing alive => backpressures the target. *Fix:* `match` + continue, like `common.rs:89-96`. +- **datadog capture-channel backpressure (P1):** `handle_v2_protobuf` awaits a bounded + `send().await` per metric point (`:398,412,416`) before building the response (`:296-299`) — a slow + capture drain stalls the target. *Fix:* `try_send` + count-on-drop. +- **tcp_rr bind panic (P1):** `create_listener` uses `expect`/`panic!` (`tcp_rr.rs:345-346`), and the + `threads>1` thread-0 prebuild runs in the async `run()` (`:179`) — a pre-bound data port SIGABRTs + the main task. *Fix:* return `Error::Bind`. (Confirm threads>1 is used.) +- **SQS amplification (P2):** `max_number_of_messages` (u32, target-controlled) drives an unbounded + `0..num_messages` allocation loop (`sqs.rs:257-267,362-370`); real SQS caps at 10. *Fix:* clamp. +- **Config validation gaps (P2):** `compression_level` 1-22 and `sample_period_milliseconds > 0` + unchecked at load (`config.rs:186-198,106-107`). +- **Accept-error inconsistency (systemic):** `common.rs` (http/sqs/splunk_hec/otlp) log+continue + (resilient); tcp/udp/unix_stream propagate+die (target loses sink); datadog wedges (worst). Worth a + single policy decision. +- **Recorded-traffic crash-consistency (P2, lead):** blackhole traffic recorder (SMPTNG-390, #1911, + #1895) — verify zstd framing so a SIGKILL-truncated stream is a valid decodable prefix. + +--- + +## 10. Cross-cutting: no-panic umbrella + non-instrumented paths + +- **`no-panic-anywhere` (P0):** the panic hook (`antithesis_hooks.rs:27`) is the oracle for EVERY + panic site above; `panic=abort` makes each hit a hard observable abort. The individual panic-site + properties (§5.8, §6, §8, §9) are the ADR-compliant `Result` conversions needed. +- **No SDK assertion instruments the shutdown path inside `lading/src`** — the P0 shutdown + invariants are currently observable only indirectly (panic hook + external capture-consistency + checker + sink oracle). Adding external exit-timer oracles (shutdown-signal -> exit <= 30s) is the + highest-leverage harness gap. +- Three std `unreachable!` sites (`lading.rs:347`, `splunk_hec/acknowledgements.rs:110`, + `otlp/http.rs:227,258`) are guards, not SDK assertions, but are still subject to the no-panic ADR. + +--- + +## 11. Priority scenario map (what to drive under Antithesis) + +| Scenario | Drives | Priority | +|---|---|---| +| unreachable-target (addr/socket/uri/container never binds) | §4.2, §4.4 connect-loop hangs; exit <= 30s | P0 | +| stalled-receiver (accepts, never reads) | §4.3 unix_stream spin; §5.8 http backpressure | P0 | +| multi-generator config (two Http / two SplunkHec) | §5.8 OnceCell double-set panic | P1 | +| small-and-zero block size | §6 block-cache hang / `random_range` panic | P0 | +| oversized-block-vs-capacity (N>1) | §5.1/§5.3 divide stall + livelock; blocks_discarded>0 | P0 | +| linear-ramp-parallel | §5.2 aggregate slope == rate_of_change | P0 | +| determinism-replay (+perturbed clock) | §6 byte-identical + monotone timestamps | P1 | +| node-termination mid-run (SIGKILL) | §7 jsonl prefix valid; readable parquet consistent | P0 | +| sigterm-stop | §4.1 graceful drain, footer written, no orphans | P0 | +| capture-write-fault (disk full/EIO) | §7 clean fatal exit, footer written, no SIGABRT | P0 | +| target-lifecycle-races (exit-before-PID, vanish, PID reuse) | §8 observer degrades, no impostor metrics | P0 | +| blackhole-restart-stale-socket | §9 unix_datagram bind succeeds | P1 | +| accept-fault-injection (fd exhaustion) | §9 datadog keeps serving | P1 | +| slow-capture-drain | §4.6 finalize bound; §9 datadog response latency bound | P1 | +| malformed-config (addr/uri/level/period/tag_length) | clean startup errors, no panic | P1 | +| split-mode-partial-kill | §7 merge/attribution well-defined | P2 | +| tcp_rr threads>1 bind conflict | §9 Error::Bind not panic | P1 | + +--- + +## 12. Standing open questions (validate against source / the deployment) + +1. Does the deployment ever deliver SIGTERM, or only whole-container SIGKILL? (Determines whether §4.1 + is exercised outside an explicit `kill -TERM`.) +2. Does the parquet writer finalize a readable footer on `flush_seconds:60`, or only at graceful + close? If only at close, every non-graceful stop loses ALL data, not just the last 60s. +3. Are captures now zstd-compressed (SMPTNG-694) such that a truncated stream is undecodable past the + last frame — invalidating the "valid jsonl prefix" assumption? +4. Does `node_termination` preserve OS page cache to the persisted named volume (§7 no-fsync — P2 vs P1)? +5. Are the unix socket paths on a persisted named volume (§9 stale-socket impact)? +6. Is the undivided per-connection throttle intentional for unix_stream (§5.5)? +7. Is the ~0 target grace period intentional, or should `max_shutdown_delay` gate it (§4.5)? +8. Is `tcp_rr` ever configured with `threads>1` in the harness/deployment (§9)? + +> All backup/blt-branch hardening fixes referenced above (divide stall, linear ramp, grpc throttle, +> observer panic trio, logrotate stale-tick, per-generator semaphore, capture abort/histogram drops, +> silent discards, unbounded labels) are **NOT on `main`** — they are live regressions here.