From d4214e0a14c209fe914e1585b23d8fb0e1c0f4d1 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Thu, 23 Jul 2026 22:04:47 +0000 Subject: [PATCH] Introducing lading MVP scenario This commit introduces a 'general' scenario for lading which is intended to be very simple: start lading up with a static config, confirm that it alive. We have demonstrated via Antithesis launch that this rigs up correctly. --- AGENTS.md | 22 +++ Cargo.lock | 25 ++- Cargo.toml | 3 + deny.toml | 5 +- docs/adr/009-antithesis-test-harness.md | 123 +++++++++++++++ docs/adr/README.md | 9 ++ lading/Cargo.toml | 3 + lading/src/antithesis_hooks.rs | 33 ++++ lading/src/bin/lading.rs | 15 ++ lading/src/lib.rs | 2 + test/antithesis/.gitignore | 2 + test/antithesis/CLAUDE.md | 46 ++++++ test/antithesis/bin/launch.sh | 142 ++++++++++++++++++ test/antithesis/scenarios/general/Dockerfile | 112 ++++++++++++++ .../scenarios/general/docker-compose.yaml | 59 ++++++++ test/antithesis/scenarios/general/lading.yaml | 9 ++ test/antithesis/scenarios/general/launch.env | 5 + .../scenarios/general/workload/entrypoint.sh | 12 ++ .../general/workload/setup-complete.sh | 18 +++ test/antithesis/sink/Cargo.toml | 33 ++++ test/antithesis/sink/src/lib.rs | 123 +++++++++++++++ test/antithesis/sink/src/main.rs | 84 +++++++++++ 22 files changed, 883 insertions(+), 2 deletions(-) create mode 100644 docs/adr/009-antithesis-test-harness.md create mode 100644 lading/src/antithesis_hooks.rs create mode 100644 test/antithesis/.gitignore create mode 100644 test/antithesis/CLAUDE.md create mode 100755 test/antithesis/bin/launch.sh create mode 100644 test/antithesis/scenarios/general/Dockerfile create mode 100644 test/antithesis/scenarios/general/docker-compose.yaml create mode 100644 test/antithesis/scenarios/general/lading.yaml create mode 100644 test/antithesis/scenarios/general/launch.env create mode 100755 test/antithesis/scenarios/general/workload/entrypoint.sh create mode 100755 test/antithesis/scenarios/general/workload/setup-complete.sh create mode 100644 test/antithesis/sink/Cargo.toml create mode 100644 test/antithesis/sink/src/lib.rs create mode 100644 test/antithesis/sink/src/main.rs diff --git a/AGENTS.md b/AGENTS.md index 0048cc544..2c65d7e01 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,7 @@ behind these rules, see the [Architecture Decision Records](docs/adr/README.md). | Shape rule | 3+ repetitions = create abstraction | | Dependencies | When in doubt, implement rather than import | | HashMap allowed? | No - use `IndexMap`/`IndexSet` for determinism | +| Recording a design decision | Write an ADR in `docs/adr/` (see [README](docs/adr/README.md)) | ## Task Routing @@ -27,6 +28,26 @@ behind these rules, see the [Architecture Decision Records](docs/adr/README.md). | Adding a dependency | Evaluate: performance, determinism, control | [008](docs/adr/008-dependency-philosophy.md) | | Code style questions | Shape rule, no mod.rs, imports at top | [007](docs/adr/007-code-style-and-abstraction.md) | +## Architecture Decisions + +Significant architectural or design decisions are recorded as Architecture +Decision Records (ADRs) in [`docs/adr/`](docs/adr/), never as ad-hoc design or +spec documents elsewhere. + +When a task involves a design decision worth capturing -- a new subsystem, a +cross-cutting policy, a non-obvious trade-off: + +1. Copy [`docs/adr/000-template.md`](docs/adr/000-template.md) to + `docs/adr/NNN-title-with-dashes.md` using the next sequential number. +2. Fill in every section: Context, Decision, Consequences (positive, negative, + neutral), Alternatives Considered, References. +3. Update the [ADR README](docs/adr/README.md) index and knowledge map. + +Do not write standalone design docs, brainstorming specs, or plan files in other +locations. The ADR is the durable record; a decision that is not in `docs/adr/` +is not recorded. See the [ADR README](docs/adr/README.md) for the full workflow, +including how to supersede an existing decision. + ## Architecture Overview ``` @@ -182,3 +203,4 @@ Override locally: `PROPTEST_CASES=512 ci/test` 8. Document invariants with formal language in tests/proofs 9. Consider measurement interference 10. No mod.rs files, imports at file top only +11. Record design decisions as ADRs in `docs/adr/`, not ad-hoc spec docs diff --git a/Cargo.lock b/Cargo.lock index 5076711bd..2335abe1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,6 +72,15 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" +[[package]] +name = "antithesis-instrumentation" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb6b548668212c6d3a942a4ac7be13bc4fc25715bf661328438f30e3fd342cf0" +dependencies = [ + "cc", +] + [[package]] name = "antithesis_sdk" version = "0.2.9" @@ -1925,6 +1934,7 @@ dependencies = [ name = "lading" version = "0.32.0" dependencies = [ + "antithesis-instrumentation", "anyhow", "arrow-array", "arrow-schema", @@ -1948,6 +1958,7 @@ dependencies = [ "k8s-openapi", "kube", "kube-core", + "lading-antithesis", "lading-capture", "lading-capture-schema", "lading-payload", @@ -3904,6 +3915,18 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "sink" +version = "0.1.0" +dependencies = [ + "anyhow", + "lading-antithesis", + "proptest", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "sketches-ddsketch" version = "0.3.1" @@ -4033,7 +4056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", diff --git a/Cargo.toml b/Cargo.toml index 32859f801..8b5c9a15a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "lading_fuzz", "lading_payload", "lading_throttle", + "test/antithesis/sink", ] [workspace.dependencies] @@ -71,6 +72,8 @@ datadog-protos = { git = "https://github.com/DataDog/saluki", rev = "f47a7ef588c protobuf = { version = "3.7" } enum_dispatch = { version = "0.3" } antithesis_sdk = { git = "https://github.com/antithesishq/antithesis-sdk-rust", rev = "78c9db56771f0f6b01cb5404765c5a96852c159c", default-features = false } # 0.2.8 is pinned to rand 0.8, rev version allows us to select workspace rand +antithesis-instrumentation = { version = "0.1" } +lading-antithesis = { path = "lading_antithesis" } [workspace.lints.clippy] all = "deny" diff --git a/deny.toml b/deny.toml index 3a19e3ec6..3589710f0 100644 --- a/deny.toml +++ b/deny.toml @@ -18,7 +18,10 @@ unused-allowed-license = "allow" [sources] unknown-git = "deny" -allow-git = ["https://github.com/DataDog/saluki"] +allow-git = [ + "https://github.com/DataDog/saluki", + "https://github.com/antithesishq/antithesis-sdk-rust", +] [advisories] version = 2 diff --git a/docs/adr/009-antithesis-test-harness.md b/docs/adr/009-antithesis-test-harness.md new file mode 100644 index 000000000..1e0230cb6 --- /dev/null +++ b/docs/adr/009-antithesis-test-harness.md @@ -0,0 +1,123 @@ +# ADR-009: Antithesis Test Harness Architecture + +## Status + +**Draft** + +## Date + +2026-07-23 + +## Context + +lading's integration correctness is currently checked by sheepdog/ducks, under +`integration/`. sheepdog orchestrates: it builds lading and ducks, spawns ducks, +tells it over a gRPC-on-unix-socket control plane to listen on a random port, +templates that port into a lading config, spawns lading, waits for lading to +exit, pulls counters back from ducks, and asserts "lading pushed load" via +`request_count > 10` or `total_bytes > 100_000`. + +We want to test lading under Antithesis, a deterministic hypervisor that controls +scheduling and randomness, injects faults, and observes properties through +in-process SDK assertions. This requires deciding how lading is driven and where +the "load arrived" claim lives. + +Three constraints shape the decision: + +1. **Antithesis is the orchestrator.** docker-compose brings containers up. This + replaces 'sheepdog'. +2. **lading is its own load driver.** Unlike a typical system under test that + needs a separate workload container to exercise it, driving load is lading's + function. +3. **The observer must be independent of the config under test.** In Antithesis + the lading config is the search space. A `sometimes!` assertion that never + executes is reported as unsatisfied, so if the claim lived in lading's own + blackhole, a healthy config that omits a blackhole would fault the harness. + +## Decision + +Introduce an Antithesis harness under `test/antithesis/`, mirroring +Datadog/saluki's layout, with a "general" MVP scenario of three containers: + +- **sink** (oracle): the standalone `test/antithesis/sink/` crate that binds a + fixed TCP port, counts received bytes, and owns the claim + `sometimes!(total_bytes > 0, "sink received bytes")` through the + `lading_antithesis` SDK facade. Not faulted. SDK-linked but built without + sancov coverage instrumentation. +- **lading** (system under test): the real lading binary, built + `--features antithesis` with sancov coverage instrumentation and + `panic="abort"`, run with `--no-target --experiment-duration-infinite` against + a hard-wired `lading.yaml`, a tcp generator pointed at the sink, with telemetry + exposed via `--prometheus-addr` (lading requires telemetry to be configured). + Faulted. +- **workload** (driver): emits the Antithesis `setup_complete` signal, then + idles. lading drives the load, so this container is the seam where + config-variation test commands will land later. + +In a like manner to saluki we introduce a generic +`test/antithesis/bin/launch.sh`, driven by per-scenario `launch.env`, tags +images by git SHA, builds, renders the compose with concrete tags, and submits +through `snouty launch` with a pinned fault profile. `test/antithesis/CLAUDE.md` +documents the launcher and the Antithesis skills: launch, triage, and +setup/research/workload. + +The SDK facade `lading_antithesis`, feature-gated and a no-op when the feature +is off, is the single path both the sink and lading's bootstrap use to reach +`antithesis_sdk`. lading's instrumentation wiring lives in a feature-gated +`lading/src/antithesis_hooks.rs`, referenced from `lading/src/bin/lading.rs`. It +does SDK init plus a panic-reporting hook. + +Key sub-decisions: + +- **Standalone sink, not lading's blackhole, as the oracle, per constraint 3.** + The observer is separate and always-on, as ducks was. +- **The sink is SDK-linked but uninstrumented.** sancov coverage and the SDK are + independent mechanisms. sancov steers exploration by coverage, and that budget + must target lading, not the oracle. Instrumenting the sink would add its + branches to the coverage surface and pull the search off the system under test. + The oracle must be correct by construction: kept minimal and covered by + ordinary unit and property tests, so it stays a trusted, fixed instrument + rather than something the fuzzer probes. This matches saluki's `tools-builder` + stage. +- **Three containers.** `setup_complete` and future config-variation + live in a dedicated workload container rather than being owned by the faulted + system under test. +- **Config hard-wired for the MVP.** Config variation and test commands are + deferred to the workload seam. + +## Alternatives Considered + +### Assert in lading's own blackhole (self-loop, or lading-generator to lading-blackhole) + +Rejected: the claim's reachability would depend on the config under test, and a +config without a blackhole would fault the harness with an unsatisfied +`sometimes!`. The observer must be independent of the search space. + +### Reuse ducks as the Antithesis target + +Rejected: ducks carries the sheepdog gRPC control plane it no longer needs, and +the goal is to replace ducks, not extend it. The sink fills ducks' structural +role as an independent, always-on receiver, without the orchestration baggage. + +### Instrument the sink with sancov + +Rejected: coverage should guide exploration into the system under test, not the +oracle. Instrumenting the sink dilutes the coverage signal and spends exploration +budget on harness code. The oracle must be bug-free, but that is achieved by +keeping the sink minimal and testing it conventionally, not by fuzzing it. The +sink still links the SDK to emit its assertion. + +### Two containers (fold `setup_complete` into the system under test) + +Rejected: the faulted system under test would own the setup signal, and +config-variation test commands would have no home. A dedicated workload +container keeps those concerns separate. + +## References + +- `lading_antithesis/` - SDK facade over `antithesis_sdk` +- `test/antithesis/` - harness (to be created) +- `integration/sheepdog/`, `integration/ducks/` - the mechanism this replaces +- saluki `test/antithesis/` - pattern source +- ADR-001: Generator-Target-Blackhole Architecture (the sink is an + out-of-process, blackhole-like oracle) diff --git a/docs/adr/README.md b/docs/adr/README.md index 861a5cfff..344c0d6b5 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -22,6 +22,7 @@ helping future contributors understand the reasoning behind the current design. | [006](006-testing-strategy.md) | Testing Strategy (Property Tests + Kani) | Accepted | | [007](007-code-style-and-abstraction.md) | Code Style and Abstraction Rules | Accepted | | [008](008-dependency-philosophy.md) | Dependency Philosophy | Accepted | +| [009](009-antithesis-test-harness.md) | Antithesis Test Harness Architecture | Draft | ## Knowledge Map @@ -50,6 +51,13 @@ How to write code for lading: - **ADR-007**: Code style and abstraction - The "shape rule" and naive style - **ADR-008**: Dependency philosophy - When to implement vs. import +### Testing Infrastructure + +How lading is tested end-to-end: + +- **ADR-006**: Testing strategy - Property tests and Kani proofs +- **ADR-009**: Antithesis test harness - Fault-injected exploration and the load-arrived oracle + ### Decision Dependencies ``` @@ -78,6 +86,7 @@ When working on lading, consult these ADRs based on the task: | Modifying throttle | 003, 005, 006 | | Adding a dependency | 008 | | Writing tests | 006 | +| Antithesis harness work | 009, 001, 006 | | Error handling | 004 | | Code style questions | 007 | | Performance work | 002, 005 | diff --git a/lading/Cargo.toml b/lading/Cargo.toml index a2ecd3e6c..5efec3cb3 100644 --- a/lading/Cargo.toml +++ b/lading/Cargo.toml @@ -21,6 +21,8 @@ lading-capture-schema = { path = "../lading_capture_schema" } lading-payload = { version = "0.1", path = "../lading_payload" } lading-throttle = { version = "0.1", path = "../lading_throttle" } lading-signal = { version = "0.1", path = "../lading_signal" } +lading-antithesis = { workspace = true } +antithesis-instrumentation = { workspace = true, optional = true } anyhow = { workspace = true } arrow-array = { workspace = true } @@ -115,6 +117,7 @@ tonic-prost-build = { workspace = true } [features] default = [] logrotate_fs = ["fuser"] +antithesis = ["lading-antithesis/antithesis", "dep:antithesis-instrumentation"] [lib] doctest = false diff --git a/lading/src/antithesis_hooks.rs b/lading/src/antithesis_hooks.rs new file mode 100644 index 000000000..44d46dc29 --- /dev/null +++ b/lading/src/antithesis_hooks.rs @@ -0,0 +1,33 @@ +//! Antithesis system-under-test bootstrap for the lading binary. +//! +//! Compiled only under the `antithesis` feature. Initializes the Antithesis SDK +//! and installs a panic hook that reports any panic as an Antithesis +//! `unreachable!` before the process aborts. All SDK access goes through the +//! `lading_antithesis` facade, never `antithesis_sdk` directly. + +/// Initialize the Antithesis SDK and install a panic-reporting hook. +/// +/// Call once, as early in startup as possible, before any panic can occur. The +/// installed hook forwards to the previous hook after reporting, so normal panic +/// output is preserved. +pub fn init() { + lading_antithesis::init(); + + let default_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + let location = info + .location() + .map_or_else(String::new, ToString::to_string); + let payload = info.payload(); + let message = payload + .downcast_ref::<&str>() + .map(|s| (*s).to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "".to_string()); + lading_antithesis::unreachable!( + "lading panicked", + { "message": message, "location": location } + ); + default_hook(info); + })); +} diff --git a/lading/src/bin/lading.rs b/lading/src/bin/lading.rs index 6161fac55..9fd987507 100644 --- a/lading/src/bin/lading.rs +++ b/lading/src/bin/lading.rs @@ -33,6 +33,13 @@ use tokio::{ use tracing::{Instrument, debug, error, info, info_span, warn}; use tracing_subscriber::{EnvFilter, util::SubscriberInitExt}; +// Pull in the Antithesis coverage-instrumentation runtime shim only when +// building for antithesis. Load-bearing: the `use ... as _` keeps the shim from +// being dropped as unused, so its sancov symbols stay linked in the binary. +#[cfg(feature = "antithesis")] +// ast-grep-ignore: no-as-imports +use antithesis_instrumentation as _; + #[derive(thiserror::Error, Debug)] enum Error { #[error("Target related error: {0}")] @@ -730,6 +737,10 @@ fn init_tracing(json_output: bool) { } fn main() -> Result<(), Error> { + // Initialize the Antithesis SDK and panic hook before anything can panic. + #[cfg(feature = "antithesis")] + lading::antithesis_hooks::init(); + // Two-parser fallback logic until CliFlatLegacy is removed let (json_output, args) = match CliWithSubcommands::try_parse() { Ok(cli) => match cli.command { @@ -777,6 +788,10 @@ fn main() -> Result<(), Error> { let max_shutdown_delay = Duration::from_secs(args.max_shutdown_delay.into()); let disable_inspector = args.disable_inspector; + // Bootstrap probe: proves the SDK is linked and the instrumentation path is + // wired. No-op when the `antithesis` feature is off. + lading_antithesis::reachable!("lading completed bootstrap"); + let runtime = Builder::new_multi_thread() .enable_io() .enable_time() diff --git a/lading/src/lib.rs b/lading/src/lib.rs index f301d63ee..6e514d01e 100644 --- a/lading/src/lib.rs +++ b/lading/src/lib.rs @@ -11,6 +11,8 @@ use http_body_util::BodyExt; +#[cfg(feature = "antithesis")] +pub mod antithesis_hooks; pub mod blackhole; pub(crate) mod codec; mod common; diff --git a/test/antithesis/.gitignore b/test/antithesis/.gitignore new file mode 100644 index 000000000..eeee66e4a --- /dev/null +++ b/test/antithesis/.gitignore @@ -0,0 +1,2 @@ +# Rendered compose files produced by bin/launch.sh at submit time. +scenarios/*/.launch/ diff --git a/test/antithesis/CLAUDE.md b/test/antithesis/CLAUDE.md new file mode 100644 index 000000000..f3a502fef --- /dev/null +++ b/test/antithesis/CLAUDE.md @@ -0,0 +1,46 @@ +# Antithesis harness + +This directory holds the Antithesis test harness for lading. The design and +rationale live in [ADR-009](../../docs/adr/009-antithesis-test-harness.md). + +## Skills + +- **Launch** a run with the `antithesis-launch` skill or the + `test/antithesis/bin/launch.sh` wrapper. Do not hand-type `snouty launch`. +- **Triage** a completed run with `antithesis-triage` (single run) or + `antithesis-campaign` (launch, wait, triage, and correlate against tickets). +- **Scaffold and grow** the harness with `antithesis-setup`, + `antithesis-research` (build the property catalog), and `antithesis-workload` + (implement assertions and test commands). +- **Validate** scaffolding changes quickly with `snouty validate`. + +## launch.sh + +`test/antithesis/bin/launch.sh ` builds the scenario's images, renders +the compose with concrete git-SHA tags into `.launch/`, and submits through +`snouty launch` with the cpu, clock, and node fault profile the scenario's +`launch.env` sets. Node faults apply only to `SCENARIO_FAULT_NODES`. See the +script header for env overrides (`DURATION`, `SOURCE`, `WEBHOOK`, +`FORCE_DISABLE_ALL_FAULTS`, `DRY_RUN`, ...). + +The `SOURCE` (antithesis.source) defaults to `lading`; confirm the exact string +this tenant expects before relying on tracked runs. + +## setup-complete.sh + +Antithesis runs no test commands until it receives `setup_complete`. Each +scenario's `workload/setup-complete.sh` emits it once the system under test is +ready; the workload container runs it from its entrypoint. + +## Directory layout + +- `sink/` — the TCP byte-counting sink oracle crate (`sink`), a workspace member. + It receives lading's load and owns the "load arrived" assertion. SDK-linked but + built without coverage instrumentation. Built, linted, and tested from the repo + root like any other crate. +- `scenarios/general/` — the MVP scenario: `Dockerfile` (three targets: the + instrumented lading system under test, the sink, the workload), a + `docker-compose.yaml` snouty consumes as `--config`, `launch.env`, the + hard-wired `lading.yaml`, and the `workload/` build inputs. Config variation is + deferred to the workload seam. +- `bin/launch.sh` — the generic launcher shared by every scenario. diff --git a/test/antithesis/bin/launch.sh b/test/antithesis/bin/launch.sh new file mode 100755 index 000000000..4363e4295 --- /dev/null +++ b/test/antithesis/bin/launch.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# Generic Antithesis launcher shared by every lading scenario. +# +# ./launch.sh [extra snouty flags] +# +# names a directory under `test/antithesis/scenarios/` holding a +# docker-compose.yaml and a launch.env. launch.env supplies the per-scenario +# bits; everything else -- image tagging, the fault profile shape, +# build-before-submit -- is common and lives here so every shot is identical and +# comparable and no fault flag is ever fumbled or forgotten. Change a shot's +# faults by editing launch.env's node list, not by passing one-off flags. +# +# launch.env, sourced from the scenario directory, sets: +# SCENARIO_TEST_NAME test name reported to Antithesis +# SCENARIO_DESCRIPTION human description; the git commit is appended +# SCENARIO_FAULT_NODES space-separated SUT container names to node-fault; +# empty means no node termination/hang/throttle at all +# SCENARIO_WEBHOOK optional; tenant webhook, default run_test +# +# Required environment, read by snouty: +# ANTITHESIS_TENANT tenant name +# ANTITHESIS_API_KEY api key, or ANTITHESIS_USERNAME + ANTITHESIS_PASSWORD +# ANTITHESIS_REPOSITORY registry to push the built config + service images to +# +# Optional overrides win over launch.env / defaults: +# DURATION= default 30 +# TEST_NAME= default SCENARIO_TEST_NAME +# DESCRIPTION= default SCENARIO_DESCRIPTION; commit is appended +# FAULT_NODES= default SCENARIO_FAULT_NODES +# WEBHOOK= default SCENARIO_WEBHOOK or run_test +# SOURCE= antithesis.source, default datadog_agent +# SIMULTANEOUS_FAULTS= default false. Same node faults on all +# FAULT_NODES at once, no-op without FAULT_NODES +# FORCE_DISABLE_ALL_FAULTS= default false. Master switch over every +# fault above, runs with none +# DRY_RUN=1 print the exact command and exit without submitting +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ANTITHESIS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +REPO_ROOT="$(cd "$ANTITHESIS_DIR/../.." && pwd)" + +SCENARIO="${1:?usage: launch.sh [extra snouty flags]}" +shift || true +SCENARIO_DIR="$ANTITHESIS_DIR/scenarios/$SCENARIO" +[ -d "$SCENARIO_DIR" ] || { echo "error: no scenario directory $SCENARIO_DIR" >&2; exit 1; } +[ -f "$SCENARIO_DIR/docker-compose.yaml" ] || { echo "error: $SCENARIO_DIR/docker-compose.yaml not found" >&2; exit 1; } +[ -f "$SCENARIO_DIR/launch.env" ] || { echo "error: $SCENARIO_DIR/launch.env not found" >&2; exit 1; } + +# Per-scenario settings. Declared here so a missing one is caught, not silently empty. +SCENARIO_TEST_NAME="" +SCENARIO_DESCRIPTION="" +SCENARIO_FAULT_NODES="" +SCENARIO_WEBHOOK="" +# shellcheck source=/dev/null +. "$SCENARIO_DIR/launch.env" + +# Immutable per-build revision: the short commit, marked -dirty when the working +# tree has uncommitted changes so the tag never claims to be a clean commit it is +# not. Images are tagged by this, never :latest, so a shot can never reuse a stale +# mutable tag and every pushed image traces back to the source it was built from. +GIT_SHA="$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || echo unknown)" +if [[ -n "$(git -C "$REPO_ROOT" status --porcelain 2>/dev/null)" ]]; then + GIT_SHA="${GIT_SHA}-dirty" +fi +export ANTITHESIS_IMAGE_TAG="$GIT_SHA" + +WEBHOOK="${WEBHOOK:-${SCENARIO_WEBHOOK:-run_test}}" +DURATION="${DURATION:-30}" +TEST_NAME="${TEST_NAME:-${SCENARIO_TEST_NAME:?launch.env must set SCENARIO_TEST_NAME}}" +DESCRIPTION="${DESCRIPTION:-$SCENARIO_DESCRIPTION} (commit ${GIT_SHA})" +# May be empty: an empty node list means no node faults for this scenario. +FAULT_NODES="${FAULT_NODES-$SCENARIO_FAULT_NODES}" +SIMULTANEOUS_FAULTS="${SIMULTANEOUS_FAULTS:-false}" +FORCE_DISABLE_ALL_FAULTS="${FORCE_DISABLE_ALL_FAULTS:-false}" + +# antithesis.source groups this project's property history and selects tenant +# customizations. Fixed to `datadog_agent` to match saluki on the shared datadog +# tenant: it selects the tenant customizations and the tracked property-history +# bucket. Without a recognized source snouty runs ephemeral and surfaces no +# findings to triage. +SOURCE="${SOURCE:-datadog_agent}" + +# Pinned fault profile, submitted to the run_test endpoint. cpu_mod and +# clock_jitter are global and symmetric, so every scenario gets them. Network +# faults stay on everywhere and heal before judging. Node termination, hang, and +# throttle apply only to the containers in FAULT_NODES, so a scenario gets none by +# leaving it empty. +FAULTS=( + --param custom.cpu_mod=true + --param custom.clock_jitter=true + --param custom.force_disable_all_faults="$FORCE_DISABLE_ALL_FAULTS" +) +if [[ -n "$FAULT_NODES" ]]; then + FAULTS+=( + --param custom.include_for_node_termination="$FAULT_NODES" + --param custom.include_for_node_hang="$FAULT_NODES" + --param custom.include_for_node_throttle="$FAULT_NODES" + --param custom.simultaneous_faults="$SIMULTANEOUS_FAULTS" + ) +fi + +for v in ANTITHESIS_TENANT ANTITHESIS_REPOSITORY; do + if [[ -z "${!v:-}" ]]; then + echo "error: $v is not set (required to build and submit the run)" >&2 + exit 1 + fi +done + +# Build the images with the concrete tag so snouty reuses them instead of +# rebuilding, and so a shot can never ship a stale mutable tag. lading has no +# converged base image, so this is a plain compose build; layer caching keeps it +# near-instant when nothing changed. +build=(docker compose -f "$SCENARIO_DIR/docker-compose.yaml" build) + +# Launch from a rendered copy so the image tag is concrete. snouty ships the compose +# uninterpolated, so an `${ANTITHESIS_IMAGE_TAG:-latest}` tag reaches the platform as +# the never-pushed `:latest`; `docker compose config` bakes in the tag snouty pushed. +LAUNCH_DIR="$SCENARIO_DIR/.launch" +render=(docker compose -f "$SCENARIO_DIR/docker-compose.yaml" config) + +cmd=(snouty launch + --webhook "$WEBHOOK" + --config "$LAUNCH_DIR" + --test-name "$TEST_NAME" + --description "$DESCRIPTION" + --source "$SOURCE" + --duration "$DURATION" + "${FAULTS[@]}" + "$@") + +printf 'build:'; printf ' %q' "${build[@]}"; printf '\n' +printf 'render:'; printf ' %q' "${render[@]}"; printf ' > %q\n' "$LAUNCH_DIR/docker-compose.yaml" +printf 'launch:'; printf ' %q' "${cmd[@]}"; printf '\n' +if [[ "${DRY_RUN:-0}" == "1" ]]; then + echo "(dry run; not building or submitting)" + exit 0 +fi +"${build[@]}" +mkdir -p "$LAUNCH_DIR" +"${render[@]}" >"$LAUNCH_DIR/docker-compose.yaml" +exec "${cmd[@]}" diff --git a/test/antithesis/scenarios/general/Dockerfile b/test/antithesis/scenarios/general/Dockerfile new file mode 100644 index 000000000..790b5f10e --- /dev/null +++ b/test/antithesis/scenarios/general/Dockerfile @@ -0,0 +1,112 @@ +# syntax=docker/dockerfile:1 +# +# Antithesis harness image for the lading "general" scenario. +# +# Build context is the repository root. Named runtime targets: +# - lading : the instrumented lading binary (the system under test), sancov on +# - sink : the TCP byte-counting oracle, SDK-linked but uninstrumented +# - workload : emits setup_complete then idles (the driver seam) +# +# lading is built native x86_64-unknown-linux-gnu (glibc). + +# --------------------------------------------------------------------------- +# Shared build environment: Rust toolchain (pinned by rust-toolchain.toml) plus +# native build dependencies (protoc for prost/tonic, openssl for reqwest). +# --------------------------------------------------------------------------- +FROM debian:bookworm AS build-base +ENV DEBIAN_FRONTEND=noninteractive \ + NO_COLOR=1 \ + CARGO_TERM_COLOR=never +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + build-essential ca-certificates curl pkg-config libssl-dev cmake protobuf-compiler && \ + rm -rf /var/lib/apt/lists/* +# Install rustup with no default toolchain; the pinned toolchain auto-installs on +# first cargo invocation inside the repo (rust-toolchain.toml). +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --profile minimal --default-toolchain none +ENV PATH="/root/.cargo/bin:${PATH}" + +# --------------------------------------------------------------------------- +# Build the instrumented lading (the system under test). +# +# Coverage instrumentation uses the modern Antithesis Rust flow: the +# `antithesis-instrumentation` crate (referenced once in the binary behind the +# `antithesis` feature) provides the runtime shim, and these RUSTFLAGS enable +# LLVM sancov coverage. `--build-id` is required for symbolization. LTO is forced +# off for predictable sancov instrumentation, and split-debuginfo is forced off +# so the binary keeps embedded DWARF. The release profile already sets +# `panic = "abort"`, so any lading panic becomes SIGABRT, caught as a hard crash. +# +# The sancov RUSTFLAGS are passed via `--config target..rustflags` with an +# explicit `--target`, NOT via the RUSTFLAGS env var. With an explicit target, +# Cargo builds host artifacts (build scripts, proc-macros) for the host and does +# NOT apply the target rustflags to them, so they are not instrumented and link +# cleanly. +# --------------------------------------------------------------------------- +FROM build-base AS lading-builder +ENV CARGO_PROFILE_RELEASE_LTO=off \ + CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO=off +WORKDIR /lading +COPY . /lading +RUN --mount=type=cache,target=/lading/target,id=antithesis-lading-target \ + --mount=type=cache,target=/root/.cargo/registry,id=cargo-registry \ + --mount=type=cache,target=/root/.cargo/git,id=cargo-git \ + cargo build --release --package lading --bin lading --features antithesis \ + --target x86_64-unknown-linux-gnu \ + --config 'target.x86_64-unknown-linux-gnu.rustflags=["-Ccodegen-units=1","-Cpasses=sancov-module","-Cllvm-args=-sanitizer-coverage-level=3","-Cllvm-args=-sanitizer-coverage-trace-pc-guard","-Clink-args=-Wl,--build-id"]' && \ + cp /lading/target/x86_64-unknown-linux-gnu/release/lading /usr/local/bin/lading && \ + echo "Validating Antithesis instrumentation symbols..." && \ + nm /usr/local/bin/lading | grep -q "antithesis_load_libvoidstar" && \ + nm /usr/local/bin/lading | grep -q "sanitizer_cov_trace_pc_guard" && \ + echo "Instrumentation symbols present." + +# --------------------------------------------------------------------------- +# Build the sink oracle, uninstrumented. It is a supporting harness component, +# not the system under test, so it needs no coverage instrumentation. +# --------------------------------------------------------------------------- +FROM build-base AS tools-builder +WORKDIR /tools +COPY . /tools +RUN --mount=type=cache,target=/tools/target,id=antithesis-tools-target \ + --mount=type=cache,target=/root/.cargo/registry,id=cargo-registry \ + --mount=type=cache,target=/root/.cargo/git,id=cargo-git \ + cargo build --release --package sink --bin sink && \ + cp /tools/target/release/sink /usr/local/bin/sink + +# --------------------------------------------------------------------------- +# Runtime: instrumented lading (system under test). +# --------------------------------------------------------------------------- +FROM debian:bookworm-slim AS lading +ENV NO_COLOR=1 +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates libssl3 && \ + rm -rf /var/lib/apt/lists/* +COPY --from=lading-builder /usr/local/bin/lading /usr/local/bin/lading +COPY test/antithesis/scenarios/general/lading.yaml /etc/lading/lading.yaml +# Antithesis scans /symbols in every pushed image and matches by build-id. +RUN mkdir -p /symbols && ln -s /usr/local/bin/lading /symbols/lading +# lading requires telemetry to be configured; --prometheus-addr satisfies it +# without the unbounded disk growth a capture file would incur over an infinite run. +ENTRYPOINT ["/usr/local/bin/lading", "--no-target", "--experiment-duration-infinite", "--prometheus-addr", "0.0.0.0:9102", "--config-path", "/etc/lading/lading.yaml"] + +# --------------------------------------------------------------------------- +# Runtime: sink oracle (SDK-linked, uninstrumented). +# --------------------------------------------------------------------------- +FROM debian:bookworm-slim AS sink +ENV NO_COLOR=1 +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates && \ + rm -rf /var/lib/apt/lists/* +COPY --from=tools-builder /usr/local/bin/sink /usr/local/bin/sink +EXPOSE 9000 +ENTRYPOINT ["/usr/local/bin/sink"] + +# --------------------------------------------------------------------------- +# Runtime: workload driver. Emits setup_complete, then idles. +# --------------------------------------------------------------------------- +FROM debian:bookworm-slim AS workload +ENV NO_COLOR=1 +COPY --chmod=755 test/antithesis/scenarios/general/workload/setup-complete.sh /opt/antithesis/setup-complete.sh +COPY --chmod=755 test/antithesis/scenarios/general/workload/entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/test/antithesis/scenarios/general/docker-compose.yaml b/test/antithesis/scenarios/general/docker-compose.yaml new file mode 100644 index 000000000..b164127c5 --- /dev/null +++ b/test/antithesis/scenarios/general/docker-compose.yaml @@ -0,0 +1,59 @@ +name: lading + +services: + # Oracle: the TCP sink that counts received bytes and owns the "load arrived" + # assertion. Always up, healthchecked, never faulted. + sink: + container_name: sink + hostname: sink + platform: linux/amd64 + init: true + build: + context: ../../../.. + dockerfile: test/antithesis/scenarios/general/Dockerfile + target: sink + image: sink:${ANTITHESIS_IMAGE_TAG:-latest} + environment: + NO_COLOR: "1" + healthcheck: + # /dev/tcp avoids needing curl in the image. + test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/localhost/9000'"] + interval: 2s + timeout: 2s + retries: 30 + + # System under test: the instrumented lading, pushing TCP load at the sink. + lading: + container_name: lading + hostname: lading + platform: linux/amd64 + init: true + build: + context: ../../../.. + dockerfile: test/antithesis/scenarios/general/Dockerfile + target: lading + image: lading:${ANTITHESIS_IMAGE_TAG:-latest} + environment: + NO_COLOR: "1" + RUST_LOG: "info" + depends_on: + sink: + condition: service_healthy + + # Driver: emits setup_complete then idles. lading drives the load, so this is + # the seam where config-variation test commands land later. + workload: + container_name: workload + hostname: workload + platform: linux/amd64 + init: true + build: + context: ../../../.. + dockerfile: test/antithesis/scenarios/general/Dockerfile + target: workload + image: workload:${ANTITHESIS_IMAGE_TAG:-latest} + environment: + NO_COLOR: "1" + depends_on: + sink: + condition: service_healthy diff --git a/test/antithesis/scenarios/general/lading.yaml b/test/antithesis/scenarios/general/lading.yaml new file mode 100644 index 000000000..e02beba40 --- /dev/null +++ b/test/antithesis/scenarios/general/lading.yaml @@ -0,0 +1,9 @@ +generator: + - tcp: + seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, + 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + addr: "sink:9000" + bytes_per_second: "10 MiB" + variant: ascii + maximum_prebuild_cache_size_bytes: "8 MiB" + parallel_connections: 1 diff --git a/test/antithesis/scenarios/general/launch.env b/test/antithesis/scenarios/general/launch.env new file mode 100644 index 000000000..555550868 --- /dev/null +++ b/test/antithesis/scenarios/general/launch.env @@ -0,0 +1,5 @@ +# Per-scenario launch settings for test/antithesis/bin/launch.sh. +SCENARIO_TEST_NAME="lading-general" +SCENARIO_DESCRIPTION="lading general: TCP generator pushing load at the sink oracle" +# The single system under test. sink (oracle) and workload (driver) are not faulted. +SCENARIO_FAULT_NODES="lading" diff --git a/test/antithesis/scenarios/general/workload/entrypoint.sh b/test/antithesis/scenarios/general/workload/entrypoint.sh new file mode 100755 index 000000000..ae2d7a7e4 --- /dev/null +++ b/test/antithesis/scenarios/general/workload/entrypoint.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Workload driver entrypoint. +# +# Gated on sink-healthy (compose depends_on). lading drives the load itself, so +# the workload only emits setup_complete and then idles, awaiting the Antithesis +# test commands that will vary configs in a later iteration. + +/opt/antithesis/setup-complete.sh +echo "setup_complete emitted; workload idle, awaiting Antithesis test commands." +exec tail -f /dev/null diff --git a/test/antithesis/scenarios/general/workload/setup-complete.sh b/test/antithesis/scenarios/general/workload/setup-complete.sh new file mode 100755 index 000000000..63c12f935 --- /dev/null +++ b/test/antithesis/scenarios/general/workload/setup-complete.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Notify Antithesis that setup is complete and test commands may start. Antithesis +# sets ANTITHESIS_OUTPUT_DIR automatically; this writes setup_complete to the +# sdk.jsonl file there. Only run once the whole system is ready for testing. + +OUTPUT_PATH="/tmp/antithesis_sdk.jsonl" +if [[ -n "${ANTITHESIS_OUTPUT_DIR:-}" ]]; then + OUTPUT_PATH="${ANTITHESIS_OUTPUT_DIR}/sdk.jsonl" + echo "Running in Antithesis, emitting setup_complete to ${OUTPUT_PATH}" +elif [[ -n "${ANTITHESIS_SDK_LOCAL_OUTPUT:-}" ]]; then + OUTPUT_PATH="${ANTITHESIS_SDK_LOCAL_OUTPUT}" + echo "Antithesis SDK local output override detected, emitting setup_complete to ${OUTPUT_PATH}" +fi + +mkdir -p "$(dirname "$OUTPUT_PATH")" +echo '{"antithesis_setup":{"status":"complete","details":{"message":"ready to go"}}}' >> "${OUTPUT_PATH}" diff --git a/test/antithesis/sink/Cargo.toml b/test/antithesis/sink/Cargo.toml new file mode 100644 index 000000000..7698e511f --- /dev/null +++ b/test/antithesis/sink/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "sink" +version = "0.1.0" +edition = "2024" +license = "MIT" +publish = false +description = "TCP byte-counting sink oracle for the lading Antithesis harness." + +[[bin]] +name = "sink" +path = "src/main.rs" + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +lading-antithesis = { workspace = true, features = ["antithesis"] } +anyhow = { workspace = true } +tokio = { workspace = true, features = [ + "io-util", + "macros", + "net", + "rt-multi-thread", + "signal", +] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/test/antithesis/sink/src/lib.rs b/test/antithesis/sink/src/lib.rs new file mode 100644 index 000000000..b876f0f69 --- /dev/null +++ b/test/antithesis/sink/src/lib.rs @@ -0,0 +1,123 @@ +//! TCP byte-counting sink oracle for the lading Antithesis harness. +//! +//! The sink is the observer in the Antithesis "general" scenario: it receives +//! the load lading pushes and owns the claim that load arrived. The pure +//! byte-accounting lives here so it can be tested conventionally; the network +//! loop and the SDK assertion live in the binary. + +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Thread-safe accumulator for bytes the sink has received. +#[derive(Debug, Default)] +pub struct Counter { + bytes: AtomicU64, +} + +impl Counter { + /// Create an empty counter. + #[must_use] + pub const fn new() -> Self { + Self { + bytes: AtomicU64::new(0), + } + } + + /// Record `n` received bytes and return the new running total. + /// + /// The total saturates at `u64::MAX` rather than wrapping. + pub fn record(&self, n: u64) -> u64 { + let mut current = self.bytes.load(Ordering::Relaxed); + loop { + let next = current.saturating_add(n); + match self.bytes.compare_exchange_weak( + current, + next, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return next, + Err(actual) => current = actual, + } + } + } + + /// The running total of recorded bytes. + #[must_use] + pub fn total(&self) -> u64 { + self.bytes.load(Ordering::Relaxed) + } +} + +#[cfg(test)] +mod tests { + use super::Counter; + use proptest::prelude::*; + use std::sync::Arc; + use std::thread; + + #[test] + fn new_counter_is_empty() { + let c = Counter::new(); + assert_eq!(c.total(), 0); + } + + #[test] + fn record_accumulates_and_returns_total() { + let c = Counter::new(); + assert_eq!(c.record(10), 10); + assert_eq!(c.record(5), 15); + assert_eq!(c.total(), 15); + } + + #[test] + fn record_zero_leaves_total_unchanged() { + let c = Counter::new(); + assert_eq!(c.record(0), 0); + assert_eq!(c.record(7), 7); + assert_eq!(c.record(0), 7); + } + + #[test] + fn record_saturates_at_max() { + let c = Counter::new(); + assert_eq!(c.record(u64::MAX), u64::MAX); + assert_eq!(c.record(1), u64::MAX); + assert_eq!(c.total(), u64::MAX); + } + + #[test] + fn concurrent_records_sum_exactly() { + let c = Arc::new(Counter::new()); + let threads: u64 = 8; + let per_thread: u64 = 10_000; + let handles: Vec<_> = (0..threads) + .map(|_| { + let c = Arc::clone(&c); + thread::spawn(move || { + for _ in 0..per_thread { + c.record(1); + } + }) + }) + .collect(); + for h in handles { + h.join().expect("worker thread panicked"); + } + assert_eq!(c.total(), threads * per_thread); + } + + proptest! { + #[test] + fn total_equals_saturating_sum( + increments in proptest::collection::vec(any::(), 0..64) + ) { + let c = Counter::new(); + let mut expected: u64 = 0; + for &n in &increments { + expected = expected.saturating_add(n); + prop_assert_eq!(c.record(n), expected); + } + prop_assert_eq!(c.total(), expected); + } + } +} diff --git a/test/antithesis/sink/src/main.rs b/test/antithesis/sink/src/main.rs new file mode 100644 index 000000000..0dc2fb53a --- /dev/null +++ b/test/antithesis/sink/src/main.rs @@ -0,0 +1,84 @@ +//! TCP sink binary for the lading Antithesis harness. +//! +//! Binds a fixed TCP port, counts every byte it receives, and fires the +//! Antithesis `sometimes!` assertion that load arrived. It is the oracle in the +//! "general" scenario: always on, never faulted, and independent of the lading +//! config under test. + +use std::sync::Arc; + +use anyhow::Context; +use sink::Counter; +use tokio::io::AsyncReadExt; +use tokio::net::{TcpListener, TcpStream}; +use tracing::{debug, error, info}; +use tracing_subscriber::EnvFilter; + +/// Address the sink listens on unless `SINK_LISTEN_ADDR` overrides it. +const DEFAULT_LISTEN_ADDR: &str = "0.0.0.0:9000"; + +/// Per-connection read buffer size. +const READ_BUFFER_BYTES: usize = 64 * 1024; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + ) + .init(); + + // Initialize the Antithesis SDK so the sink's assertion reaches the platform. + lading_antithesis::init(); + + let addr = + std::env::var("SINK_LISTEN_ADDR").unwrap_or_else(|_| DEFAULT_LISTEN_ADDR.to_string()); + let listener = TcpListener::bind(&addr) + .await + .with_context(|| format!("failed to bind sink listener on {addr}"))?; + info!("sink listening on {addr}"); + + let counter = Arc::new(Counter::new()); + + loop { + tokio::select! { + accepted = listener.accept() => match accepted { + Ok((stream, peer)) => { + debug!("accepted connection from {peer}"); + let counter = Arc::clone(&counter); + tokio::spawn(async move { + if let Err(e) = handle_connection(stream, &counter).await { + error!("connection from {peer} ended with error: {e}"); + } + }); + } + Err(e) => error!("accept failed: {e}"), + }, + _ = tokio::signal::ctrl_c() => { + info!("shutdown signal received, sink exiting"); + break; + } + } + } + + Ok(()) +} + +/// Read from `stream` until EOF, recording received bytes into `counter` and +/// asserting that load arrived. +async fn handle_connection(mut stream: TcpStream, counter: &Counter) -> anyhow::Result<()> { + let mut buf = vec![0u8; READ_BUFFER_BYTES]; + loop { + let n = stream + .read(&mut buf) + .await + .context("read from connection")?; + if n == 0 { + return Ok(()); // peer closed the connection + } + let recorded = u64::try_from(n).unwrap_or(u64::MAX); + let total = counter.record(recorded); + // The claim: lading pushed load and it arrived here. + lading_antithesis::sometimes!(total > 0, "sink received bytes"); + } +}