From 4ebfa74ec8f88edb4d8a51b36df76fbad3deca0d Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Wed, 27 May 2026 18:18:08 +0300 Subject: [PATCH 01/30] feat(nodectl): add audit log event types, config, and JSON serde tests --- src/Cargo.lock | 2 + src/node-control/service/Cargo.toml | 2 + src/node-control/service/src/audit/config.rs | 45 +++ src/node-control/service/src/audit/enums.rs | 126 +++++++ src/node-control/service/src/audit/event.rs | 354 ++++++++++++++++++ src/node-control/service/src/audit/mod.rs | 12 + .../service/src/audit/participant.rs | 28 ++ src/node-control/service/src/lib.rs | 1 + 8 files changed, 570 insertions(+) create mode 100644 src/node-control/service/src/audit/config.rs create mode 100644 src/node-control/service/src/audit/enums.rs create mode 100644 src/node-control/service/src/audit/event.rs create mode 100644 src/node-control/service/src/audit/mod.rs create mode 100644 src/node-control/service/src/audit/participant.rs diff --git a/src/Cargo.lock b/src/Cargo.lock index f71991a4..f79455af 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -4930,6 +4930,7 @@ dependencies = [ "async-trait", "axum", "base64 0.22.1", + "chrono", "common", "contracts", "control-client", @@ -4947,6 +4948,7 @@ dependencies = [ "tower", "tracing", "utoipa", + "uuid", ] [[package]] diff --git a/src/node-control/service/Cargo.toml b/src/node-control/service/Cargo.toml index fc417122..fff7db8a 100644 --- a/src/node-control/service/Cargo.toml +++ b/src/node-control/service/Cargo.toml @@ -26,6 +26,8 @@ utoipa = "4" axum = "0.8" jsonwebtoken = "9" base64 = "0.22" +chrono = { version = "0.4", features = ["serde"] } +uuid = { version = "1", features = ["serde", "v4"] } [dev-dependencies] mockall = "0.13" diff --git a/src/node-control/service/src/audit/config.rs b/src/node-control/service/src/audit/config.rs new file mode 100644 index 00000000..e3ab481f --- /dev/null +++ b/src/node-control/service/src/audit/config.rs @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AuditLogConfig { + pub path: PathBuf, + pub max_size_bytes: u64, + pub max_files: usize, + pub batch_interval_ms: u64, + pub batch_max_events: usize, + pub queue_capacity: usize, + pub queue_full_timeout_ms: u64, + pub fsync_on_batch: bool, + pub include_payload: bool, + pub record_client_ip: bool, + pub ip_anonymize: bool, + pub ring_buffer_capacity: usize, +} + +impl Default for AuditLogConfig { + fn default() -> Self { + Self { + path: PathBuf::from("./logs/audit.jsonl"), + max_size_bytes: 100 * 1024 * 1024, + max_files: 10, + batch_interval_ms: 1000, + batch_max_events: 100, + queue_capacity: 10_000, + queue_full_timeout_ms: 250, + fsync_on_batch: false, + include_payload: true, + record_client_ip: false, + ip_anonymize: false, + ring_buffer_capacity: 10_000, + } + } +} diff --git a/src/node-control/service/src/audit/enums.rs b/src/node-control/service/src/audit/enums.rs new file mode 100644 index 00000000..96f05b85 --- /dev/null +++ b/src/node-control/service/src/audit/enums.rs @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "event_type", content = "data", rename_all = "snake_case")] +#[non_exhaustive] +pub enum AuditEventPayload { + #[serde(rename = "elections.tick_started")] + ElectionsTickStarted { election_id: u64 }, + + #[serde(rename = "elections.tick_completed")] + ElectionsTickCompleted { election_id: u64, duration_ms: u64 }, + + #[serde(rename = "elections.tick_failed")] + ElectionsTickFailed { election_id: Option, error: String }, + + #[serde(rename = "elections.stake_submitted")] + ElectionsStakeSubmitted { + stake_nanotons: String, + max_factor: u32, + policy: String, + submission_time: u64, + }, + + #[serde(rename = "elections.stake_skipped")] + ElectionsStakeSkipped { + reason: StakeSkipReason, + required_nanotons: Option, + available_nanotons: Option, + }, + + #[serde(rename = "elections.withdraw_processed")] + ElectionsWithdrawProcessed { tx_hash: String }, + + #[serde(rename = "elections.withdraw_process_failed")] + ElectionsWithdrawProcessFailed { error: String }, + + #[serde(rename = "rest_api.config_updated")] + RestApiConfigUpdated { + operation: String, + changes: serde_json::Value, // diff stays free-form + }, + + #[serde(rename = "rest_api.auth_login_success")] + RestApiAuthLoginSuccess { username: String }, + + #[serde(rename = "rest_api.auth_login_rejected")] + RestApiAuthLoginRejected { username: String, reason: String }, + + #[serde(rename = "rest_api.token_rejected")] + RestApiTokenRejected { reason: String }, + + #[serde(rename = "system.service_started")] + SystemServiceStarted { version: String }, + + #[serde(rename = "system.service_stopped")] + SystemServiceStopped, + + #[serde(rename = "system.audit_events_dropped")] + SystemAuditEventsDropped { dropped_events: u64, reason: String }, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StakeSkipReason { + LowWalletBalance, + WithdrawRequestsPending, + PoolNotReady, + Other, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AuditSource { + Elections, + Rewards, + RestApi, + Vault, + System, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AuditSeverity { + Debug, + Info, + Warn, + Error, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AuditOutcome { + Success, + Failure, + Skipped, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AuditActorKind { + Service, + User, + Scheduler, + System, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AuditSubjectKind { + Node, + Elections, + Config, + Wallet, + VaultKey, + User, + RewardRound, + Recipient, +} diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs new file mode 100644 index 00000000..52d074c6 --- /dev/null +++ b/src/node-control/service/src/audit/event.rs @@ -0,0 +1,354 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use crate::audit::{ + enums::{AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, AuditSubjectKind}, + participant::{AuditActor, AuditSubject}, +}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AuditEvent { + pub schema_version: u16, + pub id: Uuid, + pub ts: DateTime, + pub source: AuditSource, + pub severity: AuditSeverity, + pub outcome: AuditOutcome, + pub actor: AuditActor, + pub subject: AuditSubject, + pub message: Option, + #[serde(flatten)] + pub payload: AuditEventPayload, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::audit::{ + config::AuditLogConfig, + enums::{AuditActorKind, AuditEventPayload, StakeSkipReason}, + }; + use serde_json::{Value, json}; + use std::{collections::BTreeMap, path::PathBuf}; + + const FIXTURE_ID: &str = "9b6c2b5a-9f9d-4a9f-bc31-9a89b0e9d111"; + const FIXTURE_TS: &str = "2026-05-22T12:10:30.123Z"; + + fn fixture_id() -> Uuid { + FIXTURE_ID.parse().unwrap() + } + + fn fixture_ts() -> DateTime { + FIXTURE_TS.parse().unwrap() + } + + fn assert_json_eq(actual: &AuditEvent, expected: Value) { + let actual_value = serde_json::to_value(actual).expect("serialize event"); + assert_eq!(actual_value, expected); + } + + #[test] + fn serializes_stake_submitted_to_expected_json() { + let event = AuditEvent { + schema_version: 1, + id: fixture_id(), + ts: fixture_ts(), + source: AuditSource::Elections, + severity: AuditSeverity::Info, + outcome: AuditOutcome::Success, + actor: AuditActor { + kind: AuditActorKind::Service, + id: Some("elections-task".into()), + role: None, + ip: None, + }, + subject: AuditSubject { + kind: AuditSubjectKind::Node, + id: Some("node1".into()), + election_id: Some(1_779_265_552), + labels: BTreeMap::new(), + }, + message: None, + payload: AuditEventPayload::ElectionsStakeSubmitted { + stake_nanotons: "50000000000000".into(), + max_factor: 196_608, + policy: "adaptive_split50".into(), + submission_time: 1_779_265_400, + }, + }; + + assert_json_eq( + &event, + json!({ + "schema_version": 1, + "id": FIXTURE_ID, + "ts": FIXTURE_TS, + "source": "elections", + "severity": "info", + "outcome": "success", + "actor": { + "kind": "service", + "id": "elections-task", + "role": null, + "ip": null + }, + "subject": { + "kind": "node", + "id": "node1", + "election_id": 1779265552 + }, + "message": null, + "event_type": "elections.stake_submitted", + "data": { + "stake_nanotons": "50000000000000", + "max_factor": 196608, + "policy": "adaptive_split50", + "submission_time": 1779265400 + } + }), + ); + } + + #[test] + fn serializes_stake_skipped_to_expected_json() { + let event = AuditEvent { + schema_version: 1, + id: fixture_id(), + ts: fixture_ts(), + source: AuditSource::Elections, + severity: AuditSeverity::Warn, + outcome: AuditOutcome::Skipped, + actor: AuditActor { + kind: AuditActorKind::Service, + id: Some("elections-task".into()), + role: None, + ip: None, + }, + subject: AuditSubject { + kind: AuditSubjectKind::Node, + id: Some("node6".into()), + election_id: Some(1_779_265_552), + labels: BTreeMap::new(), + }, + message: None, + payload: AuditEventPayload::ElectionsStakeSkipped { + reason: StakeSkipReason::LowWalletBalance, + required_nanotons: Some("1200000000".into()), + available_nanotons: Some("900000000".into()), + }, + }; + + assert_json_eq( + &event, + json!({ + "schema_version": 1, + "id": FIXTURE_ID, + "ts": FIXTURE_TS, + "source": "elections", + "severity": "warn", + "outcome": "skipped", + "actor": { + "kind": "service", + "id": "elections-task", + "role": null, + "ip": null + }, + "subject": { + "kind": "node", + "id": "node6", + "election_id": 1779265552 + }, + "message": null, + "event_type": "elections.stake_skipped", + "data": { + "reason": "low_wallet_balance", + "required_nanotons": "1200000000", + "available_nanotons": "900000000" + } + }), + ); + } + + #[test] + fn serializes_config_updated_to_expected_json() { + let event = AuditEvent { + schema_version: 1, + id: fixture_id(), + ts: fixture_ts(), + source: AuditSource::RestApi, + severity: AuditSeverity::Info, + outcome: AuditOutcome::Success, + actor: AuditActor { + kind: AuditActorKind::User, + id: Some("admin".into()), + role: Some("operator".into()), + ip: None, + }, + subject: AuditSubject { + kind: AuditSubjectKind::Config, + id: Some("elections".into()), + election_id: None, + labels: BTreeMap::new(), + }, + message: None, + payload: AuditEventPayload::RestApiConfigUpdated { + operation: "elections.wait_updated".into(), + changes: json!({ + "sleep_period_pct": { "old": 0.2, "new": 0.9 }, + "waiting_period_pct": { "old": 0.4, "new": 0.95 } + }), + }, + }; + + assert_json_eq( + &event, + json!({ + "schema_version": 1, + "id": FIXTURE_ID, + "ts": FIXTURE_TS, + "source": "rest_api", + "severity": "info", + "outcome": "success", + "actor": { + "kind": "user", + "id": "admin", + "role": "operator", + "ip": null + }, + "subject": { + "kind": "config", + "id": "elections", + "election_id": null + }, + "message": null, + "event_type": "rest_api.config_updated", + "data": { + "operation": "elections.wait_updated", + "changes": { + "sleep_period_pct": { "old": 0.2, "new": 0.9 }, + "waiting_period_pct": { "old": 0.4, "new": 0.95 } + } + } + }), + ); + } + + fn sample_event(payload: AuditEventPayload) -> AuditEvent { + AuditEvent { + schema_version: 1, + id: fixture_id(), + ts: fixture_ts(), + source: AuditSource::System, + severity: AuditSeverity::Info, + outcome: AuditOutcome::Success, + actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, + subject: AuditSubject { + kind: AuditSubjectKind::Node, + id: Some("node1".into()), + election_id: None, + labels: BTreeMap::new(), + }, + message: None, + payload, + } + } + + fn all_payload_variants() -> Vec { + vec![ + AuditEventPayload::ElectionsTickStarted { election_id: 1 }, + AuditEventPayload::ElectionsTickCompleted { election_id: 1, duration_ms: 42 }, + AuditEventPayload::ElectionsTickFailed { election_id: Some(1), error: "boom".into() }, + AuditEventPayload::ElectionsStakeSubmitted { + stake_nanotons: "1".into(), + max_factor: 1, + policy: "all".into(), + submission_time: 1, + }, + AuditEventPayload::ElectionsStakeSkipped { + reason: StakeSkipReason::WithdrawRequestsPending, + required_nanotons: None, + available_nanotons: None, + }, + AuditEventPayload::ElectionsWithdrawProcessed { tx_hash: "abc".into() }, + AuditEventPayload::ElectionsWithdrawProcessFailed { error: "oops".into() }, + AuditEventPayload::RestApiConfigUpdated { + operation: "patch".into(), + changes: json!({ "path": "/v1/elections/settings" }), + }, + AuditEventPayload::RestApiAuthLoginSuccess { username: "admin".into() }, + AuditEventPayload::RestApiAuthLoginRejected { + username: "admin".into(), + reason: "bad password".into(), + }, + AuditEventPayload::RestApiTokenRejected { reason: "expired".into() }, + AuditEventPayload::SystemServiceStarted { version: "0.5.0".into() }, + AuditEventPayload::SystemServiceStopped, + AuditEventPayload::SystemAuditEventsDropped { + dropped_events: 3, + reason: "queue_full_after_timeout".into(), + }, + ] + } + + #[test] + fn round_trip_all_variants() { + for payload in all_payload_variants() { + let event = sample_event(payload); + let expected = serde_json::to_value(&event).expect("serialize"); + let json = serde_json::to_string(&event).expect("serialize string"); + let restored: AuditEvent = serde_json::from_str(&json).expect("deserialize"); + let actual = serde_json::to_value(&restored).expect("reserialize"); + assert_eq!(expected, actual, "json: {json}"); + } + } + + #[test] + fn subject_labels_omitted_when_empty() { + let event = AuditEvent { + schema_version: 1, + id: fixture_id(), + ts: fixture_ts(), + source: AuditSource::Elections, + severity: AuditSeverity::Info, + outcome: AuditOutcome::Success, + actor: AuditActor { kind: AuditActorKind::Service, id: None, role: None, ip: None }, + subject: AuditSubject { + kind: AuditSubjectKind::Node, + id: Some("node1".into()), + election_id: None, + labels: BTreeMap::new(), + }, + message: None, + payload: AuditEventPayload::ElectionsTickStarted { election_id: 1 }, + }; + + let value = serde_json::to_value(&event).expect("serialize"); + let subject = value.get("subject").expect("subject").as_object().expect("object"); + assert!(!subject.contains_key("labels"), "empty labels must not appear in JSON: {value}"); + } + + #[test] + fn default_config_matches_spec_defaults() { + let cfg = AuditLogConfig::default(); + assert_eq!(cfg.path, PathBuf::from("./logs/audit.jsonl")); + assert_eq!(cfg.max_size_bytes, 100 * 1024 * 1024); + assert_eq!(cfg.max_files, 10); + assert_eq!(cfg.batch_interval_ms, 1000); + assert_eq!(cfg.batch_max_events, 100); + assert_eq!(cfg.queue_capacity, 10_000); + assert_eq!(cfg.queue_full_timeout_ms, 250); + assert!(!cfg.fsync_on_batch); + assert!(cfg.include_payload); + assert!(!cfg.record_client_ip); + assert!(!cfg.ip_anonymize); + assert_eq!(cfg.ring_buffer_capacity, 10_000); + } +} diff --git a/src/node-control/service/src/audit/mod.rs b/src/node-control/service/src/audit/mod.rs new file mode 100644 index 00000000..658c8419 --- /dev/null +++ b/src/node-control/service/src/audit/mod.rs @@ -0,0 +1,12 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +pub mod config; +pub mod enums; +pub mod event; +pub mod participant; diff --git a/src/node-control/service/src/audit/participant.rs b/src/node-control/service/src/audit/participant.rs new file mode 100644 index 00000000..002ea2b6 --- /dev/null +++ b/src/node-control/service/src/audit/participant.rs @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use crate::audit::enums::{AuditActorKind, AuditSubjectKind}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AuditActor { + pub kind: AuditActorKind, + pub id: Option, + pub role: Option, + pub ip: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AuditSubject { + pub kind: AuditSubjectKind, + pub id: Option, + pub election_id: Option, // first-class for hot filtering + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub labels: BTreeMap, +} diff --git a/src/node-control/service/src/lib.rs b/src/node-control/service/src/lib.rs index 55775177..cbfe1745 100644 --- a/src/node-control/service/src/lib.rs +++ b/src/node-control/service/src/lib.rs @@ -6,6 +6,7 @@ * * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ +pub mod audit; pub mod auth; pub mod contracts; pub mod elections; From edb0b94dc1f144ca936d399bd701b1b2d5c137e0 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Thu, 28 May 2026 13:03:19 +0300 Subject: [PATCH 02/30] fix: review comments --- src/node-control/service/src/audit/enums.rs | 49 ++++++++------ src/node-control/service/src/audit/event.rs | 64 +++++++++++-------- src/node-control/service/src/audit/mod.rs | 8 +++ .../service/src/audit/participant.rs | 9 ++- 4 files changed, 83 insertions(+), 47 deletions(-) diff --git a/src/node-control/service/src/audit/enums.rs b/src/node-control/service/src/audit/enums.rs index 96f05b85..71ccf94e 100644 --- a/src/node-control/service/src/audit/enums.rs +++ b/src/node-control/service/src/audit/enums.rs @@ -8,39 +8,49 @@ */ use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "event_type", content = "data", rename_all = "snake_case")] #[non_exhaustive] pub enum AuditEventPayload { - #[serde(rename = "elections.tick_started")] - ElectionsTickStarted { election_id: u64 }, - - #[serde(rename = "elections.tick_completed")] - ElectionsTickCompleted { election_id: u64, duration_ms: u64 }, - - #[serde(rename = "elections.tick_failed")] - ElectionsTickFailed { election_id: Option, error: String }, + #[serde(rename = "elections.key_generated")] + ElectionsKeyGenerated { + election_id: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pubkey: Option, + }, #[serde(rename = "elections.stake_submitted")] ElectionsStakeSubmitted { + election_id: u64, stake_nanotons: String, max_factor: u32, policy: String, submission_time: u64, }, + #[serde(rename = "elections.stake_accepted")] + ElectionsStakeAccepted { election_id: u64, stake_nanotons: String }, + #[serde(rename = "elections.stake_skipped")] ElectionsStakeSkipped { + election_id: u64, reason: StakeSkipReason, + #[serde(skip_serializing_if = "Option::is_none")] required_nanotons: Option, + #[serde(skip_serializing_if = "Option::is_none")] available_nanotons: Option, }, #[serde(rename = "elections.withdraw_processed")] - ElectionsWithdrawProcessed { tx_hash: String }, - - #[serde(rename = "elections.withdraw_process_failed")] - ElectionsWithdrawProcessFailed { error: String }, + ElectionsWithdrawProcessed { election_id: u64, tx_hash: String }, + + #[serde(rename = "elections.stake_recovered")] + ElectionsStakeRecovered { + election_id: u64, + amount_nanotons: String, + #[serde(skip_serializing_if = "Option::is_none")] + tx_hash: Option, + }, #[serde(rename = "rest_api.config_updated")] RestApiConfigUpdated { @@ -67,8 +77,9 @@ pub enum AuditEventPayload { SystemAuditEventsDropped { dropped_events: u64, reason: String }, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] +#[non_exhaustive] pub enum StakeSkipReason { LowWalletBalance, WithdrawRequestsPending, @@ -76,7 +87,7 @@ pub enum StakeSkipReason { Other, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum AuditSource { Elections, @@ -86,7 +97,7 @@ pub enum AuditSource { System, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum AuditSeverity { Debug, @@ -95,7 +106,7 @@ pub enum AuditSeverity { Error, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum AuditOutcome { Success, @@ -103,7 +114,7 @@ pub enum AuditOutcome { Skipped, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum AuditActorKind { Service, @@ -112,7 +123,7 @@ pub enum AuditActorKind { System, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum AuditSubjectKind { Node, diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index 52d074c6..bcda2f06 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -7,14 +7,14 @@ * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ use crate::audit::{ - enums::{AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, AuditSubjectKind}, + enums::{AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource}, participant::{AuditActor, AuditSubject}, }; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct AuditEvent { pub schema_version: u16, pub id: Uuid, @@ -24,6 +24,7 @@ pub struct AuditEvent { pub outcome: AuditOutcome, pub actor: AuditActor, pub subject: AuditSubject, + #[serde(skip_serializing_if = "Option::is_none")] pub message: Option, #[serde(flatten)] pub payload: AuditEventPayload, @@ -34,7 +35,7 @@ mod tests { use super::*; use crate::audit::{ config::AuditLogConfig, - enums::{AuditActorKind, AuditEventPayload, StakeSkipReason}, + enums::{AuditActorKind, AuditEventPayload, AuditSubjectKind, StakeSkipReason}, }; use serde_json::{Value, json}; use std::{collections::BTreeMap, path::PathBuf}; @@ -78,6 +79,7 @@ mod tests { }, message: None, payload: AuditEventPayload::ElectionsStakeSubmitted { + election_id: 1_779_265_552, stake_nanotons: "50000000000000".into(), max_factor: 196_608, policy: "adaptive_split50".into(), @@ -96,18 +98,16 @@ mod tests { "outcome": "success", "actor": { "kind": "service", - "id": "elections-task", - "role": null, - "ip": null + "id": "elections-task" }, "subject": { "kind": "node", "id": "node1", "election_id": 1779265552 }, - "message": null, "event_type": "elections.stake_submitted", "data": { + "election_id": 1779265552, "stake_nanotons": "50000000000000", "max_factor": 196608, "policy": "adaptive_split50", @@ -140,6 +140,7 @@ mod tests { }, message: None, payload: AuditEventPayload::ElectionsStakeSkipped { + election_id: 1_779_265_552, reason: StakeSkipReason::LowWalletBalance, required_nanotons: Some("1200000000".into()), available_nanotons: Some("900000000".into()), @@ -157,18 +158,16 @@ mod tests { "outcome": "skipped", "actor": { "kind": "service", - "id": "elections-task", - "role": null, - "ip": null + "id": "elections-task" }, "subject": { "kind": "node", "id": "node6", "election_id": 1779265552 }, - "message": null, "event_type": "elections.stake_skipped", "data": { + "election_id": 1779265552, "reason": "low_wallet_balance", "required_nanotons": "1200000000", "available_nanotons": "900000000" @@ -220,15 +219,12 @@ mod tests { "actor": { "kind": "user", "id": "admin", - "role": "operator", - "ip": null + "role": "operator" }, "subject": { "kind": "config", - "id": "elections", - "election_id": null + "id": "elections" }, - "message": null, "event_type": "rest_api.config_updated", "data": { "operation": "elections.wait_updated", @@ -262,23 +258,38 @@ mod tests { } fn all_payload_variants() -> Vec { + const ELECTION_ID: u64 = 1_779_265_552; vec![ - AuditEventPayload::ElectionsTickStarted { election_id: 1 }, - AuditEventPayload::ElectionsTickCompleted { election_id: 1, duration_ms: 42 }, - AuditEventPayload::ElectionsTickFailed { election_id: Some(1), error: "boom".into() }, + AuditEventPayload::ElectionsKeyGenerated { + election_id: ELECTION_ID, + pubkey: Some("aabb".into()), + }, AuditEventPayload::ElectionsStakeSubmitted { + election_id: ELECTION_ID, stake_nanotons: "1".into(), max_factor: 1, policy: "all".into(), submission_time: 1, }, + AuditEventPayload::ElectionsStakeAccepted { + election_id: ELECTION_ID, + stake_nanotons: "50000000000000".into(), + }, AuditEventPayload::ElectionsStakeSkipped { + election_id: ELECTION_ID, reason: StakeSkipReason::WithdrawRequestsPending, required_nanotons: None, available_nanotons: None, }, - AuditEventPayload::ElectionsWithdrawProcessed { tx_hash: "abc".into() }, - AuditEventPayload::ElectionsWithdrawProcessFailed { error: "oops".into() }, + AuditEventPayload::ElectionsWithdrawProcessed { + election_id: ELECTION_ID, + tx_hash: "abc".into(), + }, + AuditEventPayload::ElectionsStakeRecovered { + election_id: ELECTION_ID, + amount_nanotons: "50000000000000".into(), + tx_hash: Some("def".into()), + }, AuditEventPayload::RestApiConfigUpdated { operation: "patch".into(), changes: json!({ "path": "/v1/elections/settings" }), @@ -302,11 +313,9 @@ mod tests { fn round_trip_all_variants() { for payload in all_payload_variants() { let event = sample_event(payload); - let expected = serde_json::to_value(&event).expect("serialize"); - let json = serde_json::to_string(&event).expect("serialize string"); + let json = serde_json::to_string(&event).expect("serialize"); let restored: AuditEvent = serde_json::from_str(&json).expect("deserialize"); - let actual = serde_json::to_value(&restored).expect("reserialize"); - assert_eq!(expected, actual, "json: {json}"); + assert_eq!(event, restored, "json: {json}"); } } @@ -327,7 +336,10 @@ mod tests { labels: BTreeMap::new(), }, message: None, - payload: AuditEventPayload::ElectionsTickStarted { election_id: 1 }, + payload: AuditEventPayload::ElectionsWithdrawProcessed { + election_id: 1_779_265_552, + tx_hash: "abc".into(), + }, }; let value = serde_json::to_value(&event).expect("serialize"); diff --git a/src/node-control/service/src/audit/mod.rs b/src/node-control/service/src/audit/mod.rs index 658c8419..d7ebb428 100644 --- a/src/node-control/service/src/audit/mod.rs +++ b/src/node-control/service/src/audit/mod.rs @@ -10,3 +10,11 @@ pub mod config; pub mod enums; pub mod event; pub mod participant; + +pub use config::AuditLogConfig; +pub use enums::{ + AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, AuditSubjectKind, + StakeSkipReason, +}; +pub use event::AuditEvent; +pub use participant::{AuditActor, AuditSubject}; diff --git a/src/node-control/service/src/audit/participant.rs b/src/node-control/service/src/audit/participant.rs index 002ea2b6..b6fca698 100644 --- a/src/node-control/service/src/audit/participant.rs +++ b/src/node-control/service/src/audit/participant.rs @@ -10,18 +10,23 @@ use crate::audit::enums::{AuditActorKind, AuditSubjectKind}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct AuditActor { pub kind: AuditActorKind, + #[serde(skip_serializing_if = "Option::is_none")] pub id: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub role: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ip: Option, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct AuditSubject { pub kind: AuditSubjectKind, + #[serde(skip_serializing_if = "Option::is_none")] pub id: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub election_id: Option, // first-class for hot filtering #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub labels: BTreeMap, From 459c74b88cc8724bb5107a1604a4a3a020b6def6 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Thu, 28 May 2026 15:51:19 +0300 Subject: [PATCH 03/30] feat(nodectl): wire AuditLog trait and factory into composition root --- src/Cargo.lock | 1 + .../src/commands/nodectl/config_cmd.rs | 1 + src/node-control/common/src/app_config.rs | 96 +++++++++++++++++++ src/node-control/service/Cargo.toml | 1 + src/node-control/service/src/audit/config.rs | 45 --------- src/node-control/service/src/audit/event.rs | 2 +- src/node-control/service/src/audit/factory.rs | 83 ++++++++++++++++ src/node-control/service/src/audit/log.rs | 64 +++++++++++++ src/node-control/service/src/audit/mod.rs | 7 +- .../service/src/auth/user_store.rs | 1 + .../service/src/contracts/contracts_task.rs | 1 + .../service/src/elections/election_task.rs | 3 +- .../service/src/http/auth_tests.rs | 4 + .../service/src/http/config_handlers_tests.rs | 2 + .../src/http/entity_crud_handlers_tests.rs | 3 + .../service/src/http/http_server_task.rs | 11 ++- .../service/src/service_main_task.rs | 14 ++- src/node-control/service/src/task/mod.rs | 6 +- .../service/src/task/task_manager.rs | 1 + 19 files changed, 294 insertions(+), 52 deletions(-) delete mode 100644 src/node-control/service/src/audit/config.rs create mode 100644 src/node-control/service/src/audit/factory.rs create mode 100644 src/node-control/service/src/audit/log.rs diff --git a/src/Cargo.lock b/src/Cargo.lock index 39a159e9..5e97df97 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -4943,6 +4943,7 @@ dependencies = [ "serde", "serde_json", "tempfile", + "thiserror 2.0.18", "tokio", "ton-http-api-client", "ton_block", diff --git a/src/node-control/commands/src/commands/nodectl/config_cmd.rs b/src/node-control/commands/src/commands/nodectl/config_cmd.rs index b4981196..e6f56bba 100644 --- a/src/node-control/commands/src/commands/nodectl/config_cmd.rs +++ b/src/node-control/commands/src/commands/nodectl/config_cmd.rs @@ -139,6 +139,7 @@ impl GenerateCmd { tick_interval: 40, automation: Default::default(), log: Some(LogConfig::default()), + audit_log: Default::default(), }; save_config(&config, path)?; diff --git a/src/node-control/common/src/app_config.rs b/src/node-control/common/src/app_config.rs index 5350f6b7..7b755c1c 100644 --- a/src/node-control/common/src/app_config.rs +++ b/src/node-control/common/src/app_config.rs @@ -927,6 +927,89 @@ impl Default for LogConfig { } } +fn default_audit_log_path() -> PathBuf { + PathBuf::from("./logs/audit.jsonl") +} + +fn default_audit_max_size_bytes() -> u64 { + 100 * 1024 * 1024 +} + +fn default_audit_max_files() -> usize { + 10 +} + +fn default_audit_batch_interval_ms() -> u64 { + 1000 +} + +fn default_audit_batch_max_events() -> usize { + 100 +} + +fn default_audit_queue_capacity() -> usize { + 10_000 +} + +fn default_audit_queue_full_timeout_ms() -> u64 { + 250 +} + +fn default_audit_include_payload() -> bool { + true +} + +fn default_audit_ring_buffer_capacity() -> usize { + 10_000 +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +pub struct AuditLogConfig { + #[serde(default = "default_audit_log_path")] + pub path: PathBuf, + #[serde(default = "default_audit_max_size_bytes")] + pub max_size_bytes: u64, + #[serde(default = "default_audit_max_files")] + pub max_files: usize, + #[serde(default = "default_audit_batch_interval_ms")] + pub batch_interval_ms: u64, + #[serde(default = "default_audit_batch_max_events")] + pub batch_max_events: usize, + #[serde(default = "default_audit_queue_capacity")] + pub queue_capacity: usize, + #[serde(default = "default_audit_queue_full_timeout_ms")] + pub queue_full_timeout_ms: u64, + #[serde(default)] + pub fsync_on_batch: bool, + #[serde(default = "default_audit_include_payload")] + pub include_payload: bool, + #[serde(default)] + pub record_client_ip: bool, + #[serde(default)] + pub ip_anonymize: bool, + #[serde(default = "default_audit_ring_buffer_capacity")] + pub ring_buffer_capacity: usize, +} + +impl Default for AuditLogConfig { + fn default() -> Self { + Self { + path: default_audit_log_path(), + max_size_bytes: default_audit_max_size_bytes(), + max_files: default_audit_max_files(), + batch_interval_ms: default_audit_batch_interval_ms(), + batch_max_events: default_audit_batch_max_events(), + queue_capacity: default_audit_queue_capacity(), + queue_full_timeout_ms: default_audit_queue_full_timeout_ms(), + fsync_on_batch: false, + include_payload: default_audit_include_payload(), + record_client_ip: false, + ip_anonymize: false, + ring_buffer_capacity: default_audit_ring_buffer_capacity(), + } + } +} + // Defaults aligned with `service/src/contracts/contracts_task.rs` (contracts task). fn default_contracts_wallet_deploy() -> u64 { @@ -1101,6 +1184,8 @@ pub struct AppConfig { #[serde(default)] pub automation: ContractsAutomationConfig, pub log: Option, + #[serde(default)] + pub audit_log: AuditLogConfig, } impl AppConfig { @@ -1756,4 +1841,15 @@ mod tests { assert_eq!(c.wallet.threshold, 5_000_000_000); assert!(c.validate().is_ok()); } + + #[test] + fn app_config_deserializes_without_audit_log_field() { + let json = r#"{ + "nodes": {}, + "ton_http_api": {}, + "http": {} + }"#; + let cfg: AppConfig = serde_json::from_str(json).unwrap(); + assert_eq!(cfg.audit_log, AuditLogConfig::default()); + } } diff --git a/src/node-control/service/Cargo.toml b/src/node-control/service/Cargo.toml index 6ff6a8ea..b1ac0345 100644 --- a/src/node-control/service/Cargo.toml +++ b/src/node-control/service/Cargo.toml @@ -27,6 +27,7 @@ axum = "0.8" jsonwebtoken = "9" base64 = "0.22" chrono = { version = "0.4", features = ["serde"] } +thiserror = "2" uuid = { version = "1", features = ["serde", "v4"] } [dev-dependencies] diff --git a/src/node-control/service/src/audit/config.rs b/src/node-control/service/src/audit/config.rs deleted file mode 100644 index e3ab481f..00000000 --- a/src/node-control/service/src/audit/config.rs +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2025-2026 RSquad Blockchain Lab. - * - * Licensed under the GNU General Public License v3.0. - * See the LICENSE file in the root of this repository. - * - * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. - */ -use serde::{Deserialize, Serialize}; -use std::path::PathBuf; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AuditLogConfig { - pub path: PathBuf, - pub max_size_bytes: u64, - pub max_files: usize, - pub batch_interval_ms: u64, - pub batch_max_events: usize, - pub queue_capacity: usize, - pub queue_full_timeout_ms: u64, - pub fsync_on_batch: bool, - pub include_payload: bool, - pub record_client_ip: bool, - pub ip_anonymize: bool, - pub ring_buffer_capacity: usize, -} - -impl Default for AuditLogConfig { - fn default() -> Self { - Self { - path: PathBuf::from("./logs/audit.jsonl"), - max_size_bytes: 100 * 1024 * 1024, - max_files: 10, - batch_interval_ms: 1000, - batch_max_events: 100, - queue_capacity: 10_000, - queue_full_timeout_ms: 250, - fsync_on_batch: false, - include_payload: true, - record_client_ip: false, - ip_anonymize: false, - ring_buffer_capacity: 10_000, - } - } -} diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index bcda2f06..ef0b259a 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -34,7 +34,7 @@ pub struct AuditEvent { mod tests { use super::*; use crate::audit::{ - config::AuditLogConfig, + AuditLogConfig, enums::{AuditActorKind, AuditEventPayload, AuditSubjectKind, StakeSkipReason}, }; use serde_json::{Value, json}; diff --git a/src/node-control/service/src/audit/factory.rs b/src/node-control/service/src/audit/factory.rs new file mode 100644 index 00000000..285aab9e --- /dev/null +++ b/src/node-control/service/src/audit/factory.rs @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use crate::audit::{ + AuditLogConfig, + log::{AuditLog, NoopAuditLog}, +}; +use std::sync::Arc; +use thiserror::Error; + +pub struct AuditLogFactory; + +impl AuditLogFactory { + pub async fn from_config(config: &AuditLogConfig) -> Result, AuditInitError> { + // SMA-99.3: spawn JsonlAuditLog writer from `config`. + let _ = config; + tracing::info!("audit log: NoopAuditLog (JsonlAuditLog wiring is SMA-99.3)"); + Ok(Arc::new(NoopAuditLog)) + } +} + +#[derive(Debug, Error)] +pub enum AuditInitError { + #[error("audit log path is invalid: {0}")] + InvalidPath(String), + #[error("failed to create audit directory: {0}")] + DirCreate(#[source] std::io::Error), + #[error("failed to open audit file: {0}")] + FileOpen(#[source] std::io::Error), +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::audit::{ + AuditEvent, + enums::{ + AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, + AuditSubjectKind, + }, + participant::{AuditActor, AuditSubject}, + }; + use chrono::Utc; + use std::{collections::BTreeMap, path::PathBuf}; + use uuid::Uuid; + + fn sample_event() -> AuditEvent { + AuditEvent { + schema_version: 1, + id: Uuid::new_v4(), + ts: Utc::now(), + source: AuditSource::System, + severity: AuditSeverity::Info, + outcome: AuditOutcome::Success, + actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, + subject: AuditSubject { + kind: AuditSubjectKind::Config, + id: None, + election_id: None, + labels: BTreeMap::new(), + }, + message: None, + payload: AuditEventPayload::SystemServiceStarted { version: "test".into() }, + } + } + + #[tokio::test] + async fn factory_returns_audit_log_that_accepts_events() { + let cfg = AuditLogConfig { + path: PathBuf::from("/tmp/custom-audit.jsonl"), + max_size_bytes: 1, + ..AuditLogConfig::default() + }; + let log = AuditLogFactory::from_config(&cfg).await.expect("factory init"); + log.record(sample_event()).await; + log.record(sample_event()).await; + } +} diff --git a/src/node-control/service/src/audit/log.rs b/src/node-control/service/src/audit/log.rs new file mode 100644 index 00000000..c1adc024 --- /dev/null +++ b/src/node-control/service/src/audit/log.rs @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use crate::audit::AuditEvent; +use async_trait::async_trait; + +#[async_trait] +pub trait AuditLog: Send + Sync { + async fn record(&self, event: AuditEvent); +} + +pub struct NoopAuditLog; + +#[async_trait] +impl AuditLog for NoopAuditLog { + async fn record(&self, _event: AuditEvent) {} +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::audit::{ + AuditEvent, + enums::{ + AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, + AuditSubjectKind, + }, + participant::{AuditActor, AuditSubject}, + }; + use chrono::Utc; + use std::collections::BTreeMap; + use uuid::Uuid; + + fn sample_event() -> AuditEvent { + AuditEvent { + schema_version: 1, + id: Uuid::new_v4(), + ts: Utc::now(), + source: AuditSource::System, + severity: AuditSeverity::Info, + outcome: AuditOutcome::Success, + actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, + subject: AuditSubject { + kind: AuditSubjectKind::Config, + id: None, + election_id: None, + labels: BTreeMap::new(), + }, + message: None, + payload: AuditEventPayload::SystemServiceStarted { version: "test".into() }, + } + } + + #[tokio::test] + async fn noop_audit_log_record_completes() { + let log = NoopAuditLog; + log.record(sample_event()).await; + } +} diff --git a/src/node-control/service/src/audit/mod.rs b/src/node-control/service/src/audit/mod.rs index d7ebb428..1be3627c 100644 --- a/src/node-control/service/src/audit/mod.rs +++ b/src/node-control/service/src/audit/mod.rs @@ -6,15 +6,18 @@ * * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ -pub mod config; pub mod enums; pub mod event; +pub mod factory; +pub mod log; pub mod participant; -pub use config::AuditLogConfig; +pub use common::app_config::AuditLogConfig; pub use enums::{ AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, AuditSubjectKind, StakeSkipReason, }; pub use event::AuditEvent; +pub use factory::{AuditInitError, AuditLogFactory}; +pub use log::{AuditLog, NoopAuditLog}; pub use participant::{AuditActor, AuditSubject}; diff --git a/src/node-control/service/src/auth/user_store.rs b/src/node-control/service/src/auth/user_store.rs index 3a8ec0f4..638610ad 100644 --- a/src/node-control/service/src/auth/user_store.rs +++ b/src/node-control/service/src/auth/user_store.rs @@ -439,6 +439,7 @@ mod tests { tick_interval: 30, automation: Default::default(), log: Some(Default::default()), + audit_log: Default::default(), } } diff --git a/src/node-control/service/src/contracts/contracts_task.rs b/src/node-control/service/src/contracts/contracts_task.rs index 39c6ecd0..746f9df8 100644 --- a/src/node-control/service/src/contracts/contracts_task.rs +++ b/src/node-control/service/src/contracts/contracts_task.rs @@ -714,6 +714,7 @@ mod tests { tick_interval: 30, automation: Default::default(), log: None, + audit_log: Default::default(), }) } diff --git a/src/node-control/service/src/elections/election_task.rs b/src/node-control/service/src/elections/election_task.rs index c40aeba0..2a8282a6 100644 --- a/src/node-control/service/src/elections/election_task.rs +++ b/src/node-control/service/src/elections/election_task.rs @@ -10,7 +10,7 @@ use super::{ providers::{DefaultElectionsProvider, ElectionsProvider}, runner::{ElectionRunner, PersistStaticAdnls}, }; -use crate::runtime_config::RuntimeConfig; +use crate::{audit::log::AuditLog, runtime_config::RuntimeConfig}; use anyhow::Context; use common::{ app_config::{AppConfig, BindingStatus, ElectionsConfig}, @@ -29,6 +29,7 @@ pub async fn run( runtime_cfg: Arc, store: Arc, on_status_change: Option, + _audit: Arc, ) -> anyhow::Result<()> { let Some(config) = app_config.elections.as_ref() else { anyhow::bail!("elections config is empty"); diff --git a/src/node-control/service/src/http/auth_tests.rs b/src/node-control/service/src/http/auth_tests.rs index 5e8b40b9..0a24d8e4 100644 --- a/src/node-control/service/src/http/auth_tests.rs +++ b/src/node-control/service/src/http/auth_tests.rs @@ -106,6 +106,7 @@ fn app_cfg_with_auth(auth: AuthConfig) -> Arc { tick_interval: 30, automation: Default::default(), log: Some(Default::default()), + audit_log: Default::default(), }) } @@ -123,6 +124,7 @@ fn app_cfg_no_auth() -> Arc { tick_interval: 30, automation: Default::default(), log: Some(Default::default()), + audit_log: Default::default(), }) } @@ -171,6 +173,7 @@ async fn state_with_auth() -> AppState { user_store: Arc::new(UserStore::new(rt as Arc)), login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), + audit: Arc::new(crate::audit::log::NoopAuditLog), } } @@ -184,6 +187,7 @@ async fn state_no_auth() -> AppState { user_store: Arc::new(UserStore::new(rt.clone() as Arc)), login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), + audit: Arc::new(crate::audit::log::NoopAuditLog), } } diff --git a/src/node-control/service/src/http/config_handlers_tests.rs b/src/node-control/service/src/http/config_handlers_tests.rs index edd2e3b8..a335560c 100644 --- a/src/node-control/service/src/http/config_handlers_tests.rs +++ b/src/node-control/service/src/http/config_handlers_tests.rs @@ -59,6 +59,7 @@ fn empty_app_cfg() -> Arc { tick_interval: 30, automation: Default::default(), log: Some(Default::default()), + audit_log: Default::default(), }) } @@ -73,6 +74,7 @@ async fn state_from_cfg(cfg: AppConfig) -> AppState { user_store: Arc::new(UserStore::new(rt as Arc)), login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), + audit: Arc::new(crate::audit::log::NoopAuditLog), } } diff --git a/src/node-control/service/src/http/entity_crud_handlers_tests.rs b/src/node-control/service/src/http/entity_crud_handlers_tests.rs index 546e4da6..557f36dc 100644 --- a/src/node-control/service/src/http/entity_crud_handlers_tests.rs +++ b/src/node-control/service/src/http/entity_crud_handlers_tests.rs @@ -82,6 +82,7 @@ fn empty_app_cfg() -> AppConfig { tick_interval: 30, automation: Default::default(), log: Some(Default::default()), + audit_log: Default::default(), } } @@ -138,6 +139,7 @@ async fn app_state(cfg: AppConfig) -> AppState { user_store: Arc::new(UserStore::new(rt as Arc)), login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), + audit: Arc::new(crate::audit::log::NoopAuditLog), } } @@ -156,6 +158,7 @@ async fn app_state_with_path(cfg: AppConfig, path: std::path::PathBuf) -> AppSta user_store: Arc::new(UserStore::new(rt as Arc)), login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), + audit: Arc::new(crate::audit::log::NoopAuditLog), } } diff --git a/src/node-control/service/src/http/http_server_task.rs b/src/node-control/service/src/http/http_server_task.rs index d043f616..d4a58e04 100644 --- a/src/node-control/service/src/http/http_server_task.rs +++ b/src/node-control/service/src/http/http_server_task.rs @@ -23,6 +23,7 @@ use super::{ login_rate_limiter::{LoginRateLimiter, login_limiter_key}, }; use crate::{ + audit::log::AuditLog, auth::{ Claims, jwt::JwtAuth, @@ -53,6 +54,7 @@ pub struct AppState { /// Signalled by mutation handlers after structural config changes /// (entity CRUD, ton-http-api) so the service loop can rebuild caches. pub config_changed: Arc, + pub audit: Arc, } pub async fn run( @@ -61,6 +63,7 @@ pub async fn run( runtime_cfg: Arc, tasks: HashMap<&'static str, Arc>, config_changed: Arc, + audit: Arc, ) { tracing::info!("http-server task started"); @@ -118,6 +121,7 @@ pub async fn run( user_store, login_rate_limiter, config_changed, + audit, }; let app = routes(enable_swagger, state); @@ -1000,7 +1004,9 @@ pub struct ApiDoc; #[cfg(test)] mod tests { use super::*; - use crate::{runtime_config::RuntimeConfigStore, task::task_manager::ServiceTask}; + use crate::{ + audit::NoopAuditLog, runtime_config::RuntimeConfigStore, task::task_manager::ServiceTask, + }; use axum::body::Body; use base64::Engine; use common::{ @@ -1058,6 +1064,7 @@ mod tests { user_store, login_rate_limiter: Arc::new(tokio::sync::Mutex::new(LoginRateLimiter::default())), config_changed: Arc::new(tokio::sync::Notify::new()), + audit: Arc::new(NoopAuditLog), } } @@ -1082,6 +1089,7 @@ mod tests { tick_interval: 30, automation: Default::default(), log: Some(LogConfig::default()), + audit_log: Default::default(), }) } @@ -1099,6 +1107,7 @@ mod tests { tick_interval: 30, automation: Default::default(), log: Some(LogConfig::default()), + audit_log: Default::default(), }) } diff --git a/src/node-control/service/src/service_main_task.rs b/src/node-control/service/src/service_main_task.rs index 9098761c..4fb0f6a6 100644 --- a/src/node-control/service/src/service_main_task.rs +++ b/src/node-control/service/src/service_main_task.rs @@ -7,6 +7,7 @@ * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ use crate::{ + audit::AuditLogFactory, elections::election_task::BindingStatusCallback, http::http_server_task, runtime_config::RuntimeConfigStore, @@ -52,6 +53,11 @@ pub async fn run_with_config( .await .context("initialize runtime config store")?; let runtime_cfg = Arc::new(runtime_cfg); + + let audit = AuditLogFactory::from_config(&app_cfg.audit_log) + .await + .map_err(|e| anyhow::anyhow!("audit log init failed: {e}"))?; + let store = Arc::new(SnapshotStore::new()); // Status callback: when the elections runner detects binding status changes, @@ -83,7 +89,12 @@ pub async fn run_with_config( "elections", Arc::new(TaskController::new( "elections", - ElectionsTask::new(runtime_cfg.clone(), store.clone(), Some(on_status_change)), + ElectionsTask::new( + runtime_cfg.clone(), + store.clone(), + Some(on_status_change), + audit.clone(), + ), runtime_cfg.clone(), )), ); @@ -113,6 +124,7 @@ pub async fn run_with_config( runtime_cfg.clone(), tasks.clone(), config_changed.clone(), + audit.clone(), )); let max_wait = std::time::Duration::from_secs(10); diff --git a/src/node-control/service/src/task/mod.rs b/src/node-control/service/src/task/mod.rs index d12c1f4b..16c90e68 100644 --- a/src/node-control/service/src/task/mod.rs +++ b/src/node-control/service/src/task/mod.rs @@ -9,7 +9,10 @@ pub mod task_macro; pub mod task_manager; -use crate::{elections::election_task::BindingStatusCallback, runtime_config::RuntimeConfig, task}; +use crate::{ + audit::log::AuditLog, elections::election_task::BindingStatusCallback, + runtime_config::RuntimeConfig, task, +}; use common::snapshot::SnapshotStore; use std::sync::Arc; @@ -25,4 +28,5 @@ task!(ElectionsTask, crate::elections::election_task::run { runtime_cfg: Arc, store: Arc, on_status_change: Option, + audit: Arc, }); diff --git a/src/node-control/service/src/task/task_manager.rs b/src/node-control/service/src/task/task_manager.rs index 2936e05a..54332728 100644 --- a/src/node-control/service/src/task/task_manager.rs +++ b/src/node-control/service/src/task/task_manager.rs @@ -249,6 +249,7 @@ mod tests { tick_interval: 30, automation: Default::default(), log: None, + audit_log: Default::default(), }), }) } From a7994862352c67e1cfad2f3dc507242c7de34fc3 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Thu, 28 May 2026 16:20:12 +0300 Subject: [PATCH 04/30] fix: copilot comments --- .github/workflows/ci.yml | 5 ++- src/node-control/service/src/audit/config.rs | 45 ------------------- .../service/src/service_main_task.rs | 5 +-- 3 files changed, 6 insertions(+), 49 deletions(-) delete mode 100644 src/node-control/service/src/audit/config.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index affee76d..dab96bb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,10 @@ name: CI on: pull_request: - branches: ["master", "release/**"] + branches: + - master + - release/** + - feature/sma-99-audit-log-architecture # delete after merge into release branch paths-ignore: - "**/*.md" - "docs/**" diff --git a/src/node-control/service/src/audit/config.rs b/src/node-control/service/src/audit/config.rs deleted file mode 100644 index e3ab481f..00000000 --- a/src/node-control/service/src/audit/config.rs +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2025-2026 RSquad Blockchain Lab. - * - * Licensed under the GNU General Public License v3.0. - * See the LICENSE file in the root of this repository. - * - * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. - */ -use serde::{Deserialize, Serialize}; -use std::path::PathBuf; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AuditLogConfig { - pub path: PathBuf, - pub max_size_bytes: u64, - pub max_files: usize, - pub batch_interval_ms: u64, - pub batch_max_events: usize, - pub queue_capacity: usize, - pub queue_full_timeout_ms: u64, - pub fsync_on_batch: bool, - pub include_payload: bool, - pub record_client_ip: bool, - pub ip_anonymize: bool, - pub ring_buffer_capacity: usize, -} - -impl Default for AuditLogConfig { - fn default() -> Self { - Self { - path: PathBuf::from("./logs/audit.jsonl"), - max_size_bytes: 100 * 1024 * 1024, - max_files: 10, - batch_interval_ms: 1000, - batch_max_events: 100, - queue_capacity: 10_000, - queue_full_timeout_ms: 250, - fsync_on_batch: false, - include_payload: true, - record_client_ip: false, - ip_anonymize: false, - ring_buffer_capacity: 10_000, - } - } -} diff --git a/src/node-control/service/src/service_main_task.rs b/src/node-control/service/src/service_main_task.rs index 4fb0f6a6..2d94147e 100644 --- a/src/node-control/service/src/service_main_task.rs +++ b/src/node-control/service/src/service_main_task.rs @@ -54,9 +54,8 @@ pub async fn run_with_config( .context("initialize runtime config store")?; let runtime_cfg = Arc::new(runtime_cfg); - let audit = AuditLogFactory::from_config(&app_cfg.audit_log) - .await - .map_err(|e| anyhow::anyhow!("audit log init failed: {e}"))?; + let audit = + AuditLogFactory::from_config(&app_cfg.audit_log).await.context("audit log init failed")?; let store = Arc::new(SnapshotStore::new()); From e3501cce6b41ef6b50175ebc8b32f0ba7edc2190 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Mon, 1 Jun 2026 12:49:52 +0300 Subject: [PATCH 05/30] feat(nodectl): implement JsonlAuditLog with background writer and graceful shutdown --- src/node-control/service/src/audit/factory.rs | 68 +- .../service/src/audit/jsonl_log.rs | 132 ++++ .../service/src/audit/jsonl_writer.rs | 626 ++++++++++++++++++ src/node-control/service/src/audit/log.rs | 6 + src/node-control/service/src/audit/mod.rs | 5 +- .../service/src/service_main_task.rs | 4 + 6 files changed, 776 insertions(+), 65 deletions(-) create mode 100644 src/node-control/service/src/audit/jsonl_log.rs create mode 100644 src/node-control/service/src/audit/jsonl_writer.rs diff --git a/src/node-control/service/src/audit/factory.rs b/src/node-control/service/src/audit/factory.rs index 285aab9e..92d4d249 100644 --- a/src/node-control/service/src/audit/factory.rs +++ b/src/node-control/service/src/audit/factory.rs @@ -8,76 +8,16 @@ */ use crate::audit::{ AuditLogConfig, - log::{AuditLog, NoopAuditLog}, + jsonl_log::{AuditInitError, JsonlAuditLog}, + log::AuditLog, }; use std::sync::Arc; -use thiserror::Error; pub struct AuditLogFactory; impl AuditLogFactory { pub async fn from_config(config: &AuditLogConfig) -> Result, AuditInitError> { - // SMA-99.3: spawn JsonlAuditLog writer from `config`. - let _ = config; - tracing::info!("audit log: NoopAuditLog (JsonlAuditLog wiring is SMA-99.3)"); - Ok(Arc::new(NoopAuditLog)) - } -} - -#[derive(Debug, Error)] -pub enum AuditInitError { - #[error("audit log path is invalid: {0}")] - InvalidPath(String), - #[error("failed to create audit directory: {0}")] - DirCreate(#[source] std::io::Error), - #[error("failed to open audit file: {0}")] - FileOpen(#[source] std::io::Error), -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::audit::{ - AuditEvent, - enums::{ - AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, - AuditSubjectKind, - }, - participant::{AuditActor, AuditSubject}, - }; - use chrono::Utc; - use std::{collections::BTreeMap, path::PathBuf}; - use uuid::Uuid; - - fn sample_event() -> AuditEvent { - AuditEvent { - schema_version: 1, - id: Uuid::new_v4(), - ts: Utc::now(), - source: AuditSource::System, - severity: AuditSeverity::Info, - outcome: AuditOutcome::Success, - actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, - subject: AuditSubject { - kind: AuditSubjectKind::Config, - id: None, - election_id: None, - labels: BTreeMap::new(), - }, - message: None, - payload: AuditEventPayload::SystemServiceStarted { version: "test".into() }, - } - } - - #[tokio::test] - async fn factory_returns_audit_log_that_accepts_events() { - let cfg = AuditLogConfig { - path: PathBuf::from("/tmp/custom-audit.jsonl"), - max_size_bytes: 1, - ..AuditLogConfig::default() - }; - let log = AuditLogFactory::from_config(&cfg).await.expect("factory init"); - log.record(sample_event()).await; - log.record(sample_event()).await; + let log = JsonlAuditLog::start(config.clone()).await?; + Ok(log) } } diff --git a/src/node-control/service/src/audit/jsonl_log.rs b/src/node-control/service/src/audit/jsonl_log.rs new file mode 100644 index 00000000..e2163703 --- /dev/null +++ b/src/node-control/service/src/audit/jsonl_log.rs @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use crate::audit::{ + AuditEvent, AuditLogConfig, + jsonl_writer::{AuditCommand, AuditWriter}, + log::AuditLog, +}; +use async_trait::async_trait; +use std::{ + sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; +use thiserror::Error; +use tokio::{sync::mpsc, task::JoinHandle}; + +#[derive(Debug, Error)] +pub enum AuditInitError { + #[error("audit log path is invalid: {0}")] + InvalidPath(String), + #[error("failed to create audit directory: {0}")] + DirCreate(#[source] std::io::Error), + #[error("failed to open audit file: {0}")] + FileOpen(#[source] std::io::Error), +} + +pub struct JsonlAuditLog { + sender: mpsc::Sender, + dropped_events: Arc, + config: Arc, + /// Writer task handle, consumed by the first [`AuditLog::shutdown`] call so + /// callers can await the final drain/flush. `None` after shutdown. + writer: Mutex>>, +} + +impl JsonlAuditLog { + pub async fn start(config: AuditLogConfig) -> Result, AuditInitError> { + Self::start_inner(config, Duration::ZERO).await + } + + #[cfg(test)] + pub(crate) async fn start_with_write_delay( + config: AuditLogConfig, + write_delay: Duration, + ) -> Result, AuditInitError> { + Self::start_inner(config, write_delay).await + } + + async fn start_inner( + config: AuditLogConfig, + write_delay: Duration, + ) -> Result, AuditInitError> { + let config = Arc::new(config); + std::fs::create_dir_all(config.path.parent().ok_or_else(|| { + AuditInitError::InvalidPath(config.path.to_string_lossy().to_string()) + })?) + .map_err(AuditInitError::DirCreate)?; + + let (tx, rx) = mpsc::channel(config.queue_capacity); + let dropped_events = Arc::new(AtomicU64::new(0)); + + let writer = AuditWriter::open(config.clone(), dropped_events.clone(), write_delay).await?; + let handle = tokio::spawn(writer.run(rx)); + + Ok(Arc::new(Self { sender: tx, dropped_events, config, writer: Mutex::new(Some(handle)) })) + } + + #[cfg(test)] + pub(crate) fn dropped_events(&self) -> u64 { + self.dropped_events.load(Ordering::Relaxed) + } +} + +#[async_trait] +impl AuditLog for JsonlAuditLog { + async fn shutdown(&self) { + // Signal the writer to drain and flush the final batch. The writer also + // stops on channel close, but sending an explicit command lets us await + // the flush deterministically. + let _ = self.sender.send(AuditCommand::Shutdown).await; + + // Take the handle without holding the lock across the await. + let handle = self.writer.lock().expect("audit writer lock poisoned").take(); + if let Some(handle) = handle { + let _ = handle.await; + } + } + + async fn record(&self, event: AuditEvent) { + let cmd = AuditCommand::Event(Box::new(event)); + + match self.sender.try_send(cmd) { + Ok(()) => return, + Err(mpsc::error::TrySendError::Full(cmd)) => { + // Capture diagnostics before `cmd` is moved into the send future, + // so a dropped event can be attributed to its source/subject. + let diag = match &cmd { + AuditCommand::Event(ev) => Some((ev.id, ev.source, ev.subject.kind)), + _ => None, + }; + let timeout = Duration::from_millis(self.config.queue_full_timeout_ms); + match tokio::time::timeout(timeout, self.sender.send(cmd)).await { + Ok(Ok(())) => return, + _ => { + self.dropped_events.fetch_add(1, Ordering::Relaxed); + let (event_id, source, subject) = match diag { + Some((id, source, subject)) => (Some(id), Some(source), Some(subject)), + None => (None, None, None), + }; + tracing::warn!( + ?event_id, + ?source, + ?subject, + "audit event dropped: queue full after timeout" + ); + } + } + } + Err(mpsc::error::TrySendError::Closed(_)) => { + tracing::error!("audit log channel closed; service likely shutting down"); + } + } + } +} diff --git a/src/node-control/service/src/audit/jsonl_writer.rs b/src/node-control/service/src/audit/jsonl_writer.rs new file mode 100644 index 00000000..7185ef57 --- /dev/null +++ b/src/node-control/service/src/audit/jsonl_writer.rs @@ -0,0 +1,626 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use crate::audit::{ + AuditEvent, AuditLogConfig, + enums::{ + AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, + AuditSubjectKind, + }, + jsonl_log::AuditInitError, + participant::{AuditActor, AuditSubject}, +}; +use chrono::Utc; +use std::{ + collections::BTreeMap, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; +use tokio::sync::mpsc; +use uuid::Uuid; + +pub(crate) enum AuditCommand { + Event(Box), + /// Forces an immediate flush of buffered events. Used only by tests to make + /// assertions deterministic without waiting for the batch interval. + #[cfg(test)] + Flush, + Shutdown, +} + +pub(crate) struct AuditWriter { + config: Arc, + /// Live append handle. `None` only transiently during rotation (the old + /// handle is closed before the on-disk rename so the swap is portable to + /// platforms that forbid renaming open files, e.g. Windows). + file: Option, + current_size: u64, + batch: Vec, + dropped_events: Arc, + last_dropped_seen: u64, + /// Artificial per-write delay. Zero in production (no effect); set non-zero + /// only by tests exercising the queue-full / backpressure path. + write_delay: Duration, +} + +impl AuditWriter { + pub(crate) async fn open( + config: Arc, + dropped: Arc, + write_delay: Duration, + ) -> Result { + let path = &config.path; + let mut opts = tokio::fs::OpenOptions::new(); + opts.append(true).create(true); + #[cfg(unix)] + opts.mode(0o600); + let file = opts.open(path).await.map_err(AuditInitError::FileOpen)?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .await + .map_err(AuditInitError::FileOpen)?; + } + + let current_size = file.metadata().await.map_err(AuditInitError::FileOpen)?.len(); + + Ok(Self { + config, + file: Some(file), + current_size, + batch: Vec::with_capacity(64 * 1024), + dropped_events: dropped, + last_dropped_seen: 0, + write_delay, + }) + } + + pub(crate) async fn run(mut self, mut rx: mpsc::Receiver) { + let mut interval = + tokio::time::interval(Duration::from_millis(self.config.batch_interval_ms)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut buffered: Vec = Vec::with_capacity(self.config.batch_max_events); + + loop { + tokio::select! { + cmd = rx.recv() => { + match cmd { + Some(AuditCommand::Event(ev)) => { + buffered.push(*ev); + if buffered.len() >= self.config.batch_max_events { + self.flush(&mut buffered).await; + } + } + #[cfg(test)] + Some(AuditCommand::Flush) => self.flush(&mut buffered).await, + Some(AuditCommand::Shutdown) | None => { + self.flush(&mut buffered).await; + self.maybe_emit_dropped_recovery().await; + return; + } + } + } + _ = interval.tick() => { + if !buffered.is_empty() { + self.flush(&mut buffered).await; + } + self.maybe_emit_dropped_recovery().await; + } + } + } + } + + async fn flush(&mut self, buffered: &mut Vec) { + if buffered.is_empty() { + return; + } + + self.batch.clear(); + for ev in buffered.drain(..) { + let line = match serde_json::to_vec(&ev) { + Ok(v) => v, + Err(e) => { + tracing::error!(error = %e, "failed to serialize audit event"); + continue; + } + }; + let needed = line.len() + 1; + if self.current_size + needed as u64 > self.config.max_size_bytes { + if let Err(e) = self.write_batch_and_clear().await { + tracing::error!(error = %e, "audit write before rotation failed"); + } + if let Err(e) = self.rotate().await { + tracing::error!(error = %e, "audit rotation failed"); + continue; + } + } + self.batch.extend_from_slice(&line); + self.batch.push(b'\n'); + self.current_size += needed as u64; + } + + if let Err(e) = self.write_batch_and_clear().await { + tracing::error!(error = %e, "audit batch write failed"); + } + } + + async fn write_batch_and_clear(&mut self) -> std::io::Result<()> { + if self.batch.is_empty() { + return Ok(()); + } + if !self.write_delay.is_zero() { + tokio::time::sleep(self.write_delay).await; + } + use tokio::io::AsyncWriteExt; + let file = self + .file + .as_mut() + .ok_or_else(|| std::io::Error::other("audit file handle not open"))?; + file.write_all(&self.batch).await?; + if self.config.fsync_on_batch { + file.sync_data().await?; + } + self.batch.clear(); + Ok(()) + } + + async fn rotate(&mut self) -> std::io::Result<()> { + let path = &self.config.path; + // Total retained files (including the live one) is at least 1; the + // number of rotated history segments is `max - 1`. Guarding against 0 + // avoids an arithmetic underflow on `max - 1`. + let max = self.config.max_files.max(1); + + if max > 1 { + // Remove the oldest segment (.{max-1}) if present. + let oldest = path.with_extension(format!("jsonl.{}", max - 1)); + if tokio::fs::try_exists(&oldest).await? { + tokio::fs::remove_file(&oldest).await?; + } + + // Shift .n -> .n+1, from the oldest down to .1. + for n in (1..max - 1).rev() { + let from = path.with_extension(format!("jsonl.{}", n)); + let to = path.with_extension(format!("jsonl.{}", n + 1)); + if tokio::fs::try_exists(&from).await? { + tokio::fs::rename(&from, &to).await?; + } + } + } + + // Close the live handle before touching the file on disk, so the rename + // is valid on platforms that forbid renaming an open file. + self.file = None; + + let mut opts = tokio::fs::OpenOptions::new(); + if max > 1 { + // Preserve history: rename current -> .1, then open a fresh live file. + let rotated = path.with_extension("jsonl.1"); + tokio::fs::rename(path, &rotated).await?; + opts.append(true).create(true); + } else { + // No history retained: truncate the live file in place. + opts.write(true).create(true).truncate(true); + } + #[cfg(unix)] + opts.mode(0o600); + self.file = Some(opts.open(path).await?); + self.current_size = 0; + Ok(()) + } + + async fn maybe_emit_dropped_recovery(&mut self) { + let current = self.dropped_events.load(Ordering::Relaxed); + let delta = current.saturating_sub(self.last_dropped_seen); + if delta == 0 { + return; + } + self.last_dropped_seen = current; + + let event = AuditEvent { + schema_version: 1, + id: Uuid::new_v4(), + ts: Utc::now(), + source: AuditSource::System, + severity: AuditSeverity::Warn, + outcome: AuditOutcome::Failure, + actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, + subject: AuditSubject { + kind: AuditSubjectKind::Config, + id: None, + election_id: None, + labels: BTreeMap::new(), + }, + message: Some("audit events dropped".into()), + payload: AuditEventPayload::SystemAuditEventsDropped { + dropped_events: delta, + reason: "queue_full_after_timeout".into(), + }, + }; + let mut buf = vec![event]; + self.flush(&mut buf).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::audit::{jsonl_log::JsonlAuditLog, log::AuditLog}; + use serde_json::Value; + use std::path::{Path, PathBuf}; + use tempfile::tempdir; + + fn sample_event(tag: &str) -> AuditEvent { + AuditEvent { + schema_version: 1, + id: Uuid::new_v4(), + ts: Utc::now(), + source: AuditSource::System, + severity: AuditSeverity::Info, + outcome: AuditOutcome::Success, + actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, + subject: AuditSubject { + kind: AuditSubjectKind::Config, + id: Some(tag.into()), + election_id: None, + labels: BTreeMap::new(), + }, + message: None, + payload: AuditEventPayload::SystemServiceStarted { version: tag.into() }, + } + } + + fn large_event(payload_kb: usize) -> AuditEvent { + let mut event = sample_event("large"); + event.payload = AuditEventPayload::RestApiConfigUpdated { + operation: "update".into(), + changes: serde_json::json!({ "blob": "x".repeat(payload_kb * 1024) }), + }; + event + } + + fn test_config(dir: &Path, mut cfg: AuditLogConfig) -> AuditLogConfig { + cfg.path = dir.join("audit.jsonl"); + cfg + } + + async fn spawn_writer( + config: AuditLogConfig, + write_delay: Duration, + ) -> (mpsc::Sender, Arc, tokio::task::JoinHandle<()>, PathBuf) { + let config = Arc::new(config); + let dropped = Arc::new(AtomicU64::new(0)); + let (tx, rx) = mpsc::channel(config.queue_capacity); + let writer = AuditWriter::open(config.clone(), dropped.clone(), write_delay).await.unwrap(); + let path = config.path.clone(); + let handle = tokio::spawn(writer.run(rx)); + (tx, dropped, handle, path) + } + + async fn send_event(tx: &mpsc::Sender, event: AuditEvent) { + tx.send(AuditCommand::Event(Box::new(event))).await.unwrap(); + } + + async fn flush(tx: &mpsc::Sender) { + tx.send(AuditCommand::Flush).await.unwrap(); + } + + async fn shutdown(tx: mpsc::Sender, handle: tokio::task::JoinHandle<()>) { + let _ = tx.send(AuditCommand::Shutdown).await; + let _ = handle.await; + } + + fn read_json_lines(path: &Path) -> Vec { + let content = std::fs::read_to_string(path).unwrap_or_default(); + content + .lines() + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_str(line).expect("valid json line")) + .collect() + } + + fn count_rotated_files(dir: &Path) -> usize { + std::fs::read_dir(dir) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| entry.file_name().to_string_lossy().starts_with("audit.jsonl.")) + .count() + } + + #[tokio::test] + async fn writes_single_event_to_file() { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + batch_interval_ms: 60_000, + batch_max_events: 100, + ..AuditLogConfig::default() + }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + send_event(&tx, sample_event("one")).await; + flush(&tx).await; + shutdown(tx, handle).await; + + let lines = read_json_lines(&path); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0]["data"]["version"], "one"); + } + + #[tokio::test] + async fn batches_events_within_interval() { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + batch_interval_ms: 50, + batch_max_events: 100, + ..AuditLogConfig::default() + }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + for i in 0..5 { + send_event(&tx, sample_event(&format!("ev-{i}"))).await; + } + tokio::time::sleep(Duration::from_millis(120)).await; + shutdown(tx, handle).await; + + let lines = read_json_lines(&path); + assert_eq!(lines.len(), 5); + } + + #[tokio::test] + async fn rotates_at_max_size() { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + max_size_bytes: 1024, + batch_interval_ms: 60_000, + batch_max_events: 1, + ..AuditLogConfig::default() + }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + for _ in 0..8 { + send_event(&tx, large_event(1)).await; + flush(&tx).await; + } + shutdown(tx, handle).await; + + let rotated = path.with_extension("jsonl.1"); + assert!(rotated.exists(), "expected rotated file at {}", rotated.display()); + } + + #[tokio::test] + async fn retains_only_max_files() { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + max_size_bytes: 512, + max_files: 3, + batch_interval_ms: 60_000, + batch_max_events: 1, + ..AuditLogConfig::default() + }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + for _ in 0..12 { + send_event(&tx, large_event(1)).await; + flush(&tx).await; + } + shutdown(tx, handle).await; + + let rotated_count = count_rotated_files(dir.path()); + assert!( + rotated_count <= 2, + "expected at most max_files-1 rotated segments, got {rotated_count}" + ); + assert!(path.exists()); + } + + #[tokio::test] + async fn max_files_one_keeps_no_history() { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + max_size_bytes: 512, + max_files: 1, + batch_interval_ms: 60_000, + batch_max_events: 1, + ..AuditLogConfig::default() + }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + for _ in 0..6 { + send_event(&tx, large_event(1)).await; + flush(&tx).await; + } + shutdown(tx, handle).await; + + assert!(path.exists(), "live audit.jsonl must exist"); + assert_eq!(count_rotated_files(dir.path()), 0, "max_files=1 must keep no rotated segments"); + } + + #[tokio::test] + async fn recovers_when_live_file_missing_on_restart() { + // Models the crash window between `rename(path -> .1)` and opening the + // new live file: on restart only the rotated segment exists. A fresh + // writer must recreate `audit.jsonl` via create(true) and write to it. + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + batch_interval_ms: 60_000, + batch_max_events: 1, + ..AuditLogConfig::default() + }, + ); + let path = dir.path().join("audit.jsonl"); + + // First run persists one event, then we simulate the post-rename state: + // move the live file aside so no `audit.jsonl` exists. + let (tx, _dropped, handle, _) = spawn_writer(cfg.clone(), Duration::ZERO).await; + send_event(&tx, sample_event("first-run")).await; + flush(&tx).await; + shutdown(tx, handle).await; + std::fs::rename(&path, path.with_extension("jsonl.1")).unwrap(); + assert!(!path.exists(), "precondition: live file moved aside"); + + // Second run must recreate the live file and write into it. + let (tx, _dropped, handle, _) = spawn_writer(cfg, Duration::ZERO).await; + send_event(&tx, sample_event("after-restart")).await; + flush(&tx).await; + shutdown(tx, handle).await; + + assert!(path.exists(), "writer must recreate the live file on restart"); + let lines = read_json_lines(&path); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0]["data"]["version"], "after-restart"); + } + + #[tokio::test] + async fn concurrent_writers_no_data_loss() { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + queue_capacity: 10_000, + batch_interval_ms: 20, + batch_max_events: 200, + max_size_bytes: 50 * 1024 * 1024, + ..AuditLogConfig::default() + }, + ); + let log = JsonlAuditLog::start(cfg).await.unwrap(); + + let mut tasks = Vec::new(); + for producer in 0..10 { + let log = log.clone(); + tasks.push(tokio::spawn(async move { + for seq in 0..100 { + log.record(sample_event(&format!("p{producer}-s{seq}"))).await; + } + })); + } + for task in tasks { + task.await.unwrap(); + } + + tokio::time::sleep(Duration::from_millis(200)).await; + + let path = dir.path().join("audit.jsonl"); + let lines = read_json_lines(&path); + assert_eq!(lines.len(), 1000, "expected 1000 audit lines, got {}", lines.len()); + } + + #[tokio::test] + async fn queue_full_drops_and_increments_counter() { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + queue_capacity: 1, + queue_full_timeout_ms: 10, + batch_interval_ms: 60_000, + batch_max_events: 10_000, + ..AuditLogConfig::default() + }, + ); + let log = + JsonlAuditLog::start_with_write_delay(cfg, Duration::from_millis(500)).await.unwrap(); + + for i in 0..50 { + log.record(sample_event(&format!("drop-{i}"))).await; + } + + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(log.dropped_events() > 0, "expected dropped events counter > 0"); + } + + #[tokio::test] + async fn shutdown_flushes_pending() { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + batch_interval_ms: 60_000, + batch_max_events: 100, + ..AuditLogConfig::default() + }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + for i in 0..3 { + send_event(&tx, sample_event(&format!("pending-{i}"))).await; + } + shutdown(tx, handle).await; + + let lines = read_json_lines(&path); + assert_eq!(lines.len(), 3); + } + + #[cfg(unix)] + #[tokio::test] + async fn file_mode_0600_on_unix() { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { batch_interval_ms: 60_000, ..AuditLogConfig::default() }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + send_event(&tx, sample_event("perm")).await; + flush(&tx).await; + shutdown(tx, handle).await; + + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + } + + #[tokio::test] + async fn synthetic_dropped_event_emitted_after_drops() { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + batch_interval_ms: 50, + batch_max_events: 100, + ..AuditLogConfig::default() + }, + ); + let (tx, dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + dropped.store(7, Ordering::Relaxed); + tokio::time::sleep(Duration::from_millis(120)).await; + shutdown(tx, handle).await; + + let lines = read_json_lines(&path); + let dropped_line = lines + .iter() + .find(|line| { + line.get("event_type") == Some(&Value::String("system.audit_events_dropped".into())) + }) + .expect("system.audit_events_dropped line"); + assert_eq!(dropped_line["data"]["dropped_events"], 7); + assert_eq!(dropped_line["data"]["reason"], "queue_full_after_timeout"); + } +} diff --git a/src/node-control/service/src/audit/log.rs b/src/node-control/service/src/audit/log.rs index c1adc024..a95b4692 100644 --- a/src/node-control/service/src/audit/log.rs +++ b/src/node-control/service/src/audit/log.rs @@ -12,6 +12,12 @@ use async_trait::async_trait; #[async_trait] pub trait AuditLog: Send + Sync { async fn record(&self, event: AuditEvent); + + /// Drains pending events and flushes the final batch. + /// + /// Called from the composition-root shutdown sequence. Implementations that + /// hold no buffered state (e.g. [`NoopAuditLog`]) keep the default no-op. + async fn shutdown(&self) {} } pub struct NoopAuditLog; diff --git a/src/node-control/service/src/audit/mod.rs b/src/node-control/service/src/audit/mod.rs index 1be3627c..d457cbdd 100644 --- a/src/node-control/service/src/audit/mod.rs +++ b/src/node-control/service/src/audit/mod.rs @@ -9,6 +9,8 @@ pub mod enums; pub mod event; pub mod factory; +pub mod jsonl_log; +pub mod jsonl_writer; pub mod log; pub mod participant; @@ -18,6 +20,7 @@ pub use enums::{ StakeSkipReason, }; pub use event::AuditEvent; -pub use factory::{AuditInitError, AuditLogFactory}; +pub use factory::AuditLogFactory; +pub use jsonl_log::AuditInitError; pub use log::{AuditLog, NoopAuditLog}; pub use participant::{AuditActor, AuditSubject}; diff --git a/src/node-control/service/src/service_main_task.rs b/src/node-control/service/src/service_main_task.rs index 2d94147e..751d071b 100644 --- a/src/node-control/service/src/service_main_task.rs +++ b/src/node-control/service/src/service_main_task.rs @@ -168,5 +168,9 @@ pub async fn run_with_config( let _ = task.disable().await; } let _ = http_task_handle.await; + + // Drain and flush the audit log after all producers have stopped, so the + // final batch is persisted before the process exits. + audit.shutdown().await; Ok(()) } From 35417a5a807be57addf9c8be2c1db47dfed3d383 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Mon, 1 Jun 2026 13:52:30 +0300 Subject: [PATCH 06/30] fix :tests --- .../service/src/audit/jsonl_writer.rs | 522 ++++++++++-------- 1 file changed, 288 insertions(+), 234 deletions(-) diff --git a/src/node-control/service/src/audit/jsonl_writer.rs b/src/node-control/service/src/audit/jsonl_writer.rs index 7185ef57..72557463 100644 --- a/src/node-control/service/src/audit/jsonl_writer.rs +++ b/src/node-control/service/src/audit/jsonl_writer.rs @@ -260,6 +260,20 @@ mod tests { use std::path::{Path, PathBuf}; use tempfile::tempdir; + /// Writer tests perform real async file I/O. When the full `service` crate + /// runs in parallel (as in CI), contention on Tokio's blocking pool can + /// cause rare empty reads after an otherwise successful shutdown. + static WRITER_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + async fn run_writer_test(f: F) + where + F: FnOnce() -> Fut, + Fut: std::future::Future, + { + let _guard = WRITER_TEST_LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + f().await; + } + fn sample_event(tag: &str) -> AuditEvent { AuditEvent { schema_version: 1, @@ -339,288 +353,328 @@ mod tests { #[tokio::test] async fn writes_single_event_to_file() { - let dir = tempdir().unwrap(); - let cfg = test_config( - dir.path(), - AuditLogConfig { - batch_interval_ms: 60_000, - batch_max_events: 100, - ..AuditLogConfig::default() - }, - ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - send_event(&tx, sample_event("one")).await; - flush(&tx).await; - shutdown(tx, handle).await; + run_writer_test(|| async { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + batch_interval_ms: 60_000, + batch_max_events: 100, + ..AuditLogConfig::default() + }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + send_event(&tx, sample_event("one")).await; + flush(&tx).await; + shutdown(tx, handle).await; - let lines = read_json_lines(&path); - assert_eq!(lines.len(), 1); - assert_eq!(lines[0]["data"]["version"], "one"); + let lines = read_json_lines(&path); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0]["data"]["version"], "one"); + }) + .await; } #[tokio::test] async fn batches_events_within_interval() { - let dir = tempdir().unwrap(); - let cfg = test_config( - dir.path(), - AuditLogConfig { - batch_interval_ms: 50, - batch_max_events: 100, - ..AuditLogConfig::default() - }, - ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - for i in 0..5 { - send_event(&tx, sample_event(&format!("ev-{i}"))).await; - } - tokio::time::sleep(Duration::from_millis(120)).await; - shutdown(tx, handle).await; + run_writer_test(|| async { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + batch_interval_ms: 50, + batch_max_events: 100, + ..AuditLogConfig::default() + }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + for i in 0..5 { + send_event(&tx, sample_event(&format!("ev-{i}"))).await; + } + tokio::time::sleep(Duration::from_millis(120)).await; + shutdown(tx, handle).await; - let lines = read_json_lines(&path); - assert_eq!(lines.len(), 5); + let lines = read_json_lines(&path); + assert_eq!(lines.len(), 5); + }) + .await; } #[tokio::test] async fn rotates_at_max_size() { - let dir = tempdir().unwrap(); - let cfg = test_config( - dir.path(), - AuditLogConfig { - max_size_bytes: 1024, - batch_interval_ms: 60_000, - batch_max_events: 1, - ..AuditLogConfig::default() - }, - ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - for _ in 0..8 { - send_event(&tx, large_event(1)).await; - flush(&tx).await; - } - shutdown(tx, handle).await; + run_writer_test(|| async { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + max_size_bytes: 1024, + batch_interval_ms: 60_000, + batch_max_events: 1, + ..AuditLogConfig::default() + }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + for _ in 0..8 { + send_event(&tx, large_event(1)).await; + flush(&tx).await; + } + shutdown(tx, handle).await; - let rotated = path.with_extension("jsonl.1"); - assert!(rotated.exists(), "expected rotated file at {}", rotated.display()); + let rotated = path.with_extension("jsonl.1"); + assert!(rotated.exists(), "expected rotated file at {}", rotated.display()); + }) + .await; } #[tokio::test] async fn retains_only_max_files() { - let dir = tempdir().unwrap(); - let cfg = test_config( - dir.path(), - AuditLogConfig { - max_size_bytes: 512, - max_files: 3, - batch_interval_ms: 60_000, - batch_max_events: 1, - ..AuditLogConfig::default() - }, - ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - for _ in 0..12 { - send_event(&tx, large_event(1)).await; - flush(&tx).await; - } - shutdown(tx, handle).await; - - let rotated_count = count_rotated_files(dir.path()); - assert!( - rotated_count <= 2, - "expected at most max_files-1 rotated segments, got {rotated_count}" - ); - assert!(path.exists()); + run_writer_test(|| async { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + max_size_bytes: 512, + max_files: 3, + batch_interval_ms: 60_000, + batch_max_events: 1, + ..AuditLogConfig::default() + }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + for _ in 0..12 { + send_event(&tx, large_event(1)).await; + flush(&tx).await; + } + shutdown(tx, handle).await; + + let rotated_count = count_rotated_files(dir.path()); + assert!( + rotated_count <= 2, + "expected at most max_files-1 rotated segments, got {rotated_count}" + ); + assert!(path.exists()); + }) + .await; } #[tokio::test] async fn max_files_one_keeps_no_history() { - let dir = tempdir().unwrap(); - let cfg = test_config( - dir.path(), - AuditLogConfig { - max_size_bytes: 512, - max_files: 1, - batch_interval_ms: 60_000, - batch_max_events: 1, - ..AuditLogConfig::default() - }, - ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - for _ in 0..6 { - send_event(&tx, large_event(1)).await; - flush(&tx).await; - } - shutdown(tx, handle).await; - - assert!(path.exists(), "live audit.jsonl must exist"); - assert_eq!(count_rotated_files(dir.path()), 0, "max_files=1 must keep no rotated segments"); + run_writer_test(|| async { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + max_size_bytes: 512, + max_files: 1, + batch_interval_ms: 60_000, + batch_max_events: 1, + ..AuditLogConfig::default() + }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + for _ in 0..6 { + send_event(&tx, large_event(1)).await; + flush(&tx).await; + } + shutdown(tx, handle).await; + + assert!(path.exists(), "live audit.jsonl must exist"); + assert_eq!( + count_rotated_files(dir.path()), + 0, + "max_files=1 must keep no rotated segments" + ); + }) + .await; } #[tokio::test] async fn recovers_when_live_file_missing_on_restart() { - // Models the crash window between `rename(path -> .1)` and opening the - // new live file: on restart only the rotated segment exists. A fresh - // writer must recreate `audit.jsonl` via create(true) and write to it. - let dir = tempdir().unwrap(); - let cfg = test_config( - dir.path(), - AuditLogConfig { - batch_interval_ms: 60_000, - batch_max_events: 1, - ..AuditLogConfig::default() - }, - ); - let path = dir.path().join("audit.jsonl"); - - // First run persists one event, then we simulate the post-rename state: - // move the live file aside so no `audit.jsonl` exists. - let (tx, _dropped, handle, _) = spawn_writer(cfg.clone(), Duration::ZERO).await; - send_event(&tx, sample_event("first-run")).await; - flush(&tx).await; - shutdown(tx, handle).await; - std::fs::rename(&path, path.with_extension("jsonl.1")).unwrap(); - assert!(!path.exists(), "precondition: live file moved aside"); - - // Second run must recreate the live file and write into it. - let (tx, _dropped, handle, _) = spawn_writer(cfg, Duration::ZERO).await; - send_event(&tx, sample_event("after-restart")).await; - flush(&tx).await; - shutdown(tx, handle).await; - - assert!(path.exists(), "writer must recreate the live file on restart"); - let lines = read_json_lines(&path); - assert_eq!(lines.len(), 1); - assert_eq!(lines[0]["data"]["version"], "after-restart"); + run_writer_test(|| async { + // Models the crash window between `rename(path -> .1)` and opening the + // new live file: on restart only the rotated segment exists. A fresh + // writer must recreate `audit.jsonl` via create(true) and write to it. + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + batch_interval_ms: 60_000, + batch_max_events: 1, + ..AuditLogConfig::default() + }, + ); + let path = dir.path().join("audit.jsonl"); + + // First run persists one event, then we simulate the post-rename state: + // move the live file aside so no `audit.jsonl` exists. + let (tx, _dropped, handle, _) = spawn_writer(cfg.clone(), Duration::ZERO).await; + send_event(&tx, sample_event("first-run")).await; + flush(&tx).await; + shutdown(tx, handle).await; + std::fs::rename(&path, path.with_extension("jsonl.1")).unwrap(); + assert!(!path.exists(), "precondition: live file moved aside"); + + // Second run must recreate the live file and write into it. + let (tx, _dropped, handle, _) = spawn_writer(cfg, Duration::ZERO).await; + send_event(&tx, sample_event("after-restart")).await; + flush(&tx).await; + shutdown(tx, handle).await; + + assert!(path.exists(), "writer must recreate the live file on restart"); + let lines = read_json_lines(&path); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0]["data"]["version"], "after-restart"); + }) + .await; } #[tokio::test] async fn concurrent_writers_no_data_loss() { - let dir = tempdir().unwrap(); - let cfg = test_config( - dir.path(), - AuditLogConfig { - queue_capacity: 10_000, - batch_interval_ms: 20, - batch_max_events: 200, - max_size_bytes: 50 * 1024 * 1024, - ..AuditLogConfig::default() - }, - ); - let log = JsonlAuditLog::start(cfg).await.unwrap(); - - let mut tasks = Vec::new(); - for producer in 0..10 { - let log = log.clone(); - tasks.push(tokio::spawn(async move { - for seq in 0..100 { - log.record(sample_event(&format!("p{producer}-s{seq}"))).await; - } - })); - } - for task in tasks { - task.await.unwrap(); - } + run_writer_test(|| async { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + queue_capacity: 10_000, + batch_interval_ms: 20, + batch_max_events: 200, + max_size_bytes: 50 * 1024 * 1024, + ..AuditLogConfig::default() + }, + ); + let log = JsonlAuditLog::start(cfg).await.unwrap(); + + let mut tasks = Vec::new(); + for producer in 0..10 { + let log = log.clone(); + tasks.push(tokio::spawn(async move { + for seq in 0..100 { + log.record(sample_event(&format!("p{producer}-s{seq}"))).await; + } + })); + } + for task in tasks { + task.await.unwrap(); + } - tokio::time::sleep(Duration::from_millis(200)).await; + log.shutdown().await; - let path = dir.path().join("audit.jsonl"); - let lines = read_json_lines(&path); - assert_eq!(lines.len(), 1000, "expected 1000 audit lines, got {}", lines.len()); + let path = dir.path().join("audit.jsonl"); + let lines = read_json_lines(&path); + assert_eq!(lines.len(), 1000, "expected 1000 audit lines, got {}", lines.len()); + }) + .await; } #[tokio::test] async fn queue_full_drops_and_increments_counter() { - let dir = tempdir().unwrap(); - let cfg = test_config( - dir.path(), - AuditLogConfig { - queue_capacity: 1, - queue_full_timeout_ms: 10, - batch_interval_ms: 60_000, - batch_max_events: 10_000, - ..AuditLogConfig::default() - }, - ); - let log = - JsonlAuditLog::start_with_write_delay(cfg, Duration::from_millis(500)).await.unwrap(); + run_writer_test(|| async { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + queue_capacity: 1, + queue_full_timeout_ms: 10, + batch_interval_ms: 60_000, + batch_max_events: 1, + ..AuditLogConfig::default() + }, + ); + let log = JsonlAuditLog::start_with_write_delay(cfg, Duration::from_millis(500)) + .await + .unwrap(); - for i in 0..50 { - log.record(sample_event(&format!("drop-{i}"))).await; - } + for i in 0..50 { + log.record(sample_event(&format!("drop-{i}"))).await; + } - tokio::time::sleep(Duration::from_millis(100)).await; - assert!(log.dropped_events() > 0, "expected dropped events counter > 0"); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(log.dropped_events() > 0, "expected dropped events counter > 0"); + log.shutdown().await; + }) + .await; } #[tokio::test] async fn shutdown_flushes_pending() { - let dir = tempdir().unwrap(); - let cfg = test_config( - dir.path(), - AuditLogConfig { - batch_interval_ms: 60_000, - batch_max_events: 100, - ..AuditLogConfig::default() - }, - ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - for i in 0..3 { - send_event(&tx, sample_event(&format!("pending-{i}"))).await; - } - shutdown(tx, handle).await; + run_writer_test(|| async { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + batch_interval_ms: 60_000, + batch_max_events: 100, + ..AuditLogConfig::default() + }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + for i in 0..3 { + send_event(&tx, sample_event(&format!("pending-{i}"))).await; + } + shutdown(tx, handle).await; - let lines = read_json_lines(&path); - assert_eq!(lines.len(), 3); + let lines = read_json_lines(&path); + assert_eq!(lines.len(), 3); + }) + .await; } #[cfg(unix)] #[tokio::test] async fn file_mode_0600_on_unix() { - let dir = tempdir().unwrap(); - let cfg = test_config( - dir.path(), - AuditLogConfig { batch_interval_ms: 60_000, ..AuditLogConfig::default() }, - ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - send_event(&tx, sample_event("perm")).await; - flush(&tx).await; - shutdown(tx, handle).await; - - use std::os::unix::fs::PermissionsExt; - let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; - assert_eq!(mode, 0o600); + run_writer_test(|| async { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { batch_interval_ms: 60_000, ..AuditLogConfig::default() }, + ); + let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + send_event(&tx, sample_event("perm")).await; + flush(&tx).await; + shutdown(tx, handle).await; + + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + }) + .await; } #[tokio::test] async fn synthetic_dropped_event_emitted_after_drops() { - let dir = tempdir().unwrap(); - let cfg = test_config( - dir.path(), - AuditLogConfig { - batch_interval_ms: 50, - batch_max_events: 100, - ..AuditLogConfig::default() - }, - ); - let (tx, dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - dropped.store(7, Ordering::Relaxed); - tokio::time::sleep(Duration::from_millis(120)).await; - shutdown(tx, handle).await; - - let lines = read_json_lines(&path); - let dropped_line = lines - .iter() - .find(|line| { - line.get("event_type") == Some(&Value::String("system.audit_events_dropped".into())) - }) - .expect("system.audit_events_dropped line"); - assert_eq!(dropped_line["data"]["dropped_events"], 7); - assert_eq!(dropped_line["data"]["reason"], "queue_full_after_timeout"); + run_writer_test(|| async { + let dir = tempdir().unwrap(); + let cfg = test_config( + dir.path(), + AuditLogConfig { + batch_interval_ms: 50, + batch_max_events: 100, + ..AuditLogConfig::default() + }, + ); + let (tx, dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; + + dropped.store(7, Ordering::Relaxed); + tokio::time::sleep(Duration::from_millis(120)).await; + shutdown(tx, handle).await; + + let lines = read_json_lines(&path); + let dropped_line = lines + .iter() + .find(|line| { + line.get("event_type") + == Some(&Value::String("system.audit_events_dropped".into())) + }) + .expect("system.audit_events_dropped line"); + assert_eq!(dropped_line["data"]["dropped_events"], 7); + assert_eq!(dropped_line["data"]["reason"], "queue_full_after_timeout"); + }) + .await; } } From f0582e3c9303f8626d3ec082d08d7604710ca13f Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Mon, 1 Jun 2026 15:21:18 +0300 Subject: [PATCH 07/30] fix:test --- .../service/src/audit/jsonl_writer.rs | 209 +++++++++++------- 1 file changed, 129 insertions(+), 80 deletions(-) diff --git a/src/node-control/service/src/audit/jsonl_writer.rs b/src/node-control/service/src/audit/jsonl_writer.rs index 72557463..fa836b92 100644 --- a/src/node-control/service/src/audit/jsonl_writer.rs +++ b/src/node-control/service/src/audit/jsonl_writer.rs @@ -137,10 +137,10 @@ impl AuditWriter { let needed = line.len() + 1; if self.current_size + needed as u64 > self.config.max_size_bytes { if let Err(e) = self.write_batch_and_clear().await { - tracing::error!(error = %e, "audit write before rotation failed"); + Self::io_failed("audit write before rotation failed", e); } if let Err(e) = self.rotate().await { - tracing::error!(error = %e, "audit rotation failed"); + Self::io_failed("audit rotation failed", e); continue; } } @@ -150,7 +150,7 @@ impl AuditWriter { } if let Err(e) = self.write_batch_and_clear().await { - tracing::error!(error = %e, "audit batch write failed"); + Self::io_failed("audit batch write failed", e); } } @@ -170,10 +170,22 @@ impl AuditWriter { if self.config.fsync_on_batch { file.sync_data().await?; } + #[cfg(test)] + file.sync_all().await?; self.batch.clear(); Ok(()) } + #[cfg(test)] + fn io_failed(context: &str, err: std::io::Error) { + panic!("{context}: {err}"); + } + + #[cfg(not(test))] + fn io_failed(context: &str, err: std::io::Error) { + tracing::error!(error = %err, "{context}"); + } + async fn rotate(&mut self) -> std::io::Result<()> { let path = &self.config.path; // Total retained files (including the live one) is at least 1; the @@ -308,17 +320,32 @@ mod tests { cfg } - async fn spawn_writer( + async fn run_writer_session( config: AuditLogConfig, write_delay: Duration, - ) -> (mpsc::Sender, Arc, tokio::task::JoinHandle<()>, PathBuf) { + f: F, + ) -> (Arc, PathBuf) + where + F: FnOnce(mpsc::Sender, PathBuf, Arc) -> Fut, + Fut: std::future::Future, + { let config = Arc::new(config); let dropped = Arc::new(AtomicU64::new(0)); - let (tx, rx) = mpsc::channel(config.queue_capacity); - let writer = AuditWriter::open(config.clone(), dropped.clone(), write_delay).await.unwrap(); let path = config.path.clone(); - let handle = tokio::spawn(writer.run(rx)); - (tx, dropped, handle, path) + let (tx, rx) = mpsc::channel(config.queue_capacity); + let writer = AuditWriter::open(config, dropped.clone(), write_delay).await.unwrap(); + + // Drive the writer on the same task set as the producer (via `join!`) so + // shutdown/flush completes before assertions — no spawned-task scheduling + // races under CI parallel test load. + let dropped_out = dropped.clone(); + let path_out = path.clone(); + tokio::join!( + async move { writer.run(rx).await }, + async move { f(tx, path, dropped).await }, + ); + + (dropped_out, path_out) } async fn send_event(tx: &mpsc::Sender, event: AuditEvent) { @@ -329,13 +356,13 @@ mod tests { tx.send(AuditCommand::Flush).await.unwrap(); } - async fn shutdown(tx: mpsc::Sender, handle: tokio::task::JoinHandle<()>) { - let _ = tx.send(AuditCommand::Shutdown).await; - let _ = handle.await; + async fn stop(tx: &mpsc::Sender) { + tx.send(AuditCommand::Shutdown).await.unwrap(); } fn read_json_lines(path: &Path) -> Vec { - let content = std::fs::read_to_string(path).unwrap_or_default(); + assert!(path.exists(), "audit file missing at {}", path.display()); + let content = std::fs::read_to_string(path).unwrap(); content .lines() .filter(|line| !line.is_empty()) @@ -351,7 +378,7 @@ mod tests { .count() } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn writes_single_event_to_file() { run_writer_test(|| async { let dir = tempdir().unwrap(); @@ -363,11 +390,13 @@ mod tests { ..AuditLogConfig::default() }, ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - send_event(&tx, sample_event("one")).await; - flush(&tx).await; - shutdown(tx, handle).await; + let (_dropped, path) = + run_writer_session(cfg, Duration::ZERO, |tx, _path, _dropped| async move { + send_event(&tx, sample_event("one")).await; + flush(&tx).await; + stop(&tx).await; + }) + .await; let lines = read_json_lines(&path); assert_eq!(lines.len(), 1); @@ -376,7 +405,7 @@ mod tests { .await; } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn batches_events_within_interval() { run_writer_test(|| async { let dir = tempdir().unwrap(); @@ -388,13 +417,15 @@ mod tests { ..AuditLogConfig::default() }, ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - for i in 0..5 { - send_event(&tx, sample_event(&format!("ev-{i}"))).await; - } - tokio::time::sleep(Duration::from_millis(120)).await; - shutdown(tx, handle).await; + let (_dropped, path) = + run_writer_session(cfg, Duration::ZERO, |tx, _path, _dropped| async move { + for i in 0..5 { + send_event(&tx, sample_event(&format!("ev-{i}"))).await; + } + tokio::time::sleep(Duration::from_millis(120)).await; + stop(&tx).await; + }) + .await; let lines = read_json_lines(&path); assert_eq!(lines.len(), 5); @@ -402,7 +433,7 @@ mod tests { .await; } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn rotates_at_max_size() { run_writer_test(|| async { let dir = tempdir().unwrap(); @@ -415,13 +446,15 @@ mod tests { ..AuditLogConfig::default() }, ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - for _ in 0..8 { - send_event(&tx, large_event(1)).await; - flush(&tx).await; - } - shutdown(tx, handle).await; + let (_dropped, path) = + run_writer_session(cfg, Duration::ZERO, |tx, _path, _dropped| async move { + for _ in 0..8 { + send_event(&tx, large_event(1)).await; + flush(&tx).await; + } + stop(&tx).await; + }) + .await; let rotated = path.with_extension("jsonl.1"); assert!(rotated.exists(), "expected rotated file at {}", rotated.display()); @@ -429,7 +462,7 @@ mod tests { .await; } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn retains_only_max_files() { run_writer_test(|| async { let dir = tempdir().unwrap(); @@ -443,13 +476,15 @@ mod tests { ..AuditLogConfig::default() }, ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - for _ in 0..12 { - send_event(&tx, large_event(1)).await; - flush(&tx).await; - } - shutdown(tx, handle).await; + let (_dropped, path) = + run_writer_session(cfg, Duration::ZERO, |tx, _path, _dropped| async move { + for _ in 0..12 { + send_event(&tx, large_event(1)).await; + flush(&tx).await; + } + stop(&tx).await; + }) + .await; let rotated_count = count_rotated_files(dir.path()); assert!( @@ -461,7 +496,7 @@ mod tests { .await; } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn max_files_one_keeps_no_history() { run_writer_test(|| async { let dir = tempdir().unwrap(); @@ -475,13 +510,15 @@ mod tests { ..AuditLogConfig::default() }, ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - for _ in 0..6 { - send_event(&tx, large_event(1)).await; - flush(&tx).await; - } - shutdown(tx, handle).await; + let (_dropped, path) = + run_writer_session(cfg, Duration::ZERO, |tx, _path, _dropped| async move { + for _ in 0..6 { + send_event(&tx, large_event(1)).await; + flush(&tx).await; + } + stop(&tx).await; + }) + .await; assert!(path.exists(), "live audit.jsonl must exist"); assert_eq!( @@ -493,7 +530,7 @@ mod tests { .await; } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn recovers_when_live_file_missing_on_restart() { run_writer_test(|| async { // Models the crash window between `rename(path -> .1)` and opening the @@ -512,18 +549,23 @@ mod tests { // First run persists one event, then we simulate the post-rename state: // move the live file aside so no `audit.jsonl` exists. - let (tx, _dropped, handle, _) = spawn_writer(cfg.clone(), Duration::ZERO).await; - send_event(&tx, sample_event("first-run")).await; - flush(&tx).await; - shutdown(tx, handle).await; + run_writer_session(cfg.clone(), Duration::ZERO, |tx, _path, _dropped| async move { + send_event(&tx, sample_event("first-run")).await; + flush(&tx).await; + stop(&tx).await; + }) + .await; std::fs::rename(&path, path.with_extension("jsonl.1")).unwrap(); assert!(!path.exists(), "precondition: live file moved aside"); // Second run must recreate the live file and write into it. - let (tx, _dropped, handle, _) = spawn_writer(cfg, Duration::ZERO).await; - send_event(&tx, sample_event("after-restart")).await; - flush(&tx).await; - shutdown(tx, handle).await; + let (_dropped, _) = + run_writer_session(cfg, Duration::ZERO, |tx, _path, _dropped| async move { + send_event(&tx, sample_event("after-restart")).await; + flush(&tx).await; + stop(&tx).await; + }) + .await; assert!(path.exists(), "writer must recreate the live file on restart"); let lines = read_json_lines(&path); @@ -533,7 +575,7 @@ mod tests { .await; } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn concurrent_writers_no_data_loss() { run_writer_test(|| async { let dir = tempdir().unwrap(); @@ -571,7 +613,7 @@ mod tests { .await; } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn queue_full_drops_and_increments_counter() { run_writer_test(|| async { let dir = tempdir().unwrap(); @@ -600,7 +642,7 @@ mod tests { .await; } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn shutdown_flushes_pending() { run_writer_test(|| async { let dir = tempdir().unwrap(); @@ -612,12 +654,14 @@ mod tests { ..AuditLogConfig::default() }, ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - for i in 0..3 { - send_event(&tx, sample_event(&format!("pending-{i}"))).await; - } - shutdown(tx, handle).await; + let (_dropped, path) = + run_writer_session(cfg, Duration::ZERO, |tx, _path, _dropped| async move { + for i in 0..3 { + send_event(&tx, sample_event(&format!("pending-{i}"))).await; + } + stop(&tx).await; + }) + .await; let lines = read_json_lines(&path); assert_eq!(lines.len(), 3); @@ -626,7 +670,7 @@ mod tests { } #[cfg(unix)] - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn file_mode_0600_on_unix() { run_writer_test(|| async { let dir = tempdir().unwrap(); @@ -634,10 +678,13 @@ mod tests { dir.path(), AuditLogConfig { batch_interval_ms: 60_000, ..AuditLogConfig::default() }, ); - let (tx, _dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - send_event(&tx, sample_event("perm")).await; - flush(&tx).await; - shutdown(tx, handle).await; + let (_dropped, path) = + run_writer_session(cfg, Duration::ZERO, |tx, _path, _dropped| async move { + send_event(&tx, sample_event("perm")).await; + flush(&tx).await; + stop(&tx).await; + }) + .await; use std::os::unix::fs::PermissionsExt; let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; @@ -646,7 +693,7 @@ mod tests { .await; } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn synthetic_dropped_event_emitted_after_drops() { run_writer_test(|| async { let dir = tempdir().unwrap(); @@ -658,11 +705,13 @@ mod tests { ..AuditLogConfig::default() }, ); - let (tx, dropped, handle, path) = spawn_writer(cfg, Duration::ZERO).await; - - dropped.store(7, Ordering::Relaxed); - tokio::time::sleep(Duration::from_millis(120)).await; - shutdown(tx, handle).await; + let (_dropped, path) = + run_writer_session(cfg, Duration::ZERO, |tx, _path, dropped| async move { + dropped.store(7, Ordering::Relaxed); + tokio::time::sleep(Duration::from_millis(120)).await; + stop(&tx).await; + }) + .await; let lines = read_json_lines(&path); let dropped_line = lines From c3588fbde0e2cd8f1fdbbc3ecdac0cd9134ced38 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Mon, 1 Jun 2026 17:49:21 +0300 Subject: [PATCH 08/30] fix(version): thiserror = "2" --- src/Cargo.lock | 2 +- src/node-control/service/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index 5e97df97..ea8145f5 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -4943,7 +4943,7 @@ dependencies = [ "serde", "serde_json", "tempfile", - "thiserror 2.0.18", + "thiserror 1.0.69", "tokio", "ton-http-api-client", "ton_block", diff --git a/src/node-control/service/Cargo.toml b/src/node-control/service/Cargo.toml index b1ac0345..7f98fa7d 100644 --- a/src/node-control/service/Cargo.toml +++ b/src/node-control/service/Cargo.toml @@ -27,7 +27,7 @@ axum = "0.8" jsonwebtoken = "9" base64 = "0.22" chrono = { version = "0.4", features = ["serde"] } -thiserror = "2" +thiserror = "1" uuid = { version = "1", features = ["serde", "v4"] } [dev-dependencies] From bb77b090255c2f2dc443e55e25f926dd2650abc0 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Mon, 1 Jun 2026 18:50:00 +0300 Subject: [PATCH 09/30] fix: copilot fixes --- .../service/src/audit/jsonl_log.rs | 13 ++++--- .../service/src/audit/jsonl_writer.rs | 38 ++++++++++++++++--- src/node-control/service/src/audit/log.rs | 5 --- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/node-control/service/src/audit/jsonl_log.rs b/src/node-control/service/src/audit/jsonl_log.rs index e2163703..0074ac28 100644 --- a/src/node-control/service/src/audit/jsonl_log.rs +++ b/src/node-control/service/src/audit/jsonl_log.rs @@ -59,12 +59,15 @@ impl JsonlAuditLog { write_delay: Duration, ) -> Result, AuditInitError> { let config = Arc::new(config); - std::fs::create_dir_all(config.path.parent().ok_or_else(|| { - AuditInitError::InvalidPath(config.path.to_string_lossy().to_string()) - })?) - .map_err(AuditInitError::DirCreate)?; + if let Some(parent) = config.path.parent() { + if !parent.as_os_str().is_empty() { + tokio::fs::create_dir_all(parent).await.map_err(AuditInitError::DirCreate)?; + } + } else { + return Err(AuditInitError::InvalidPath(config.path.to_string_lossy().to_string())); + } - let (tx, rx) = mpsc::channel(config.queue_capacity); + let (tx, rx) = mpsc::channel(config.queue_capacity.max(1)); let dropped_events = Arc::new(AtomicU64::new(0)); let writer = AuditWriter::open(config.clone(), dropped_events.clone(), write_delay).await?; diff --git a/src/node-control/service/src/audit/jsonl_writer.rs b/src/node-control/service/src/audit/jsonl_writer.rs index fa836b92..50470ace 100644 --- a/src/node-control/service/src/audit/jsonl_writer.rs +++ b/src/node-control/service/src/audit/jsonl_writer.rs @@ -29,11 +29,12 @@ use uuid::Uuid; pub(crate) enum AuditCommand { Event(Box), + Shutdown, + /// Forces an immediate flush of buffered events. Used only by tests to make /// assertions deterministic without waiting for the batch interval. #[cfg(test)] Flush, - Shutdown, } pub(crate) struct AuditWriter { @@ -86,8 +87,8 @@ impl AuditWriter { } pub(crate) async fn run(mut self, mut rx: mpsc::Receiver) { - let mut interval = - tokio::time::interval(Duration::from_millis(self.config.batch_interval_ms)); + let interval_ms = self.config.batch_interval_ms.max(1); + let mut interval = tokio::time::interval(Duration::from_millis(interval_ms)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); let mut buffered: Vec = Vec::with_capacity(self.config.batch_max_events); @@ -101,13 +102,13 @@ impl AuditWriter { self.flush(&mut buffered).await; } } - #[cfg(test)] - Some(AuditCommand::Flush) => self.flush(&mut buffered).await, Some(AuditCommand::Shutdown) | None => { self.flush(&mut buffered).await; self.maybe_emit_dropped_recovery().await; return; } + #[cfg(test)] + Some(AuditCommand::Flush) => self.flush(&mut buffered).await, } } _ = interval.tick() => { @@ -186,8 +187,19 @@ impl AuditWriter { tracing::error!(error = %err, "{context}"); } + async fn reopen_live_append(&mut self, path: &std::path::Path) -> std::io::Result<()> { + let mut opts = tokio::fs::OpenOptions::new(); + opts.append(true).create(true); + #[cfg(unix)] + opts.mode(0o600); + let file = opts.open(path).await?; + self.current_size = file.metadata().await?.len(); + self.file = Some(file); + Ok(()) + } + async fn rotate(&mut self) -> std::io::Result<()> { - let path = &self.config.path; + let path = self.config.path.clone(); // Total retained files (including the live one) is at least 1; the // number of rotated history segments is `max - 1`. Guarding against 0 // avoids an arithmetic underflow on `max - 1`. @@ -214,6 +226,20 @@ impl AuditWriter { // is valid on platforms that forbid renaming an open file. self.file = None; + if let Err(e) = self.rotate_inner(&path, max).await { + // Best-effort: reopen the live file so subsequent writes can continue + // instead of failing forever with a `None` handle until restart. + if let Err(reopen_err) = self.reopen_live_append(&path).await { + Self::io_failed("audit reopen after rotation failure failed", reopen_err); + } + return Err(e); + } + Ok(()) + } + + /// Performs the on-disk swap and opens a fresh live file. The caller is + /// responsible for recovering the handle if this returns an error. + async fn rotate_inner(&mut self, path: &std::path::Path, max: usize) -> std::io::Result<()> { let mut opts = tokio::fs::OpenOptions::new(); if max > 1 { // Preserve history: rename current -> .1, then open a fresh live file. diff --git a/src/node-control/service/src/audit/log.rs b/src/node-control/service/src/audit/log.rs index a95b4692..004caf97 100644 --- a/src/node-control/service/src/audit/log.rs +++ b/src/node-control/service/src/audit/log.rs @@ -12,11 +12,6 @@ use async_trait::async_trait; #[async_trait] pub trait AuditLog: Send + Sync { async fn record(&self, event: AuditEvent); - - /// Drains pending events and flushes the final batch. - /// - /// Called from the composition-root shutdown sequence. Implementations that - /// hold no buffered state (e.g. [`NoopAuditLog`]) keep the default no-op. async fn shutdown(&self) {} } From 9cf4d0e82a85d2b1e87069039ac91f0202b2d480 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Wed, 3 Jun 2026 10:41:49 +0300 Subject: [PATCH 10/30] feat(nodectl): emit elections audit events from ElectionRunner (SMA-103) --- src/node-control/service/src/audit/enums.rs | 16 +- src/node-control/service/src/audit/event.rs | 8 + .../service/src/audit/in_memory.rs | 82 ++++ src/node-control/service/src/audit/mod.rs | 4 + .../src/elections/adaptive_strategy.rs | 37 ++ .../service/src/elections/runner.rs | 419 +++++++++++++++++- .../service/src/elections/runner_tests.rs | 334 +++++++++++++- 7 files changed, 870 insertions(+), 30 deletions(-) create mode 100644 src/node-control/service/src/audit/in_memory.rs diff --git a/src/node-control/service/src/audit/enums.rs b/src/node-control/service/src/audit/enums.rs index 71ccf94e..50507f26 100644 --- a/src/node-control/service/src/audit/enums.rs +++ b/src/node-control/service/src/audit/enums.rs @@ -12,6 +12,13 @@ use serde::{Deserialize, Serialize}; #[serde(tag = "event_type", content = "data", rename_all = "snake_case")] #[non_exhaustive] pub enum AuditEventPayload { + #[serde(rename = "elections.tick_failed")] + ElectionsTickFailed { + #[serde(skip_serializing_if = "Option::is_none")] + election_id: Option, + error: String, + }, + #[serde(rename = "elections.key_generated")] ElectionsKeyGenerated { election_id: u64, @@ -44,6 +51,9 @@ pub enum AuditEventPayload { #[serde(rename = "elections.withdraw_processed")] ElectionsWithdrawProcessed { election_id: u64, tx_hash: String }, + #[serde(rename = "elections.withdraw_process_failed")] + ElectionsWithdrawProcessFailed { election_id: u64, error: String }, + #[serde(rename = "elections.stake_recovered")] ElectionsStakeRecovered { election_id: u64, @@ -84,7 +94,11 @@ pub enum StakeSkipReason { LowWalletBalance, WithdrawRequestsPending, PoolNotReady, - Other, + AdaptiveSleepPeriod, + AdaptiveWaitingPeriod, + NodeExcluded, + RecoverPending, + InsufficientStakeFunds, } #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index 4ea0b126..1b2f1954 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -260,6 +260,10 @@ mod tests { fn all_payload_variants() -> Vec { const ELECTION_ID: u64 = 1_779_265_552; vec![ + AuditEventPayload::ElectionsTickFailed { + election_id: Some(ELECTION_ID), + error: "tick error".into(), + }, AuditEventPayload::ElectionsKeyGenerated { election_id: ELECTION_ID, pubkey: Some("aabb".into()), @@ -285,6 +289,10 @@ mod tests { election_id: ELECTION_ID, tx_hash: "abc".into(), }, + AuditEventPayload::ElectionsWithdrawProcessFailed { + election_id: ELECTION_ID, + error: "send failed".into(), + }, AuditEventPayload::ElectionsStakeRecovered { election_id: ELECTION_ID, amount_nanotons: "50000000000000".into(), diff --git a/src/node-control/service/src/audit/in_memory.rs b/src/node-control/service/src/audit/in_memory.rs new file mode 100644 index 00000000..0d137aa3 --- /dev/null +++ b/src/node-control/service/src/audit/in_memory.rs @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use crate::audit::{AuditEvent, log::AuditLog}; +use async_trait::async_trait; +use std::sync::Mutex; + +/// Captures audit events in memory for unit tests. +pub struct InMemoryAuditLog { + pub events: Mutex>, +} + +impl InMemoryAuditLog { + pub fn new() -> Self { + Self { events: Mutex::new(Vec::new()) } + } + + pub fn drain(&self) -> Vec { + std::mem::take(&mut *self.events.lock().expect("in-memory audit lock")) + } +} + +impl Default for InMemoryAuditLog { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl AuditLog for InMemoryAuditLog { + async fn record(&self, event: AuditEvent) { + self.events.lock().expect("in-memory audit lock").push(event); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::audit::{ + AuditEvent, + enums::{ + AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, + AuditSubjectKind, + }, + participant::{AuditActor, AuditSubject}, + }; + use chrono::Utc; + use std::collections::BTreeMap; + use uuid::Uuid; + + #[tokio::test] + async fn records_and_drains_events() { + let log = InMemoryAuditLog::new(); + let event = AuditEvent { + schema_version: 1, + id: Uuid::new_v4(), + ts: Utc::now(), + source: AuditSource::Elections, + severity: AuditSeverity::Info, + outcome: AuditOutcome::Success, + actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, + subject: AuditSubject { + kind: AuditSubjectKind::Node, + id: Some("n1".into()), + election_id: None, + labels: BTreeMap::new(), + }, + message: None, + payload: AuditEventPayload::SystemServiceStarted { version: "test".into() }, + }; + log.record(event.clone()).await; + let drained = log.drain(); + assert_eq!(drained.len(), 1); + assert_eq!(drained[0].id, event.id); + assert!(log.drain().is_empty()); + } +} diff --git a/src/node-control/service/src/audit/mod.rs b/src/node-control/service/src/audit/mod.rs index d457cbdd..26a622a2 100644 --- a/src/node-control/service/src/audit/mod.rs +++ b/src/node-control/service/src/audit/mod.rs @@ -9,6 +9,8 @@ pub mod enums; pub mod event; pub mod factory; +#[cfg(test)] +pub mod in_memory; pub mod jsonl_log; pub mod jsonl_writer; pub mod log; @@ -21,6 +23,8 @@ pub use enums::{ }; pub use event::AuditEvent; pub use factory::AuditLogFactory; +#[cfg(test)] +pub use in_memory::InMemoryAuditLog; pub use jsonl_log::AuditInitError; pub use log::{AuditLog, NoopAuditLog}; pub use participant::{AuditActor, AuditSubject}; diff --git a/src/node-control/service/src/elections/adaptive_strategy.rs b/src/node-control/service/src/elections/adaptive_strategy.rs index 9572d8a7..34db8669 100644 --- a/src/node-control/service/src/elections/adaptive_strategy.rs +++ b/src/node-control/service/src/elections/adaptive_strategy.rs @@ -15,6 +15,13 @@ use ton_block::config_params::{ConfigParam16, ConfigParam17}; /// AdaptiveSplit50 wait logic: check whether enough time has passed and enough /// participants have joined before proceeding with stake calculation. /// +/// Why AdaptiveSplit50 defers staking for this tick (`calc_stake` returns 0). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AdaptiveDeferReason { + SleepPeriod, + WaitingForParticipants, +} + /// Returns `true` if staking should proceed, `false` if we should defer (return 0). pub(crate) fn is_adaptive_split50_ready( node_id: &str, @@ -67,6 +74,36 @@ pub(crate) fn is_adaptive_split50_ready( true } +/// When [`is_adaptive_split50_ready`] would return `false`, reports which wait gate blocked. +pub(crate) fn adaptive_split50_defer_reason( + elections_info: &ElectionsInfo, + cfg15_start_before: u32, + cfg15_end_before: u32, + cfg16: &ConfigParam16, + sleep_pct: f64, + waiting_pct: f64, +) -> Option { + let min_validators = cfg16.min_validators.as_u16() as usize; + let participants_count = elections_info.participants.len(); + let election_duration = cfg15_start_before.saturating_sub(cfg15_end_before) as u64; + if election_duration == 0 { + return None; + } + + let election_start = elections_info.elect_close.saturating_sub(election_duration); + let sleep_deadline = election_start + (election_duration as f64 * sleep_pct) as u64; + let wait_deadline = election_start + (election_duration as f64 * waiting_pct) as u64; + let now = common::time_format::now(); + + if now < sleep_deadline { + return Some(AdaptiveDeferReason::SleepPeriod); + } + if participants_count < min_validators && now < wait_deadline { + return Some(AdaptiveDeferReason::WaitingForParticipants); + } + None +} + /// Calculate stake for AdaptiveSplit50 policy. /// /// Determines min_eff_stake from current emulation and/or past elections, diff --git a/src/node-control/service/src/elections/runner.rs b/src/node-control/service/src/elections/runner.rs index 174f4203..792880a1 100644 --- a/src/node-control/service/src/elections/runner.rs +++ b/src/node-control/service/src/elections/runner.rs @@ -11,8 +11,17 @@ use super::{ election_emulator::ParticipantStake, providers::{ElectionsProvider, ValidatorConfig, ValidatorEntry}, }; -use crate::audit::log::AuditLog; +use crate::audit::{ + AuditEvent, StakeSkipReason, + enums::{ + AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, + AuditSubjectKind, + }, + log::AuditLog, + participant::{AuditActor, AuditSubject}, +}; use anyhow::Context as _; +use chrono::Utc; use common::{ app_config::{BindingStatus, ElectionsConfig, NodeBinding, StakePolicy}, clock::{Clock, SystemClock}, @@ -33,7 +42,7 @@ use contracts::{ elector::PastElections, nominator, }; use std::{ - collections::{HashMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, sync::Arc, time::Duration, }; @@ -42,6 +51,7 @@ use ton_block::{ config_params::{ConfigParam16, ConfigParam17}, write_boc, }; +use uuid::Uuid; #[cfg(test)] #[path = "runner_tests.rs"] @@ -270,8 +280,6 @@ pub(crate) struct ElectionRunner { /// Callback to persist freshly generated static ADNL addresses into runtime config. /// `None` in tests that don't care about persistence. persist_static_adnls: Option, - /// Reserved for elections audit events (SMA-99.4 call sites). - #[allow(dead_code)] audit: Arc, clock: Arc, } @@ -360,6 +368,82 @@ impl ElectionRunner { } } + async fn audit_emit( + &self, + severity: AuditSeverity, + outcome: AuditOutcome, + election_id: Option, + node_id: Option, + message: &str, + payload: AuditEventPayload, + ) { + elections_audit_emit( + &self.audit, + severity, + outcome, + election_id, + node_id, + message, + payload, + ) + .await; + } + + async fn audit_stake_skipped( + &self, + election_id: u64, + node_id: &str, + reason: StakeSkipReason, + required_nanotons: Option, + available_nanotons: Option, + ) { + elections_audit_stake_skipped( + &self.audit, + election_id, + node_id, + reason, + required_nanotons, + available_nanotons, + ) + .await; + } + + async fn classify_stake_zero( + node: &mut Node, + elections_stake: u64, + configs: &ConfigParams<'_>, + ctx: &StakeContext<'_>, + ) -> (StakeSkipReason, Option, Option) { + if matches!(&node.stake_policy, StakePolicy::AdaptiveSplit50) { + if let Some(defer) = adaptive_strategy::adaptive_split50_defer_reason( + configs.elections_info, + configs.cfg15.elections_start_before, + configs.cfg15.elections_end_before, + configs.cfg16, + ctx.sleep_pct, + ctx.waiting_pct, + ) { + return match defer { + adaptive_strategy::AdaptiveDeferReason::SleepPeriod => { + (StakeSkipReason::AdaptiveSleepPeriod, None, None) + } + adaptive_strategy::AdaptiveDeferReason::WaitingForParticipants => { + (StakeSkipReason::AdaptiveWaitingPeriod, None, None) + } + }; + } + let min_stake = configs.elections_info.min_stake; + let fee = ELECTOR_STAKE_FEE + NPOOL_COMPUTE_FEE; + let pool_free = node.stake_balance(fee).await.unwrap_or(0); + return ( + StakeSkipReason::InsufficientStakeFunds, + Some(min_stake), + Some(pool_free.saturating_add(elections_stake)), + ); + } + (StakeSkipReason::InsufficientStakeFunds, None, None) + } + pub(crate) fn new( elections_config: &ElectionsConfig, bindings: &HashMap, @@ -457,6 +541,9 @@ impl ElectionRunner { tokio::select! { _ = interval.tick() => { tracing::info!("TICK"); + let audit = self.audit.clone(); + let tick_election_id = + self.elector.get_active_election_id().await.ok().filter(|&id| id > 0); // Clear per-node last_error at the start of the tick (best-effort). for node in self.nodes.values_mut() { @@ -467,7 +554,24 @@ impl ElectionRunner { self.refresh_next_validator_set().await; self.refresh_validator_configs().await; - if let Err(e) = &self.run().await { + if let Err(e) = self.run().await { + let election_id = tick_election_id.or({ + let id = self.past_elections_cache_id; + (id > 0).then_some(id) + }); + elections_audit_emit( + &audit, + AuditSeverity::Error, + AuditOutcome::Failure, + election_id, + None, + "Election tick failed", + AuditEventPayload::ElectionsTickFailed { + election_id, + error: format!("{e:#}"), + }, + ) + .await; tracing::error!("runner tick error: {:#}", e); } @@ -640,10 +744,20 @@ impl ElectionRunner { .filter(|id| !skip_tick_nodes.contains(id)) .collect::>(); nodes.sort(); + for node_id in &skip_tick_nodes { + self.audit_stake_skipped( + election_id, + node_id, + StakeSkipReason::PoolNotReady, + None, + None, + ) + .await; + } for node_id in nodes { tracing::info!("node [{}] recover stake", node_id); let excluded = self.nodes.get(&node_id).map(|node| node.excluded).unwrap_or(true); - let recover_amount = match self.recover_stake(&node_id).await { + let recover_amount = match self.recover_stake(&node_id, election_id).await { Ok(amount) => amount, Err(e) => { if let Some(node) = self.nodes.get_mut(&node_id) { @@ -663,6 +777,25 @@ impl ElectionRunner { excluded, recover_amount as f64 / 1_000_000_000.0 ); + if excluded { + self.audit_stake_skipped( + election_id, + &node_id, + StakeSkipReason::NodeExcluded, + None, + None, + ) + .await; + } else { + self.audit_stake_skipped( + election_id, + &node_id, + StakeSkipReason::RecoverPending, + Some(recover_amount), + None, + ) + .await; + } continue; } @@ -676,6 +809,14 @@ impl ElectionRunner { "node [{}] skip participate this tick: withdraw requests sent, awaiting pool drain", node_id ); + self.audit_stake_skipped( + election_id, + &node_id, + StakeSkipReason::WithdrawRequestsPending, + None, + None, + ) + .await; continue; } Ok(false) => {} @@ -777,6 +918,7 @@ impl ElectionRunner { election_id: u64, params: &ConfigParams<'_>, ) -> anyhow::Result<()> { + let audit = self.audit.clone(); let configured_raw = self.configured_max_factor_raw(); let (max_factor, _) = self.calc_max_factor(params.cfg17.max_stake_factor); if max_factor != configured_raw { @@ -874,12 +1016,37 @@ impl ElectionRunner { node.accepted_stake_amount = Some(participant.stake); } let elections_stake = participant.as_ref().map(|p| p.stake).unwrap_or(0); - let stake = Self::calc_stake(node, node_id, elections_stake, params, &stake_ctx) - .await - .context("stake calculation error")?; + let stake = match Self::calc_stake(node, node_id, elections_stake, params, &stake_ctx).await + { + Ok(stake) => stake, + Err(e) => { + let (required, available) = parse_balance_error(&e.to_string()); + elections_audit_stake_skipped( + &audit, + election_id, + node_id, + StakeSkipReason::LowWalletBalance, + required, + available, + ) + .await; + return Err(e).context("stake calculation error"); + } + }; if stake == 0 { tracing::info!("node [{}] skipping elections this tick (stake=0)", node_id); + let (reason, required, available) = + Self::classify_stake_zero(node, elections_stake, params, &stake_ctx).await; + elections_audit_stake_skipped( + &audit, + election_id, + node_id, + reason, + required, + available, + ) + .await; return Ok(()); } @@ -927,6 +1094,7 @@ impl ElectionRunner { adnl_addr.as_slice() ) ); + let pubkey_hex = hex::encode(pub_key.as_slice()); node.participant = Some(Participant { stake_message_boc: None, pub_key, @@ -937,7 +1105,33 @@ impl ElectionRunner { max_factor, }); node.key_id = key_id; - Self::send_stake(node_id, node, stake, to_addr).await?; + elections_audit_emit( + &audit, + AuditSeverity::Info, + AuditOutcome::Success, + Some(election_id), + Some(node_id.to_string()), + "Validator key generated", + AuditEventPayload::ElectionsKeyGenerated { + election_id, + pubkey: Some(pubkey_hex), + }, + ) + .await; + if let Err(e) = Self::send_stake(node_id, node, stake, to_addr).await { + let (required, available) = parse_balance_error(&e.to_string()); + elections_audit_stake_skipped( + &audit, + election_id, + node_id, + StakeSkipReason::LowWalletBalance, + required, + available, + ) + .await; + return Err(e); + } + elections_audit_stake_submitted(&audit, node_id, node, stake).await; Ok(()) } Some(entry) => { @@ -963,8 +1157,21 @@ impl ElectionRunner { nanotons_to_tons_f64(old_stake + stake), nanotons_to_tons_f64(stake), ); - Self::send_stake(node_id, node, stake, to_addr).await?; + if let Err(e) = Self::send_stake(node_id, node, stake, to_addr).await { + let (required, available) = parse_balance_error(&e.to_string()); + elections_audit_stake_skipped( + &audit, + election_id, + node_id, + StakeSkipReason::LowWalletBalance, + required, + available, + ) + .await; + return Err(e); + } node.participant.as_mut().map(|p| p.stake += stake); + elections_audit_stake_submitted(&audit, node_id, node, stake).await; } } None => { @@ -972,7 +1179,20 @@ impl ElectionRunner { if let Some(p) = node.participant.as_mut() { p.stake = stake; } - Self::send_stake(node_id, node, stake, to_addr).await?; + if let Err(e) = Self::send_stake(node_id, node, stake, to_addr).await { + let (required, available) = parse_balance_error(&e.to_string()); + elections_audit_stake_skipped( + &audit, + election_id, + node_id, + StakeSkipReason::LowWalletBalance, + required, + available, + ) + .await; + return Err(e); + } + elections_audit_stake_submitted(&audit, node_id, node, stake).await; } } Ok(()) @@ -1135,7 +1355,27 @@ impl ElectionRunner { .await .context("build process_withdraw_requests message")?; let msg_boc = write_boc(&msg).context("encode process_withdraw_requests boc")?; - node.api.send_boc(&msg_boc).await.context("send process_withdraw_requests boc")?; + let tx_hash = format!("{:x}", msg.repr_hash()); + let send_result = { + let node = self.nodes.get_mut(node_id).expect("node not found"); + node.api.send_boc(&msg_boc).await.context("send process_withdraw_requests boc") + }; + + if let Err(e) = send_result { + self.audit_emit( + AuditSeverity::Error, + AuditOutcome::Failure, + Some(election_id), + Some(node_id.to_string()), + "Withdraw process failed", + AuditEventPayload::ElectionsWithdrawProcessFailed { + election_id, + error: format!("{e:#}"), + }, + ) + .await; + return Err(e); + } tracing::info!( "node [{}] process_withdraw_requests sent (limit={}, election_id={})", @@ -1143,10 +1383,19 @@ impl ElectionRunner { WITHDRAW_PROCESS_LIMIT, election_id ); + self.audit_emit( + AuditSeverity::Info, + AuditOutcome::Success, + Some(election_id), + Some(node_id.to_string()), + "Withdraw requests processed", + AuditEventPayload::ElectionsWithdrawProcessed { election_id, tx_hash }, + ) + .await; Ok(true) } - async fn recover_stake(&mut self, node_id: &str) -> anyhow::Result { + async fn recover_stake(&mut self, node_id: &str, election_id: u64) -> anyhow::Result { let node = self.nodes.get_mut(node_id).expect("node not found"); let amount = self.elector.compute_returned_stake(&node.stake_addr().await?).await?; @@ -1171,13 +1420,26 @@ impl ElectionRunner { // pool_target() errors if pool is set but its address is not cached yet — avoids // routing recover stake to the elector when the pool is actually configured. let to_addr = node.pool_target()?.cloned().unwrap_or(elector_addr); - let msg_boc = write_boc( - &node - .wallet - .message(to_addr, RECOVER_FEE, Self::build_recover_stake_payload().await?) - .await?, - )?; + let msg = node + .wallet + .message(to_addr, RECOVER_FEE, Self::build_recover_stake_payload().await?) + .await?; + let tx_hash = format!("{:x}", msg.repr_hash()); + let msg_boc = write_boc(&msg)?; node.api.send_boc(&msg_boc).await?; + self.audit_emit( + AuditSeverity::Info, + AuditOutcome::Success, + Some(election_id), + Some(node_id.to_string()), + "Stake recovered", + AuditEventPayload::ElectionsStakeRecovered { + election_id, + amount_nanotons: nanotons_to_dec_string(amount), + tx_hash: Some(tx_hash), + }, + ) + .await; } Ok(amount) } @@ -1303,7 +1565,7 @@ impl ElectionRunner { ); if total_balance < min_stake { anyhow::bail!( - "not enough funds: available={} TON, min_stake={} TON", + "not enough funds: available={} TON, required={} TON", total_balance as f64 / 1_000_000_000.0, min_stake as f64 / 1_000_000_000.0 ); @@ -1815,6 +2077,121 @@ impl ElectionRunner { } } +fn elections_audit_actor() -> AuditActor { + AuditActor { + kind: AuditActorKind::Service, + id: Some("elections-task".into()), + role: None, + ip: None, + } +} + +async fn elections_audit_emit( + audit: &Arc, + severity: AuditSeverity, + outcome: AuditOutcome, + election_id: Option, + node_id: Option, + message: &str, + payload: AuditEventPayload, +) { + audit + .record(AuditEvent { + schema_version: 1, + id: Uuid::new_v4(), + ts: Utc::now(), + source: AuditSource::Elections, + severity, + outcome, + actor: elections_audit_actor(), + subject: AuditSubject { + kind: if node_id.is_some() { + AuditSubjectKind::Node + } else { + AuditSubjectKind::Elections + }, + id: node_id, + election_id, + labels: BTreeMap::new(), + }, + message: Some(message.into()), + payload, + }) + .await; +} + +async fn elections_audit_stake_skipped( + audit: &Arc, + election_id: u64, + node_id: &str, + reason: StakeSkipReason, + required_nanotons: Option, + available_nanotons: Option, +) { + elections_audit_emit( + audit, + AuditSeverity::Warn, + AuditOutcome::Skipped, + Some(election_id), + Some(node_id.to_string()), + "Stake skipped", + AuditEventPayload::ElectionsStakeSkipped { + election_id, + reason, + required_nanotons: required_nanotons.map(nanotons_to_dec_string), + available_nanotons: available_nanotons.map(nanotons_to_dec_string), + }, + ) + .await; +} + +async fn elections_audit_stake_submitted( + audit: &Arc, + node_id: &str, + node: &Node, + stake: u64, +) { + let Some(participant) = node.participant.as_ref() else { + return; + }; + let submission_time = node.submission_time.unwrap_or_else(time_format::now); + elections_audit_emit( + audit, + AuditSeverity::Info, + AuditOutcome::Success, + Some(participant.election_id), + Some(node_id.to_string()), + "Stake submitted", + AuditEventPayload::ElectionsStakeSubmitted { + election_id: participant.election_id, + stake_nanotons: nanotons_to_dec_string(stake), + max_factor: participant.max_factor, + policy: node.stake_policy.to_string(), + submission_time, + }, + ) + .await; +} + +/// Parses `required=` / `available=` TON amounts from runner balance error strings. +fn parse_balance_error(msg: &str) -> (Option, Option) { + fn tons_to_nanotons(fragment: &str) -> Option { + let tons: f64 = fragment.trim().parse().ok()?; + Some((tons * 1_000_000_000.0) as u64) + } + let required = msg + .split("required=") + .nth(1) + .and_then(|s| s.split(" TON").next()) + .and_then(tons_to_nanotons); + let available = msg + .split("available=") + .nth(1) + .and_then(|s| s.split(" TON").next()) + .and_then(tons_to_nanotons); + (required, available) +} + async fn find_validator_entries( node: &mut Node, current_vset: Option<&ValidatorSet>, diff --git a/src/node-control/service/src/elections/runner_tests.rs b/src/node-control/service/src/elections/runner_tests.rs index b60d59f7..27dcd823 100644 --- a/src/node-control/service/src/elections/runner_tests.rs +++ b/src/node-control/service/src/elections/runner_tests.rs @@ -7,7 +7,13 @@ * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ use super::*; -use crate::audit::log::{AuditLog, NoopAuditLog}; +use crate::audit::{ + AuditEvent, InMemoryAuditLog, + enums::{ + AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, AuditSubjectKind, + StakeSkipReason, + }, +}; use common::{ app_config::{ElectionsConfig, NodeBinding, StakePolicy}, clock::MockClock, @@ -364,14 +370,34 @@ fn validate_message_parameters( // ---- Builder helpers ---- -fn noop_audit() -> Arc { - Arc::new(NoopAuditLog) -} - fn default_binding(enable: bool) -> NodeBinding { NodeBinding { wallet: "wallet".to_string(), pool: None, enable, status: Default::default() } } +fn find_audit_event<'a, F>(events: &'a [AuditEvent], pred: F) -> &'a AuditEvent +where + F: Fn(&AuditEventPayload) -> bool, +{ + events + .iter() + .find(|ev| pred(&ev.payload)) + .unwrap_or_else(|| panic!("expected audit event not found: {events:#?}")) +} + +fn payload_stake_submitted(payload: &AuditEventPayload) -> &AuditEventPayload { + match payload { + AuditEventPayload::ElectionsStakeSubmitted { .. } => payload, + other => panic!("expected ElectionsStakeSubmitted, got {other:?}"), + } +} + +fn payload_stake_skipped(payload: &AuditEventPayload) -> &AuditEventPayload { + match payload { + AuditEventPayload::ElectionsStakeSkipped { .. } => payload, + other => panic!("expected ElectionsStakeSkipped, got {other:?}"), + } +} + struct TestHarness { elector_mock: MockElectorWrapperImpl, provider_mock: MockElectionsProviderImpl, @@ -380,6 +406,7 @@ struct TestHarness { toncore_nominator_mocks: Option<(MockSingleNominatorWrapper, MockSingleNominatorWrapper)>, elections_config: ElectionsConfig, bindings: HashMap, + pub audit: Arc, /// Captures static ADNL addresses persisted by `ensure_static_adnls`. When set, /// `build()` installs a callback that writes generated entries here. persisted_static_adnls: Option>>>, @@ -393,6 +420,7 @@ impl TestHarness { wallet_mock: MockTonWalletImpl::new(), pool_mock: None, toncore_nominator_mocks: None, + audit: Arc::new(InMemoryAuditLog::new()), elections_config: ElectionsConfig { policy: StakePolicy::Split50, policy_overrides: HashMap::new(), @@ -480,7 +508,7 @@ impl TestHarness { Arc::new(wallets), Arc::new(pools), persist, - noop_audit(), + self.audit.clone(), ) } } @@ -1406,7 +1434,7 @@ async fn test_multiple_nodes_one_excluded() { Arc::new(wallets), Arc::new(pools), None, - noop_audit(), + Arc::new(InMemoryAuditLog::new()), ); let result = runner.run().await; @@ -1809,7 +1837,7 @@ async fn test_node_without_wallet_skipped() { Arc::new(wallets), Arc::new(pools), None, - noop_audit(), + Arc::new(InMemoryAuditLog::new()), ); assert!( @@ -3777,3 +3805,293 @@ async fn cached_prev_min_eff_updates_on_refresh() { "cached_prev_min_eff must reflect the refreshed past_elections snapshot" ); } + +#[tokio::test] +async fn tick_success_does_not_emit_tick_audit_events() { + let node_id = "node-1"; + let mut harness = TestHarness::new(); + + setup_default_elector(&mut harness.elector_mock, ELECTION_ID, 0); + setup_default_provider(&mut harness.provider_mock, WALLET_BALANCE, None); + setup_wallet(&mut harness.wallet_mock); + harness.wallet_mock.expect_message().returning(|_dest, _value, _payload| Ok(dummy_cell())); + + let audit = harness.audit.clone(); + let mut runner = harness.build(node_id).await; + let mut ctx = CancellationCtx::new(); + let cancel_ctx = ctx.clone(); + let store = Arc::new(SnapshotStore::new()); + + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(200)).await; + ctx.cancel(CancellationReason::GracefullyShutdown()); + }); + + runner.run_loop(Duration::from_millis(50), cancel_ctx, store, None).await.unwrap(); + + let events = audit.drain(); + assert!( + !events.iter().any(|e| matches!(e.payload, AuditEventPayload::ElectionsTickFailed { .. })), + "successful ticks must not emit tick_failed audit events" + ); +} + +#[tokio::test] +async fn tick_emits_failed_on_error() { + let node_id = "node-1"; + let mut harness = TestHarness::new(); + + harness.elector_mock.expect_address().returning(|| Ok(elector_address())); + harness.elector_mock.expect_get_active_election_id().returning(|| Ok(ELECTION_ID)); + harness + .elector_mock + .expect_elections_info() + .returning(|| Err(anyhow::anyhow!("simulated elections_info failure"))); + harness.elector_mock.expect_past_elections().returning(|| Ok(vec![])); + harness.elector_mock.expect_compute_returned_stake().returning(|_| Ok(0)); + setup_default_provider_without_account(&mut harness.provider_mock, WALLET_BALANCE); + setup_wallet(&mut harness.wallet_mock); + + let audit = harness.audit.clone(); + let mut runner = harness.build(node_id).await; + let mut ctx = CancellationCtx::new(); + let cancel_ctx = ctx.clone(); + let store = Arc::new(SnapshotStore::new()); + + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(200)).await; + ctx.cancel(CancellationReason::GracefullyShutdown()); + }); + + runner.run_loop(Duration::from_millis(50), cancel_ctx, store, None).await.unwrap(); + + let events = audit.drain(); + let failed = + find_audit_event(&events, |p| matches!(p, AuditEventPayload::ElectionsTickFailed { .. })); + + assert_eq!(failed.severity, AuditSeverity::Error); + assert_eq!(failed.outcome, AuditOutcome::Failure); + let AuditEventPayload::ElectionsTickFailed { election_id, error } = &failed.payload else { + unreachable!(); + }; + assert_eq!(*election_id, Some(ELECTION_ID)); + assert!(error.contains("simulated elections_info failure")); +} + +#[tokio::test] +async fn stake_submitted_event_contains_correct_payload() { + let node_id = "node-1"; + let mut harness = TestHarness::new(); + + setup_default_elector(&mut harness.elector_mock, ELECTION_ID, 0); + setup_default_provider(&mut harness.provider_mock, WALLET_BALANCE, None); + setup_wallet(&mut harness.wallet_mock); + let expected_stake = + (WALLET_BALANCE - (ELECTOR_STAKE_FEE + NPOOL_COMPUTE_FEE) - WALLET_STORAGE_RESERVE) / 2; + harness.wallet_mock.expect_message().returning(|_dest, _value, _payload| Ok(dummy_cell())); + + let audit = harness.audit.clone(); + let mut runner = harness.build(node_id).await; + runner.run().await.unwrap(); + + let events = audit.drain(); + let ev = find_audit_event(&events, |p| { + matches!(p, AuditEventPayload::ElectionsStakeSubmitted { .. }) + }); + payload_stake_submitted(&ev.payload); + + assert_eq!(ev.source, AuditSource::Elections); + assert_eq!(ev.severity, AuditSeverity::Info); + assert_eq!(ev.outcome, AuditOutcome::Success); + assert_eq!(ev.subject.kind, AuditSubjectKind::Node); + assert_eq!(ev.subject.id.as_deref(), Some(node_id)); + assert_eq!(ev.subject.election_id, Some(ELECTION_ID)); + + let AuditEventPayload::ElectionsStakeSubmitted { + election_id, + stake_nanotons, + max_factor, + policy, + submission_time, + } = &ev.payload + else { + unreachable!(); + }; + assert_eq!(*election_id, ELECTION_ID); + assert_eq!(stake_nanotons, &expected_stake.to_string()); + assert_eq!(*max_factor, 196608); + assert_eq!(policy, "split50"); + assert!(*submission_time > 0); +} + +#[tokio::test] +async fn stake_skipped_event_has_skipped_outcome_and_warn_severity() { + let node_id = "node-1"; + let mut harness = TestHarness::new(); + harness.bindings.insert(node_id.to_string(), default_binding(false)); + + setup_default_elector(&mut harness.elector_mock, ELECTION_ID, 0); + setup_wallet(&mut harness.wallet_mock); + let provider = &mut harness.provider_mock; + provider.expect_election_parameters().returning(|| Ok(default_cfg15())); + provider.expect_validator_config().returning(|| Ok(ValidatorConfig::new())); + provider.expect_export_public_key().returning(|_| Ok(PUB_KEY.to_vec())); + provider.expect_account().returning(|_| Ok(fake_account(WALLET_BALANCE))); + provider.expect_config_param_16().returning(|| Ok(default_cfg16())); + provider.expect_config_param_17().returning(|| Ok(default_cfg17())); + provider.expect_shutdown().returning(|| Ok(())); + + let audit = harness.audit.clone(); + let mut runner = harness.build(node_id).await; + runner.run().await.unwrap(); + + let events = audit.drain(); + let ev = find_audit_event(&events, |p| { + matches!( + p, + AuditEventPayload::ElectionsStakeSkipped { reason: StakeSkipReason::NodeExcluded, .. } + ) + }); + payload_stake_skipped(&ev.payload); + + assert_eq!(ev.severity, AuditSeverity::Warn); + assert_eq!(ev.outcome, AuditOutcome::Skipped); + assert_eq!(ev.subject.election_id, Some(ELECTION_ID)); +} + +#[tokio::test] +async fn withdraw_processed_emits_tx_hash() { + let node_id = "node-1"; + let mut harness = TestHarness::new().with_toncore_nominator_pair(); + + setup_default_elector(&mut harness.elector_mock, ELECTION_ID, 0); + setup_wallet(&mut harness.wallet_mock); + + let (p0, p1) = harness.toncore_nominator_mocks.as_mut().unwrap(); + setup_toncore_nominator_slot_with(p0, pool_address(), 0, None); + p0.expect_has_withdraw_requests().returning(|| Ok(true)); + p0.expect_send_process_withdraw_requests().returning(|_w, _q, _l, _g| Ok(dummy_cell())); + setup_toncore_nominator_slot(p1, pool_address_1(), 2); + + let pool0_hex = hex::encode(POOL_ADDR); + harness.provider_mock.expect_account().returning(move |address| { + if address.contains(&pool0_hex) { + Ok(fake_account(POOL_BALANCE)) + } else { + Ok(fake_account(WALLET_BALANCE)) + } + }); + setup_default_provider_without_account(&mut harness.provider_mock, WALLET_BALANCE); + + let audit = harness.audit.clone(); + let mut runner = harness.build(node_id).await; + runner.run().await.unwrap(); + + let events = audit.drain(); + let ev = find_audit_event(&events, |p| { + matches!(p, AuditEventPayload::ElectionsWithdrawProcessed { .. }) + }); + + assert_eq!(ev.outcome, AuditOutcome::Success); + assert_eq!(ev.subject.election_id, Some(ELECTION_ID)); + + let AuditEventPayload::ElectionsWithdrawProcessed { election_id, tx_hash } = &ev.payload else { + unreachable!(); + }; + assert_eq!(*election_id, ELECTION_ID); + assert!(!tx_hash.is_empty(), "tx_hash must be the sent message cell hash"); +} + +#[tokio::test] +async fn withdraw_failed_emits_error_string() { + let node_id = "node-1"; + let mut harness = TestHarness::new().with_toncore_nominator_pair(); + + setup_default_elector(&mut harness.elector_mock, ELECTION_ID, 0); + setup_wallet(&mut harness.wallet_mock); + harness.wallet_mock.expect_message().returning(|_dest, _value, _payload| Ok(dummy_cell())); + + let (p0, p1) = harness.toncore_nominator_mocks.as_mut().unwrap(); + setup_toncore_nominator_slot_with(p0, pool_address(), 0, None); + p0.expect_has_withdraw_requests().returning(|| Ok(true)); + p0.expect_send_process_withdraw_requests().returning(|_w, _q, _l, _g| Ok(dummy_cell())); + setup_toncore_nominator_slot(p1, pool_address_1(), 2); + + let pool0_hex = hex::encode(POOL_ADDR); + harness.provider_mock.expect_account().returning(move |address| { + if address.contains(&pool0_hex) { + Ok(fake_account(POOL_BALANCE)) + } else { + Ok(fake_account(WALLET_BALANCE)) + } + }); + + harness + .provider_mock + .expect_send_boc() + .times(1) + .returning(|_| Err(anyhow::anyhow!("simulated withdraw send_boc failure"))); + harness.provider_mock.expect_send_boc().times(1).returning(|_| Ok(())); + setup_default_provider_without_account(&mut harness.provider_mock, WALLET_BALANCE); + + let audit = harness.audit.clone(); + let mut runner = harness.build(node_id).await; + runner.run().await.unwrap(); + + let events = audit.drain(); + let ev = find_audit_event(&events, |p| { + matches!(p, AuditEventPayload::ElectionsWithdrawProcessFailed { .. }) + }); + + assert_eq!(ev.severity, AuditSeverity::Error); + assert_eq!(ev.outcome, AuditOutcome::Failure); + assert_eq!(ev.subject.election_id, Some(ELECTION_ID)); + + let AuditEventPayload::ElectionsWithdrawProcessFailed { election_id, error } = &ev.payload + else { + unreachable!(); + }; + assert_eq!(*election_id, ELECTION_ID); + assert!(error.contains("simulated withdraw send_boc failure")); +} + +#[tokio::test] +async fn subject_election_id_populated_for_election_events() { + let node_id = "node-1"; + let mut harness = TestHarness::new(); + + setup_default_elector(&mut harness.elector_mock, ELECTION_ID, 0); + setup_default_provider(&mut harness.provider_mock, WALLET_BALANCE, None); + setup_wallet(&mut harness.wallet_mock); + harness.wallet_mock.expect_message().returning(|_dest, _value, _payload| Ok(dummy_cell())); + + let audit = harness.audit.clone(); + let mut runner = harness.build(node_id).await; + runner.run().await.unwrap(); + + let events = audit.drain(); + let election_events: Vec<_> = events + .iter() + .filter(|ev| ev.source == AuditSource::Elections) + .filter(|ev| { + matches!( + ev.payload, + AuditEventPayload::ElectionsKeyGenerated { .. } + | AuditEventPayload::ElectionsStakeSubmitted { .. } + ) + }) + .collect(); + + assert!( + !election_events.is_empty(), + "expected at least key_generated and stake_submitted audit events" + ); + for ev in election_events { + assert_eq!( + ev.subject.election_id, + Some(ELECTION_ID), + "event {:?} missing subject.election_id", + ev.payload + ); + } +} From 4b8e8d598e6da29471cc7a6d30f5afae9e7de11a Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Wed, 3 Jun 2026 20:58:39 +0300 Subject: [PATCH 11/30] refactor(nodectl): redesign audit event format and file envelope --- src/node-control/service/Cargo.toml | 2 +- src/node-control/service/src/audit/enums.rs | 188 +++++-- src/node-control/service/src/audit/event.rs | 509 ++++++++++-------- src/node-control/service/src/audit/factory.rs | 30 +- .../service/src/audit/in_memory.rs | 30 +- .../service/src/audit/jsonl_log.rs | 15 +- .../service/src/audit/jsonl_writer.rs | 118 ++-- src/node-control/service/src/audit/log.rs | 34 +- src/node-control/service/src/audit/mod.rs | 7 +- .../service/src/audit/participant.rs | 90 +++- .../service/src/elections/runner.rs | 214 ++------ .../service/src/elections/runner_tests.rs | 64 ++- 12 files changed, 657 insertions(+), 644 deletions(-) diff --git a/src/node-control/service/Cargo.toml b/src/node-control/service/Cargo.toml index 7f98fa7d..9e9de3bc 100644 --- a/src/node-control/service/Cargo.toml +++ b/src/node-control/service/Cargo.toml @@ -28,7 +28,7 @@ jsonwebtoken = "9" base64 = "0.22" chrono = { version = "0.4", features = ["serde"] } thiserror = "1" -uuid = { version = "1", features = ["serde", "v4"] } +uuid = { version = "1", features = ["serde", "v4", "v7"] } [dev-dependencies] mockall = "0.13" diff --git a/src/node-control/service/src/audit/enums.rs b/src/node-control/service/src/audit/enums.rs index 50507f26..7ede2ea3 100644 --- a/src/node-control/service/src/audit/enums.rs +++ b/src/node-control/service/src/audit/enums.rs @@ -8,27 +8,28 @@ */ use serde::{Deserialize, Serialize}; +/// Typed event payload. The wire shape is +/// `{ "event_type": "elections.stake_submitted", "data": { ... } }`. +/// +/// `election_id` is intentionally absent from the variants: it lives on the +/// event `target` (`AuditTarget::Node { election_id }` / +/// `AuditTarget::Elections { election_id }`), so it is never duplicated. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "event_type", content = "data", rename_all = "snake_case")] #[non_exhaustive] pub enum AuditEventPayload { + // ── elections ────────────────────────────────────────────────────────── #[serde(rename = "elections.tick_failed")] - ElectionsTickFailed { - #[serde(skip_serializing_if = "Option::is_none")] - election_id: Option, - error: String, - }, + ElectionsTickFailed { reason: String }, #[serde(rename = "elections.key_generated")] ElectionsKeyGenerated { - election_id: u64, #[serde(skip_serializing_if = "Option::is_none")] pubkey: Option, }, #[serde(rename = "elections.stake_submitted")] ElectionsStakeSubmitted { - election_id: u64, stake_nanotons: String, max_factor: u32, policy: String, @@ -36,11 +37,10 @@ pub enum AuditEventPayload { }, #[serde(rename = "elections.stake_accepted")] - ElectionsStakeAccepted { election_id: u64, stake_nanotons: String }, + ElectionsStakeAccepted { stake_nanotons: String }, #[serde(rename = "elections.stake_skipped")] ElectionsStakeSkipped { - election_id: u64, reason: StakeSkipReason, #[serde(skip_serializing_if = "Option::is_none")] required_nanotons: Option, @@ -48,46 +48,64 @@ pub enum AuditEventPayload { available_nanotons: Option, }, - #[serde(rename = "elections.withdraw_processed")] - ElectionsWithdrawProcessed { election_id: u64, tx_hash: String }, - - #[serde(rename = "elections.withdraw_process_failed")] - ElectionsWithdrawProcessFailed { election_id: u64, error: String }, - #[serde(rename = "elections.stake_recovered")] ElectionsStakeRecovered { - election_id: u64, amount_nanotons: String, #[serde(skip_serializing_if = "Option::is_none")] tx_hash: Option, }, + #[serde(rename = "elections.withdraw_processed")] + ElectionsWithdrawProcessed { tx_hash: String }, + + #[serde(rename = "elections.withdraw_process_failed")] + ElectionsWithdrawProcessFailed { reason: String }, + + // ── rewards (reserved; producers not wired yet) ───────────────────────── + #[serde(rename = "rewards.distribution_started")] + RewardsDistributionStarted { recipients_count: u32 }, + + #[serde(rename = "rewards.distribution_completed")] + RewardsDistributionCompleted { recipients_count: u32, total_nanotons: String }, + + #[serde(rename = "rewards.distribution_failed")] + RewardsDistributionFailed { reason: String }, + + #[serde(rename = "rewards.recipient_skipped")] + RewardsRecipientSkipped { reason: String }, + + // ── rest_api ──────────────────────────────────────────────────────────── #[serde(rename = "rest_api.config_updated")] - RestApiConfigUpdated { - operation: String, - changes: serde_json::Value, // diff stays free-form - }, + RestApiConfigUpdated { operation: String, changes: Vec }, #[serde(rename = "rest_api.auth_login_success")] - RestApiAuthLoginSuccess { username: String }, + RestApiAuthLoginSuccess {}, #[serde(rename = "rest_api.auth_login_rejected")] - RestApiAuthLoginRejected { username: String, reason: String }, + RestApiAuthLoginRejected { reason: String }, #[serde(rename = "rest_api.token_rejected")] RestApiTokenRejected { reason: String }, + // ── vault (reserved; producers not wired yet) ─────────────────────────── + #[serde(rename = "vault.key_created")] + VaultKeyCreated {}, + + #[serde(rename = "vault.key_removed")] + VaultKeyRemoved {}, + + // ── system ─────────────────────────────────────────────────────────────── #[serde(rename = "system.service_started")] SystemServiceStarted { version: String }, #[serde(rename = "system.service_stopped")] - SystemServiceStopped, + SystemServiceStopped {}, #[serde(rename = "system.audit_events_dropped")] SystemAuditEventsDropped { dropped_events: u64, reason: String }, } -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] #[non_exhaustive] pub enum StakeSkipReason { @@ -101,26 +119,17 @@ pub enum StakeSkipReason { InsufficientStakeFunds, } -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AuditSource { - Elections, - Rewards, - RestApi, - Vault, - System, -} - -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AuditSeverity { - Debug, - Info, - Warn, - Error, +/// A single typed field change for `rest_api.config_updated`. Replaces the +/// previous free-form `serde_json::Value` diff. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConfigFieldChange { + /// Dotted path, e.g. `elections.sleep_period_pct`. + pub field: String, + pub old: serde_json::Value, + pub new: serde_json::Value, } -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum AuditOutcome { Success, @@ -128,24 +137,85 @@ pub enum AuditOutcome { Skipped, } -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AuditActorKind { - Service, - User, - Scheduler, - System, +// ── Derived properties — kept in code, never serialized onto the wire ──────── + +/// Log-level-like severity, derived from the event type at the display layer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditSeverity { + Info, + Warn, + Error, } -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AuditSubjectKind { - Node, +/// Originating subsystem, derived from the `event_type` prefix. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditSource { Elections, - Config, - Wallet, - VaultKey, - User, - RewardRound, - Recipient, + Rewards, + RestApi, + Vault, + System, +} + +impl AuditEventPayload { + pub fn severity(&self) -> AuditSeverity { + use AuditEventPayload::*; + use AuditSeverity::*; + match self { + ElectionsKeyGenerated { .. } + | ElectionsStakeSubmitted { .. } + | ElectionsStakeAccepted { .. } + | ElectionsStakeRecovered { .. } + | ElectionsWithdrawProcessed { .. } + | RewardsDistributionStarted { .. } + | RewardsDistributionCompleted { .. } + | RestApiConfigUpdated { .. } + | RestApiAuthLoginSuccess {} + | VaultKeyCreated {} + | VaultKeyRemoved {} + | SystemServiceStarted { .. } + | SystemServiceStopped {} => Info, + + ElectionsStakeSkipped { .. } + | RewardsRecipientSkipped { .. } + | RestApiAuthLoginRejected { .. } + | RestApiTokenRejected { .. } + | SystemAuditEventsDropped { .. } => Warn, + + ElectionsTickFailed { .. } + | ElectionsWithdrawProcessFailed { .. } + | RewardsDistributionFailed { .. } => Error, + } + } + + pub fn source(&self) -> AuditSource { + use AuditEventPayload::*; + use AuditSource::*; + match self { + ElectionsTickFailed { .. } + | ElectionsKeyGenerated { .. } + | ElectionsStakeSubmitted { .. } + | ElectionsStakeAccepted { .. } + | ElectionsStakeSkipped { .. } + | ElectionsStakeRecovered { .. } + | ElectionsWithdrawProcessed { .. } + | ElectionsWithdrawProcessFailed { .. } => Elections, + + RewardsDistributionStarted { .. } + | RewardsDistributionCompleted { .. } + | RewardsDistributionFailed { .. } + | RewardsRecipientSkipped { .. } => Rewards, + + RestApiConfigUpdated { .. } + | RestApiAuthLoginSuccess {} + | RestApiAuthLoginRejected { .. } + | RestApiTokenRejected { .. } => RestApi, + + VaultKeyCreated {} | VaultKeyRemoved {} => Vault, + + SystemServiceStarted { .. } + | SystemServiceStopped {} + | SystemAuditEventsDropped { .. } => System, + } + } } diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index 1b2f1954..6d9f050e 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -7,38 +7,230 @@ * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ use crate::audit::{ - enums::{AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource}, - participant::{AuditActor, AuditSubject}, + enums::{AuditEventPayload, AuditOutcome, StakeSkipReason}, + participant::{AuditActor, AuditTarget}, }; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; +use chrono::{DateTime, SecondsFormat, Utc}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use uuid::Uuid; +/// Renders timestamps as RFC3339 with millisecond precision and a trailing `Z` +/// (e.g. `2026-05-22T12:10:30.123Z`), used for `ts` and `started_at`. +mod ts_millis_rfc3339 { + use super::*; + + pub fn serialize(ts: &DateTime, s: S) -> Result { + s.serialize_str(&ts.to_rfc3339_opts(SecondsFormat::Millis, true)) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let raw = String::deserialize(d)?; + DateTime::parse_from_rfc3339(&raw) + .map(|dt| dt.with_timezone(&Utc)) + .map_err(serde::de::Error::custom) + } +} + +/// First JSONL line of every (rotated) audit file. Readers distinguish it from +/// events by the absence of an `event_type` field. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AuditEvent { +pub struct AuditFileHeader { pub schema_version: u16, + /// Logical service name, e.g. `"nodectl"`. + pub service: String, + /// Service semver. + pub service_version: String, + pub host: String, + #[serde(with = "ts_millis_rfc3339")] + pub started_at: DateTime, +} + +/// A single audit record. +/// +/// Wire shape: `id`, `ts`, `outcome`, the flattened payload +/// (`event_type` + `data`), `actor`, `target`. `severity`/`source` are derived +/// from the payload at the display layer and `schema_version` lives in +/// [`AuditFileHeader`], so none of them are stored per event. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AuditEvent { + /// UUID v7 — sortable by creation time. pub id: Uuid, + #[serde(with = "ts_millis_rfc3339")] pub ts: DateTime, - pub source: AuditSource, - pub severity: AuditSeverity, pub outcome: AuditOutcome, - pub actor: AuditActor, - pub subject: AuditSubject, - #[serde(skip_serializing_if = "Option::is_none")] - pub message: Option, #[serde(flatten)] pub payload: AuditEventPayload, + pub actor: AuditActor, + pub target: AuditTarget, +} + +impl AuditEvent { + /// Internal constructor that stamps `id`/`ts`. Crate-private so call sites + /// must go through the typed constructors below, which bake the canonical + /// outcome per event type. + pub(crate) fn new( + actor: AuditActor, + target: AuditTarget, + outcome: AuditOutcome, + payload: AuditEventPayload, + ) -> Self { + Self { id: Uuid::now_v7(), ts: Utc::now(), outcome, payload, actor, target } + } + + /// `target` for a per-node election event: always `Node { election_id }`. + fn node_target(node_id: impl Into, election_id: u64) -> AuditTarget { + AuditTarget::Node { id: node_id.into(), election_id: Some(election_id) } + } + + pub fn elections_tick_failed( + actor: AuditActor, + election_id: Option, + reason: impl Into, + ) -> Self { + // A tick can fail before the active election id is known; fall back to a + // system target in that case (source still resolves to `elections`). + let target = election_id + .map(|election_id| AuditTarget::Elections { election_id }) + .unwrap_or(AuditTarget::System); + Self::new( + actor, + target, + AuditOutcome::Failure, + AuditEventPayload::ElectionsTickFailed { reason: reason.into() }, + ) + } + + pub fn elections_key_generated( + actor: AuditActor, + node_id: impl Into, + election_id: u64, + pubkey: Option, + ) -> Self { + Self::new( + actor, + Self::node_target(node_id, election_id), + AuditOutcome::Success, + AuditEventPayload::ElectionsKeyGenerated { pubkey }, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn elections_stake_submitted( + actor: AuditActor, + node_id: impl Into, + election_id: u64, + stake_nanotons: impl Into, + max_factor: u32, + policy: impl Into, + submission_time: u64, + ) -> Self { + Self::new( + actor, + Self::node_target(node_id, election_id), + AuditOutcome::Success, + AuditEventPayload::ElectionsStakeSubmitted { + stake_nanotons: stake_nanotons.into(), + max_factor, + policy: policy.into(), + submission_time, + }, + ) + } + + pub fn elections_stake_skipped( + actor: AuditActor, + node_id: impl Into, + election_id: u64, + reason: StakeSkipReason, + required_nanotons: Option, + available_nanotons: Option, + ) -> Self { + Self::new( + actor, + Self::node_target(node_id, election_id), + AuditOutcome::Skipped, + AuditEventPayload::ElectionsStakeSkipped { + reason, + required_nanotons, + available_nanotons, + }, + ) + } + + pub fn elections_stake_recovered( + actor: AuditActor, + node_id: impl Into, + election_id: u64, + amount_nanotons: impl Into, + tx_hash: Option, + ) -> Self { + Self::new( + actor, + Self::node_target(node_id, election_id), + AuditOutcome::Success, + AuditEventPayload::ElectionsStakeRecovered { + amount_nanotons: amount_nanotons.into(), + tx_hash, + }, + ) + } + + pub fn elections_withdraw_processed( + actor: AuditActor, + node_id: impl Into, + election_id: u64, + tx_hash: impl Into, + ) -> Self { + Self::new( + actor, + Self::node_target(node_id, election_id), + AuditOutcome::Success, + AuditEventPayload::ElectionsWithdrawProcessed { tx_hash: tx_hash.into() }, + ) + } + + pub fn elections_withdraw_process_failed( + actor: AuditActor, + node_id: impl Into, + election_id: u64, + reason: impl Into, + ) -> Self { + Self::new( + actor, + Self::node_target(node_id, election_id), + AuditOutcome::Failure, + AuditEventPayload::ElectionsWithdrawProcessFailed { reason: reason.into() }, + ) + } + + pub fn system_service_started(version: impl Into) -> Self { + Self::new( + AuditActor::System, + AuditTarget::System, + AuditOutcome::Success, + AuditEventPayload::SystemServiceStarted { version: version.into() }, + ) + } + + pub fn system_audit_events_dropped(dropped: u64) -> Self { + Self::new( + AuditActor::System, + AuditTarget::System, + AuditOutcome::Failure, + AuditEventPayload::SystemAuditEventsDropped { + dropped_events: dropped, + reason: "queue_full_after_timeout".into(), + }, + ) + } } #[cfg(test)] mod tests { use super::*; - use crate::audit::{ - AuditLogConfig, - enums::{AuditActorKind, AuditEventPayload, AuditSubjectKind, StakeSkipReason}, - }; + use crate::audit::{AuditLogConfig, enums::ConfigFieldChange}; use serde_json::{Value, json}; - use std::{collections::BTreeMap, path::PathBuf}; + use std::path::PathBuf; const FIXTURE_ID: &str = "9b6c2b5a-9f9d-4a9f-bc31-9a89b0e9d111"; const FIXTURE_TS: &str = "2026-05-22T12:10:30.123Z"; @@ -56,260 +248,156 @@ mod tests { assert_eq!(actual_value, expected); } + fn fixed( + outcome: AuditOutcome, + actor: AuditActor, + target: AuditTarget, + payload: AuditEventPayload, + ) -> AuditEvent { + AuditEvent { id: fixture_id(), ts: fixture_ts(), outcome, payload, actor, target } + } + #[test] fn serializes_stake_submitted_to_expected_json() { - let event = AuditEvent { - schema_version: 1, - id: fixture_id(), - ts: fixture_ts(), - source: AuditSource::Elections, - severity: AuditSeverity::Info, - outcome: AuditOutcome::Success, - actor: AuditActor { - kind: AuditActorKind::Service, - id: Some("elections-task".into()), - role: None, - ip: None, - }, - subject: AuditSubject { - kind: AuditSubjectKind::Node, - id: Some("node1".into()), - election_id: Some(1_779_265_552), - labels: BTreeMap::new(), - }, - message: None, - payload: AuditEventPayload::ElectionsStakeSubmitted { - election_id: 1_779_265_552, + let event = fixed( + AuditOutcome::Success, + AuditActor::service("elections-task"), + AuditTarget::Node { id: "node1".into(), election_id: Some(1_779_265_552) }, + AuditEventPayload::ElectionsStakeSubmitted { stake_nanotons: "50000000000000".into(), max_factor: 196_608, policy: "adaptive_split50".into(), submission_time: 1_779_265_400, }, - }; + ); assert_json_eq( &event, json!({ - "schema_version": 1, "id": FIXTURE_ID, "ts": FIXTURE_TS, - "source": "elections", - "severity": "info", "outcome": "success", - "actor": { - "kind": "service", - "id": "elections-task" - }, - "subject": { - "kind": "node", - "id": "node1", - "election_id": 1779265552 - }, "event_type": "elections.stake_submitted", "data": { - "election_id": 1779265552, "stake_nanotons": "50000000000000", "max_factor": 196608, "policy": "adaptive_split50", "submission_time": 1779265400 - } + }, + "actor": { "kind": "service", "id": "elections-task" }, + "target": { "kind": "node", "id": "node1", "election_id": 1779265552 } }), ); } #[test] fn serializes_stake_skipped_to_expected_json() { - let event = AuditEvent { - schema_version: 1, - id: fixture_id(), - ts: fixture_ts(), - source: AuditSource::Elections, - severity: AuditSeverity::Warn, - outcome: AuditOutcome::Skipped, - actor: AuditActor { - kind: AuditActorKind::Service, - id: Some("elections-task".into()), - role: None, - ip: None, - }, - subject: AuditSubject { - kind: AuditSubjectKind::Node, - id: Some("node6".into()), - election_id: Some(1_779_265_552), - labels: BTreeMap::new(), - }, - message: None, - payload: AuditEventPayload::ElectionsStakeSkipped { - election_id: 1_779_265_552, + let event = fixed( + AuditOutcome::Skipped, + AuditActor::service("elections-task"), + AuditTarget::Node { id: "node6".into(), election_id: Some(1_779_265_552) }, + AuditEventPayload::ElectionsStakeSkipped { reason: StakeSkipReason::LowWalletBalance, required_nanotons: Some("1200000000".into()), available_nanotons: Some("900000000".into()), }, - }; + ); assert_json_eq( &event, json!({ - "schema_version": 1, "id": FIXTURE_ID, "ts": FIXTURE_TS, - "source": "elections", - "severity": "warn", "outcome": "skipped", - "actor": { - "kind": "service", - "id": "elections-task" - }, - "subject": { - "kind": "node", - "id": "node6", - "election_id": 1779265552 - }, "event_type": "elections.stake_skipped", "data": { - "election_id": 1779265552, "reason": "low_wallet_balance", "required_nanotons": "1200000000", "available_nanotons": "900000000" - } + }, + "actor": { "kind": "service", "id": "elections-task" }, + "target": { "kind": "node", "id": "node6", "election_id": 1779265552 } }), ); } #[test] - fn serializes_config_updated_to_expected_json() { - let event = AuditEvent { + fn file_header_serializes_with_millis_ts() { + let header = AuditFileHeader { schema_version: 1, - id: fixture_id(), - ts: fixture_ts(), - source: AuditSource::RestApi, - severity: AuditSeverity::Info, - outcome: AuditOutcome::Success, - actor: AuditActor { - kind: AuditActorKind::User, - id: Some("admin".into()), - role: Some("operator".into()), - ip: None, - }, - subject: AuditSubject { - kind: AuditSubjectKind::Config, - id: Some("elections".into()), - election_id: None, - labels: BTreeMap::new(), - }, - message: None, - payload: AuditEventPayload::RestApiConfigUpdated { - operation: "elections.wait_updated".into(), - changes: json!({ - "sleep_period_pct": { "old": 0.2, "new": 0.9 }, - "waiting_period_pct": { "old": 0.4, "new": 0.95 } - }), - }, + service: "nodectl".into(), + service_version: "0.5.1".into(), + host: "node-host".into(), + started_at: fixture_ts(), }; - - assert_json_eq( - &event, + let value = serde_json::to_value(&header).expect("serialize header"); + assert_eq!( + value, json!({ "schema_version": 1, - "id": FIXTURE_ID, - "ts": FIXTURE_TS, - "source": "rest_api", - "severity": "info", - "outcome": "success", - "actor": { - "kind": "user", - "id": "admin", - "role": "operator" - }, - "subject": { - "kind": "config", - "id": "elections" - }, - "event_type": "rest_api.config_updated", - "data": { - "operation": "elections.wait_updated", - "changes": { - "sleep_period_pct": { "old": 0.2, "new": 0.9 }, - "waiting_period_pct": { "old": 0.4, "new": 0.95 } - } - } - }), + "service": "nodectl", + "service_version": "0.5.1", + "host": "node-host", + "started_at": FIXTURE_TS + }) ); + // Header has no event_type — that is how readers tell it apart from events. + assert!(value.get("event_type").is_none()); } fn sample_event(payload: AuditEventPayload) -> AuditEvent { - AuditEvent { - schema_version: 1, - id: fixture_id(), - ts: fixture_ts(), - source: AuditSource::System, - severity: AuditSeverity::Info, - outcome: AuditOutcome::Success, - actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, - subject: AuditSubject { - kind: AuditSubjectKind::Node, - id: Some("node1".into()), - election_id: None, - labels: BTreeMap::new(), - }, - message: None, + fixed( + AuditOutcome::Success, + AuditActor::System, + AuditTarget::Node { id: "node1".into(), election_id: Some(1_779_265_552) }, payload, - } + ) } fn all_payload_variants() -> Vec { - const ELECTION_ID: u64 = 1_779_265_552; vec![ - AuditEventPayload::ElectionsTickFailed { - election_id: Some(ELECTION_ID), - error: "tick error".into(), - }, - AuditEventPayload::ElectionsKeyGenerated { - election_id: ELECTION_ID, - pubkey: Some("aabb".into()), - }, + AuditEventPayload::ElectionsTickFailed { reason: "tick error".into() }, + AuditEventPayload::ElectionsKeyGenerated { pubkey: Some("aabb".into()) }, AuditEventPayload::ElectionsStakeSubmitted { - election_id: ELECTION_ID, stake_nanotons: "1".into(), max_factor: 1, policy: "all".into(), submission_time: 1, }, - AuditEventPayload::ElectionsStakeAccepted { - election_id: ELECTION_ID, - stake_nanotons: "50000000000000".into(), - }, + AuditEventPayload::ElectionsStakeAccepted { stake_nanotons: "50000000000000".into() }, AuditEventPayload::ElectionsStakeSkipped { - election_id: ELECTION_ID, reason: StakeSkipReason::WithdrawRequestsPending, required_nanotons: None, available_nanotons: None, }, - AuditEventPayload::ElectionsWithdrawProcessed { - election_id: ELECTION_ID, - tx_hash: "abc".into(), - }, - AuditEventPayload::ElectionsWithdrawProcessFailed { - election_id: ELECTION_ID, - error: "send failed".into(), - }, + AuditEventPayload::ElectionsWithdrawProcessed { tx_hash: "abc".into() }, + AuditEventPayload::ElectionsWithdrawProcessFailed { reason: "send failed".into() }, AuditEventPayload::ElectionsStakeRecovered { - election_id: ELECTION_ID, amount_nanotons: "50000000000000".into(), tx_hash: Some("def".into()), }, + AuditEventPayload::RewardsDistributionStarted { recipients_count: 3 }, + AuditEventPayload::RewardsDistributionCompleted { + recipients_count: 3, + total_nanotons: "9".into(), + }, + AuditEventPayload::RewardsDistributionFailed { reason: "rpc".into() }, + AuditEventPayload::RewardsRecipientSkipped { reason: "below_min".into() }, AuditEventPayload::RestApiConfigUpdated { operation: "patch".into(), - changes: json!({ "path": "/v1/elections/settings" }), - }, - AuditEventPayload::RestApiAuthLoginSuccess { username: "admin".into() }, - AuditEventPayload::RestApiAuthLoginRejected { - username: "admin".into(), - reason: "bad password".into(), + changes: vec![ConfigFieldChange { + field: "elections.x".into(), + old: json!(1), + new: json!(2), + }], }, + AuditEventPayload::RestApiAuthLoginSuccess {}, + AuditEventPayload::RestApiAuthLoginRejected { reason: "bad password".into() }, AuditEventPayload::RestApiTokenRejected { reason: "expired".into() }, + AuditEventPayload::VaultKeyCreated {}, + AuditEventPayload::VaultKeyRemoved {}, AuditEventPayload::SystemServiceStarted { version: "0.5.0".into() }, - AuditEventPayload::SystemServiceStopped, + AuditEventPayload::SystemServiceStopped {}, AuditEventPayload::SystemAuditEventsDropped { dropped_events: 3, reason: "queue_full_after_timeout".into(), @@ -328,31 +416,24 @@ mod tests { } #[test] - fn subject_labels_omitted_when_empty() { - let event = AuditEvent { - schema_version: 1, - id: fixture_id(), - ts: fixture_ts(), - source: AuditSource::Elections, - severity: AuditSeverity::Info, - outcome: AuditOutcome::Success, - actor: AuditActor { kind: AuditActorKind::Service, id: None, role: None, ip: None }, - subject: AuditSubject { - kind: AuditSubjectKind::Node, - id: Some("node1".into()), - election_id: None, - labels: BTreeMap::new(), - }, - message: None, - payload: AuditEventPayload::ElectionsWithdrawProcessed { - election_id: 1_779_265_552, - tx_hash: "abc".into(), - }, - }; + fn canonical_outcome_is_baked_into_constructors() { + let skipped = AuditEvent::elections_stake_skipped( + AuditActor::service("elections-task"), + "node1", + 1, + StakeSkipReason::PoolNotReady, + None, + None, + ); + assert_eq!(skipped.outcome, AuditOutcome::Skipped); - let value = serde_json::to_value(&event).expect("serialize"); - let subject = value.get("subject").expect("subject").as_object().expect("object"); - assert!(!subject.contains_key("labels"), "empty labels must not appear in JSON: {value}"); + let failed = AuditEvent::elections_tick_failed( + AuditActor::scheduler("elections-task"), + None, + "boom", + ); + assert_eq!(failed.outcome, AuditOutcome::Failure); + assert_eq!(failed.target, AuditTarget::System); } #[test] diff --git a/src/node-control/service/src/audit/factory.rs b/src/node-control/service/src/audit/factory.rs index f4466261..6f5b9c7f 100644 --- a/src/node-control/service/src/audit/factory.rs +++ b/src/node-control/service/src/audit/factory.rs @@ -28,37 +28,11 @@ impl AuditLogFactory { #[cfg(test)] mod tests { use super::*; - use crate::audit::{ - AuditEvent, - enums::{ - AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, - AuditSubjectKind, - }, - participant::{AuditActor, AuditSubject}, - }; - use chrono::Utc; - use std::collections::BTreeMap; + use crate::audit::AuditEvent; use tempfile::tempdir; - use uuid::Uuid; fn sample_event() -> AuditEvent { - AuditEvent { - schema_version: 1, - id: Uuid::new_v4(), - ts: Utc::now(), - source: AuditSource::System, - severity: AuditSeverity::Info, - outcome: AuditOutcome::Success, - actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, - subject: AuditSubject { - kind: AuditSubjectKind::Config, - id: None, - election_id: None, - labels: BTreeMap::new(), - }, - message: None, - payload: AuditEventPayload::SystemServiceStarted { version: "test".into() }, - } + AuditEvent::system_service_started("test") } #[tokio::test] diff --git a/src/node-control/service/src/audit/in_memory.rs b/src/node-control/service/src/audit/in_memory.rs index 0d137aa3..2de9cbba 100644 --- a/src/node-control/service/src/audit/in_memory.rs +++ b/src/node-control/service/src/audit/in_memory.rs @@ -41,38 +41,12 @@ impl AuditLog for InMemoryAuditLog { #[cfg(test)] mod tests { use super::*; - use crate::audit::{ - AuditEvent, - enums::{ - AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, - AuditSubjectKind, - }, - participant::{AuditActor, AuditSubject}, - }; - use chrono::Utc; - use std::collections::BTreeMap; - use uuid::Uuid; + use crate::audit::AuditEvent; #[tokio::test] async fn records_and_drains_events() { let log = InMemoryAuditLog::new(); - let event = AuditEvent { - schema_version: 1, - id: Uuid::new_v4(), - ts: Utc::now(), - source: AuditSource::Elections, - severity: AuditSeverity::Info, - outcome: AuditOutcome::Success, - actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, - subject: AuditSubject { - kind: AuditSubjectKind::Node, - id: Some("n1".into()), - election_id: None, - labels: BTreeMap::new(), - }, - message: None, - payload: AuditEventPayload::SystemServiceStarted { version: "test".into() }, - }; + let event = AuditEvent::system_service_started("test"); log.record(event.clone()).await; let drained = log.drain(); assert_eq!(drained.len(), 1); diff --git a/src/node-control/service/src/audit/jsonl_log.rs b/src/node-control/service/src/audit/jsonl_log.rs index 0074ac28..b9553969 100644 --- a/src/node-control/service/src/audit/jsonl_log.rs +++ b/src/node-control/service/src/audit/jsonl_log.rs @@ -98,30 +98,23 @@ impl AuditLog for JsonlAuditLog { } async fn record(&self, event: AuditEvent) { + // Capture diagnostics up front so a dropped event can still be attributed + // to its id/source after the event is moved into the send future. + let event_id = event.id; + let source = event.payload.source(); let cmd = AuditCommand::Event(Box::new(event)); match self.sender.try_send(cmd) { Ok(()) => return, Err(mpsc::error::TrySendError::Full(cmd)) => { - // Capture diagnostics before `cmd` is moved into the send future, - // so a dropped event can be attributed to its source/subject. - let diag = match &cmd { - AuditCommand::Event(ev) => Some((ev.id, ev.source, ev.subject.kind)), - _ => None, - }; let timeout = Duration::from_millis(self.config.queue_full_timeout_ms); match tokio::time::timeout(timeout, self.sender.send(cmd)).await { Ok(Ok(())) => return, _ => { self.dropped_events.fetch_add(1, Ordering::Relaxed); - let (event_id, source, subject) = match diag { - Some((id, source, subject)) => (Some(id), Some(source), Some(subject)), - None => (None, None, None), - }; tracing::warn!( ?event_id, ?source, - ?subject, "audit event dropped: queue full after timeout" ); } diff --git a/src/node-control/service/src/audit/jsonl_writer.rs b/src/node-control/service/src/audit/jsonl_writer.rs index 50470ace..4e68a97d 100644 --- a/src/node-control/service/src/audit/jsonl_writer.rs +++ b/src/node-control/service/src/audit/jsonl_writer.rs @@ -6,18 +6,9 @@ * * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ -use crate::audit::{ - AuditEvent, AuditLogConfig, - enums::{ - AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, - AuditSubjectKind, - }, - jsonl_log::AuditInitError, - participant::{AuditActor, AuditSubject}, -}; +use crate::audit::{AuditEvent, AuditFileHeader, AuditLogConfig, jsonl_log::AuditInitError}; use chrono::Utc; use std::{ - collections::BTreeMap, sync::{ Arc, atomic::{AtomicU64, Ordering}, @@ -25,7 +16,9 @@ use std::{ time::Duration, }; use tokio::sync::mpsc; -use uuid::Uuid; + +/// Schema version stamped into the per-file [`AuditFileHeader`]. +const AUDIT_SCHEMA_VERSION: u16 = 1; pub(crate) enum AuditCommand { Event(Box), @@ -75,7 +68,7 @@ impl AuditWriter { let current_size = file.metadata().await.map_err(AuditInitError::FileOpen)?.len(); - Ok(Self { + let mut writer = Self { config, file: Some(file), current_size, @@ -83,7 +76,51 @@ impl AuditWriter { dropped_events: dropped, last_dropped_seen: 0, write_delay, - }) + }; + // A brand-new (empty) live file starts with a header line so each file is + // self-describing. An existing non-empty file already has one. + writer.write_header_if_empty().await.map_err(AuditInitError::FileOpen)?; + Ok(writer) + } + + fn file_header() -> AuditFileHeader { + AuditFileHeader { + schema_version: AUDIT_SCHEMA_VERSION, + service: "nodectl".into(), + service_version: env!("CARGO_PKG_VERSION").into(), + host: Self::hostname(), + started_at: Utc::now(), + } + } + + fn hostname() -> String { + std::env::var("HOSTNAME") + .or_else(|_| std::env::var("COMPUTERNAME")) + .ok() + .filter(|h| !h.is_empty()) + .unwrap_or_else(|| "unknown".to_string()) + } + + /// Writes the file header as the first line, but only when the live segment + /// is empty (fresh file or just-rotated/truncated). No-op otherwise. + async fn write_header_if_empty(&mut self) -> std::io::Result<()> { + if self.current_size != 0 { + return Ok(()); + } + use tokio::io::AsyncWriteExt; + let mut line = serde_json::to_vec(&Self::file_header()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + line.push(b'\n'); + let file = self + .file + .as_mut() + .ok_or_else(|| std::io::Error::other("audit file handle not open"))?; + file.write_all(&line).await?; + if self.config.fsync_on_batch { + file.sync_data().await?; + } + self.current_size += line.len() as u64; + Ok(()) } pub(crate) async fn run(mut self, mut rx: mpsc::Receiver) { @@ -254,6 +291,8 @@ impl AuditWriter { opts.mode(0o600); self.file = Some(opts.open(path).await?); self.current_size = 0; + // Fresh segment: re-emit the file header as the first line. + self.write_header_if_empty().await?; Ok(()) } @@ -265,27 +304,7 @@ impl AuditWriter { } self.last_dropped_seen = current; - let event = AuditEvent { - schema_version: 1, - id: Uuid::new_v4(), - ts: Utc::now(), - source: AuditSource::System, - severity: AuditSeverity::Warn, - outcome: AuditOutcome::Failure, - actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, - subject: AuditSubject { - kind: AuditSubjectKind::Config, - id: None, - election_id: None, - labels: BTreeMap::new(), - }, - message: Some("audit events dropped".into()), - payload: AuditEventPayload::SystemAuditEventsDropped { - dropped_events: delta, - reason: "queue_full_after_timeout".into(), - }, - }; - let mut buf = vec![event]; + let mut buf = vec![AuditEvent::system_audit_events_dropped(delta)]; self.flush(&mut buf).await; } } @@ -313,32 +332,12 @@ mod tests { } fn sample_event(tag: &str) -> AuditEvent { - AuditEvent { - schema_version: 1, - id: Uuid::new_v4(), - ts: Utc::now(), - source: AuditSource::System, - severity: AuditSeverity::Info, - outcome: AuditOutcome::Success, - actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, - subject: AuditSubject { - kind: AuditSubjectKind::Config, - id: Some(tag.into()), - election_id: None, - labels: BTreeMap::new(), - }, - message: None, - payload: AuditEventPayload::SystemServiceStarted { version: tag.into() }, - } + // `system.service_started` carries `data.version`, which tests assert on. + AuditEvent::system_service_started(tag) } fn large_event(payload_kb: usize) -> AuditEvent { - let mut event = sample_event("large"); - event.payload = AuditEventPayload::RestApiConfigUpdated { - operation: "update".into(), - changes: serde_json::json!({ "blob": "x".repeat(payload_kb * 1024) }), - }; - event + AuditEvent::system_service_started("x".repeat(payload_kb * 1024)) } fn test_config(dir: &Path, mut cfg: AuditLogConfig) -> AuditLogConfig { @@ -386,13 +385,16 @@ mod tests { tx.send(AuditCommand::Shutdown).await.unwrap(); } + /// Reads event lines, skipping the per-file [`AuditFileHeader`] (the only + /// line without an `event_type` field). fn read_json_lines(path: &Path) -> Vec { assert!(path.exists(), "audit file missing at {}", path.display()); let content = std::fs::read_to_string(path).unwrap(); content .lines() .filter(|line| !line.is_empty()) - .map(|line| serde_json::from_str(line).expect("valid json line")) + .map(|line| serde_json::from_str::(line).expect("valid json line")) + .filter(|value| value.get("event_type").is_some()) .collect() } diff --git a/src/node-control/service/src/audit/log.rs b/src/node-control/service/src/audit/log.rs index 004caf97..9c95f684 100644 --- a/src/node-control/service/src/audit/log.rs +++ b/src/node-control/service/src/audit/log.rs @@ -25,41 +25,11 @@ impl AuditLog for NoopAuditLog { #[cfg(test)] mod tests { use super::*; - use crate::audit::{ - AuditEvent, - enums::{ - AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, - AuditSubjectKind, - }, - participant::{AuditActor, AuditSubject}, - }; - use chrono::Utc; - use std::collections::BTreeMap; - use uuid::Uuid; - - fn sample_event() -> AuditEvent { - AuditEvent { - schema_version: 1, - id: Uuid::new_v4(), - ts: Utc::now(), - source: AuditSource::System, - severity: AuditSeverity::Info, - outcome: AuditOutcome::Success, - actor: AuditActor { kind: AuditActorKind::System, id: None, role: None, ip: None }, - subject: AuditSubject { - kind: AuditSubjectKind::Config, - id: None, - election_id: None, - labels: BTreeMap::new(), - }, - message: None, - payload: AuditEventPayload::SystemServiceStarted { version: "test".into() }, - } - } + use crate::audit::AuditEvent; #[tokio::test] async fn noop_audit_log_record_completes() { let log = NoopAuditLog; - log.record(sample_event()).await; + log.record(AuditEvent::system_service_started("test")).await; } } diff --git a/src/node-control/service/src/audit/mod.rs b/src/node-control/service/src/audit/mod.rs index 26a622a2..11256d9e 100644 --- a/src/node-control/service/src/audit/mod.rs +++ b/src/node-control/service/src/audit/mod.rs @@ -18,13 +18,12 @@ pub mod participant; pub use common::app_config::AuditLogConfig; pub use enums::{ - AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, AuditSubjectKind, - StakeSkipReason, + AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, ConfigFieldChange, StakeSkipReason, }; -pub use event::AuditEvent; +pub use event::{AuditEvent, AuditFileHeader}; pub use factory::AuditLogFactory; #[cfg(test)] pub use in_memory::InMemoryAuditLog; pub use jsonl_log::AuditInitError; pub use log::{AuditLog, NoopAuditLog}; -pub use participant::{AuditActor, AuditSubject}; +pub use participant::{AuditActor, AuditTarget}; diff --git a/src/node-control/service/src/audit/participant.rs b/src/node-control/service/src/audit/participant.rs index b6fca698..fe054ef0 100644 --- a/src/node-control/service/src/audit/participant.rs +++ b/src/node-control/service/src/audit/participant.rs @@ -6,28 +6,82 @@ * * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ -use crate::audit::enums::{AuditActorKind, AuditSubjectKind}; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +/// Who initiated the action. Internally tagged: the wire shape is +/// `{ "kind": "...", ...variant fields }`, so illegal field combinations +/// (e.g. a `role` on a scheduler) are unrepresentable. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AuditActor { - pub kind: AuditActorKind, - #[serde(skip_serializing_if = "Option::is_none")] - pub id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub role: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ip: Option, +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AuditActor { + Service { + id: String, + }, + Scheduler { + id: String, + }, + User { + id: String, + #[serde(skip_serializing_if = "Option::is_none")] + role: Option, + #[serde(skip_serializing_if = "Option::is_none")] + ip: Option, + }, + System, } +impl AuditActor { + pub fn service(id: impl Into) -> Self { + Self::Service { id: id.into() } + } + + pub fn scheduler(id: impl Into) -> Self { + Self::Scheduler { id: id.into() } + } + + pub fn user(id: impl Into, role: Option, ip: Option) -> Self { + Self::User { id: id.into(), role, ip } + } + + pub fn system() -> Self { + Self::System + } +} + +/// What the action was applied to. Internally tagged so per-variant required +/// fields are enforced by the type system; `#[non_exhaustive]` because new +/// target kinds are expected as more producers are wired. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AuditSubject { - pub kind: AuditSubjectKind, - #[serde(skip_serializing_if = "Option::is_none")] - pub id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub election_id: Option, // first-class for hot filtering - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub labels: BTreeMap, +#[serde(tag = "kind", rename_all = "snake_case")] +#[non_exhaustive] +pub enum AuditTarget { + Node { + id: String, + #[serde(skip_serializing_if = "Option::is_none")] + election_id: Option, + }, + Elections { + election_id: u64, + }, + Config { + id: String, + }, + Wallet { + id: String, + }, + VaultKey { + id: String, + }, + User { + id: String, + }, + RewardRound { + id: String, + #[serde(skip_serializing_if = "Option::is_none")] + election_id: Option, + }, + Recipient { + id: String, + }, + System, } diff --git a/src/node-control/service/src/elections/runner.rs b/src/node-control/service/src/elections/runner.rs index 792880a1..f17928d9 100644 --- a/src/node-control/service/src/elections/runner.rs +++ b/src/node-control/service/src/elections/runner.rs @@ -11,17 +11,8 @@ use super::{ election_emulator::ParticipantStake, providers::{ElectionsProvider, ValidatorConfig, ValidatorEntry}, }; -use crate::audit::{ - AuditEvent, StakeSkipReason, - enums::{ - AuditActorKind, AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, - AuditSubjectKind, - }, - log::AuditLog, - participant::{AuditActor, AuditSubject}, -}; +use crate::audit::{AuditEvent, StakeSkipReason, log::AuditLog, participant::AuditActor}; use anyhow::Context as _; -use chrono::Utc; use common::{ app_config::{BindingStatus, ElectionsConfig, NodeBinding, StakePolicy}, clock::{Clock, SystemClock}, @@ -42,7 +33,7 @@ use contracts::{ elector::PastElections, nominator, }; use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{HashMap, HashSet}, sync::Arc, time::Duration, }; @@ -51,7 +42,6 @@ use ton_block::{ config_params::{ConfigParam16, ConfigParam17}, write_boc, }; -use uuid::Uuid; #[cfg(test)] #[path = "runner_tests.rs"] @@ -368,27 +358,6 @@ impl ElectionRunner { } } - async fn audit_emit( - &self, - severity: AuditSeverity, - outcome: AuditOutcome, - election_id: Option, - node_id: Option, - message: &str, - payload: AuditEventPayload, - ) { - elections_audit_emit( - &self.audit, - severity, - outcome, - election_id, - node_id, - message, - payload, - ) - .await; - } - async fn audit_stake_skipped( &self, election_id: u64, @@ -559,19 +528,13 @@ impl ElectionRunner { let id = self.past_elections_cache_id; (id > 0).then_some(id) }); - elections_audit_emit( - &audit, - AuditSeverity::Error, - AuditOutcome::Failure, - election_id, - None, - "Election tick failed", - AuditEventPayload::ElectionsTickFailed { + audit + .record(AuditEvent::elections_tick_failed( + elections_audit_actor(), election_id, - error: format!("{e:#}"), - }, - ) - .await; + format!("{e:#}"), + )) + .await; tracing::error!("runner tick error: {:#}", e); } @@ -1105,19 +1068,14 @@ impl ElectionRunner { max_factor, }); node.key_id = key_id; - elections_audit_emit( - &audit, - AuditSeverity::Info, - AuditOutcome::Success, - Some(election_id), - Some(node_id.to_string()), - "Validator key generated", - AuditEventPayload::ElectionsKeyGenerated { + audit + .record(AuditEvent::elections_key_generated( + elections_audit_actor(), + node_id, election_id, - pubkey: Some(pubkey_hex), - }, - ) - .await; + Some(pubkey_hex), + )) + .await; if let Err(e) = Self::send_stake(node_id, node, stake, to_addr).await { let (required, available) = parse_balance_error(&e.to_string()); elections_audit_stake_skipped( @@ -1362,18 +1320,14 @@ impl ElectionRunner { }; if let Err(e) = send_result { - self.audit_emit( - AuditSeverity::Error, - AuditOutcome::Failure, - Some(election_id), - Some(node_id.to_string()), - "Withdraw process failed", - AuditEventPayload::ElectionsWithdrawProcessFailed { + self.audit + .record(AuditEvent::elections_withdraw_process_failed( + elections_audit_actor(), + node_id, election_id, - error: format!("{e:#}"), - }, - ) - .await; + format!("{e:#}"), + )) + .await; return Err(e); } @@ -1383,15 +1337,14 @@ impl ElectionRunner { WITHDRAW_PROCESS_LIMIT, election_id ); - self.audit_emit( - AuditSeverity::Info, - AuditOutcome::Success, - Some(election_id), - Some(node_id.to_string()), - "Withdraw requests processed", - AuditEventPayload::ElectionsWithdrawProcessed { election_id, tx_hash }, - ) - .await; + self.audit + .record(AuditEvent::elections_withdraw_processed( + elections_audit_actor(), + node_id, + election_id, + tx_hash, + )) + .await; Ok(true) } @@ -1427,19 +1380,15 @@ impl ElectionRunner { let tx_hash = format!("{:x}", msg.repr_hash()); let msg_boc = write_boc(&msg)?; node.api.send_boc(&msg_boc).await?; - self.audit_emit( - AuditSeverity::Info, - AuditOutcome::Success, - Some(election_id), - Some(node_id.to_string()), - "Stake recovered", - AuditEventPayload::ElectionsStakeRecovered { + self.audit + .record(AuditEvent::elections_stake_recovered( + elections_audit_actor(), + node_id, election_id, - amount_nanotons: nanotons_to_dec_string(amount), - tx_hash: Some(tx_hash), - }, - ) - .await; + nanotons_to_dec_string(amount), + Some(tx_hash), + )) + .await; } Ok(amount) } @@ -2078,46 +2027,7 @@ impl ElectionRunner { } fn elections_audit_actor() -> AuditActor { - AuditActor { - kind: AuditActorKind::Service, - id: Some("elections-task".into()), - role: None, - ip: None, - } -} - -async fn elections_audit_emit( - audit: &Arc, - severity: AuditSeverity, - outcome: AuditOutcome, - election_id: Option, - node_id: Option, - message: &str, - payload: AuditEventPayload, -) { - audit - .record(AuditEvent { - schema_version: 1, - id: Uuid::new_v4(), - ts: Utc::now(), - source: AuditSource::Elections, - severity, - outcome, - actor: elections_audit_actor(), - subject: AuditSubject { - kind: if node_id.is_some() { - AuditSubjectKind::Node - } else { - AuditSubjectKind::Elections - }, - id: node_id, - election_id, - labels: BTreeMap::new(), - }, - message: Some(message.into()), - payload, - }) - .await; + AuditActor::service("elections-task") } async fn elections_audit_stake_skipped( @@ -2128,21 +2038,16 @@ async fn elections_audit_stake_skipped( required_nanotons: Option, available_nanotons: Option, ) { - elections_audit_emit( - audit, - AuditSeverity::Warn, - AuditOutcome::Skipped, - Some(election_id), - Some(node_id.to_string()), - "Stake skipped", - AuditEventPayload::ElectionsStakeSkipped { + audit + .record(AuditEvent::elections_stake_skipped( + elections_audit_actor(), + node_id, election_id, reason, - required_nanotons: required_nanotons.map(nanotons_to_dec_string), - available_nanotons: available_nanotons.map(nanotons_to_dec_string), - }, - ) - .await; + required_nanotons.map(nanotons_to_dec_string), + available_nanotons.map(nanotons_to_dec_string), + )) + .await; } async fn elections_audit_stake_submitted( @@ -2155,22 +2060,17 @@ async fn elections_audit_stake_submitted( return; }; let submission_time = node.submission_time.unwrap_or_else(time_format::now); - elections_audit_emit( - audit, - AuditSeverity::Info, - AuditOutcome::Success, - Some(participant.election_id), - Some(node_id.to_string()), - "Stake submitted", - AuditEventPayload::ElectionsStakeSubmitted { - election_id: participant.election_id, - stake_nanotons: nanotons_to_dec_string(stake), - max_factor: participant.max_factor, - policy: node.stake_policy.to_string(), + audit + .record(AuditEvent::elections_stake_submitted( + elections_audit_actor(), + node_id, + participant.election_id, + nanotons_to_dec_string(stake), + participant.max_factor, + node.stake_policy.to_string(), submission_time, - }, - ) - .await; + )) + .await; } /// Parses `required=` / `available=` TON amounts from runner balance error strings. diff --git a/src/node-control/service/src/elections/runner_tests.rs b/src/node-control/service/src/elections/runner_tests.rs index 27dcd823..368aeda5 100644 --- a/src/node-control/service/src/elections/runner_tests.rs +++ b/src/node-control/service/src/elections/runner_tests.rs @@ -8,11 +8,8 @@ */ use super::*; use crate::audit::{ - AuditEvent, InMemoryAuditLog, - enums::{ - AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, AuditSubjectKind, - StakeSkipReason, - }, + AuditEvent, AuditTarget, InMemoryAuditLog, + enums::{AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, StakeSkipReason}, }; use common::{ app_config::{ElectionsConfig, NodeBinding, StakePolicy}, @@ -398,6 +395,17 @@ fn payload_stake_skipped(payload: &AuditEventPayload) -> &AuditEventPayload { } } +/// Asserts the event targets the expected node and election. +fn assert_node_target(target: &AuditTarget, node_id: &str, election_id: u64) { + match target { + AuditTarget::Node { id, election_id: eid } => { + assert_eq!(id, node_id, "unexpected node id in target"); + assert_eq!(*eid, Some(election_id), "unexpected election_id in target"); + } + other => panic!("expected node target, got {other:?}"), + } +} + struct TestHarness { elector_mock: MockElectorWrapperImpl, provider_mock: MockElectionsProviderImpl, @@ -3869,13 +3877,13 @@ async fn tick_emits_failed_on_error() { let failed = find_audit_event(&events, |p| matches!(p, AuditEventPayload::ElectionsTickFailed { .. })); - assert_eq!(failed.severity, AuditSeverity::Error); + assert_eq!(failed.payload.severity(), AuditSeverity::Error); assert_eq!(failed.outcome, AuditOutcome::Failure); - let AuditEventPayload::ElectionsTickFailed { election_id, error } = &failed.payload else { + assert_eq!(failed.target, AuditTarget::Elections { election_id: ELECTION_ID }); + let AuditEventPayload::ElectionsTickFailed { reason } = &failed.payload else { unreachable!(); }; - assert_eq!(*election_id, Some(ELECTION_ID)); - assert!(error.contains("simulated elections_info failure")); + assert!(reason.contains("simulated elections_info failure")); } #[tokio::test] @@ -3900,15 +3908,12 @@ async fn stake_submitted_event_contains_correct_payload() { }); payload_stake_submitted(&ev.payload); - assert_eq!(ev.source, AuditSource::Elections); - assert_eq!(ev.severity, AuditSeverity::Info); + assert_eq!(ev.payload.source(), AuditSource::Elections); + assert_eq!(ev.payload.severity(), AuditSeverity::Info); assert_eq!(ev.outcome, AuditOutcome::Success); - assert_eq!(ev.subject.kind, AuditSubjectKind::Node); - assert_eq!(ev.subject.id.as_deref(), Some(node_id)); - assert_eq!(ev.subject.election_id, Some(ELECTION_ID)); + assert_node_target(&ev.target, node_id, ELECTION_ID); let AuditEventPayload::ElectionsStakeSubmitted { - election_id, stake_nanotons, max_factor, policy, @@ -3917,7 +3922,6 @@ async fn stake_submitted_event_contains_correct_payload() { else { unreachable!(); }; - assert_eq!(*election_id, ELECTION_ID); assert_eq!(stake_nanotons, &expected_stake.to_string()); assert_eq!(*max_factor, 196608); assert_eq!(policy, "split50"); @@ -3954,9 +3958,9 @@ async fn stake_skipped_event_has_skipped_outcome_and_warn_severity() { }); payload_stake_skipped(&ev.payload); - assert_eq!(ev.severity, AuditSeverity::Warn); + assert_eq!(ev.payload.severity(), AuditSeverity::Warn); assert_eq!(ev.outcome, AuditOutcome::Skipped); - assert_eq!(ev.subject.election_id, Some(ELECTION_ID)); + assert_node_target(&ev.target, node_id, ELECTION_ID); } #[tokio::test] @@ -3993,12 +3997,11 @@ async fn withdraw_processed_emits_tx_hash() { }); assert_eq!(ev.outcome, AuditOutcome::Success); - assert_eq!(ev.subject.election_id, Some(ELECTION_ID)); + assert_node_target(&ev.target, node_id, ELECTION_ID); - let AuditEventPayload::ElectionsWithdrawProcessed { election_id, tx_hash } = &ev.payload else { + let AuditEventPayload::ElectionsWithdrawProcessed { tx_hash } = &ev.payload else { unreachable!(); }; - assert_eq!(*election_id, ELECTION_ID); assert!(!tx_hash.is_empty(), "tx_hash must be the sent message cell hash"); } @@ -4043,16 +4046,14 @@ async fn withdraw_failed_emits_error_string() { matches!(p, AuditEventPayload::ElectionsWithdrawProcessFailed { .. }) }); - assert_eq!(ev.severity, AuditSeverity::Error); + assert_eq!(ev.payload.severity(), AuditSeverity::Error); assert_eq!(ev.outcome, AuditOutcome::Failure); - assert_eq!(ev.subject.election_id, Some(ELECTION_ID)); + assert_node_target(&ev.target, node_id, ELECTION_ID); - let AuditEventPayload::ElectionsWithdrawProcessFailed { election_id, error } = &ev.payload - else { + let AuditEventPayload::ElectionsWithdrawProcessFailed { reason } = &ev.payload else { unreachable!(); }; - assert_eq!(*election_id, ELECTION_ID); - assert!(error.contains("simulated withdraw send_boc failure")); + assert!(reason.contains("simulated withdraw send_boc failure")); } #[tokio::test] @@ -4072,7 +4073,7 @@ async fn subject_election_id_populated_for_election_events() { let events = audit.drain(); let election_events: Vec<_> = events .iter() - .filter(|ev| ev.source == AuditSource::Elections) + .filter(|ev| ev.payload.source() == AuditSource::Elections) .filter(|ev| { matches!( ev.payload, @@ -4087,11 +4088,6 @@ async fn subject_election_id_populated_for_election_events() { "expected at least key_generated and stake_submitted audit events" ); for ev in election_events { - assert_eq!( - ev.subject.election_id, - Some(ELECTION_ID), - "event {:?} missing subject.election_id", - ev.payload - ); + assert_node_target(&ev.target, node_id, ELECTION_ID); } } From 327b0c052397fbd93fb77de1c70271fd0f84d0f0 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Thu, 4 Jun 2026 14:07:01 +0300 Subject: [PATCH 12/30] fix: copilot comments --- .../service/src/audit/jsonl_log.rs | 5 +- .../src/elections/adaptive_strategy.rs | 75 ++++++++-- .../service/src/elections/runner.rs | 137 ++++++++++-------- 3 files changed, 137 insertions(+), 80 deletions(-) diff --git a/src/node-control/service/src/audit/jsonl_log.rs b/src/node-control/service/src/audit/jsonl_log.rs index b9553969..7496a1f1 100644 --- a/src/node-control/service/src/audit/jsonl_log.rs +++ b/src/node-control/service/src/audit/jsonl_log.rs @@ -110,7 +110,10 @@ impl AuditLog for JsonlAuditLog { let timeout = Duration::from_millis(self.config.queue_full_timeout_ms); match tokio::time::timeout(timeout, self.sender.send(cmd)).await { Ok(Ok(())) => return, - _ => { + Ok(Err(_)) => { + tracing::error!("audit log channel closed; service likely shutting down"); + } + Err(_) => { self.dropped_events.fetch_add(1, Ordering::Relaxed); tracing::warn!( ?event_id, diff --git a/src/node-control/service/src/elections/adaptive_strategy.rs b/src/node-control/service/src/elections/adaptive_strategy.rs index 34db8669..f0385338 100644 --- a/src/node-control/service/src/elections/adaptive_strategy.rs +++ b/src/node-control/service/src/elections/adaptive_strategy.rs @@ -22,6 +22,22 @@ pub(crate) enum AdaptiveDeferReason { WaitingForParticipants, } +/// Why AdaptiveSplit50 returns zero stake after the defer window has passed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AdaptiveStakeZero { + /// Stake already meets min effective — no top-up this tick (not an error). + NoTopUpNeeded, + /// Free pool balance is below the required delta to min effective stake. + InsufficientFree { required: u64, available: u64 }, +} + +/// Outcome of [`calc_adaptive_stake`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AdaptiveStakeResult { + Stake(u64), + Zero(AdaptiveStakeZero), +} + /// Returns `true` if staking should proceed, `false` if we should defer (return 0). pub(crate) fn is_adaptive_split50_ready( node_id: &str, @@ -121,7 +137,7 @@ pub(crate) fn calc_adaptive_stake( cfg16: &ConfigParam16, cfg17: &ConfigParam17, prev_min_stake: Option, -) -> anyhow::Result { +) -> anyhow::Result { let min_validators = cfg16.min_validators.as_u16(); let max_validators = cfg16.max_validators.as_u16(); let max_stake_factor = cfg17.max_stake_factor; @@ -192,7 +208,7 @@ pub(crate) fn calc_adaptive_stake( nanotons_to_tons_f64(current_stake), nanotons_to_tons_f64(min_eff_stake) ); - return Ok(0); + return Ok(AdaptiveStakeResult::Zero(AdaptiveStakeZero::NoTopUpNeeded)); } // Insufficient funds guard — if the pool doesn't have enough free @@ -208,7 +224,10 @@ pub(crate) fn calc_adaptive_stake( nanotons_to_tons_f64(required), nanotons_to_tons_f64(min_eff_stake), ); - return Ok(0); + return Ok(AdaptiveStakeResult::Zero(AdaptiveStakeZero::InsufficientFree { + required, + available: free_balance, + })); } // Decide between staking half or min_eff_stake. @@ -232,9 +251,12 @@ pub(crate) fn calc_adaptive_stake( nanotons_to_tons_f64(stake), nanotons_to_tons_f64(free_balance), ); - return Ok(0); + return Ok(AdaptiveStakeResult::Zero(AdaptiveStakeZero::InsufficientFree { + required: stake, + available: free_balance, + })); } - Ok(stake) + Ok(AdaptiveStakeResult::Stake(stake)) } else { // half < min_eff — splitting is not viable. // Since half < min_eff, it follows that total < 2 * min_eff, @@ -247,13 +269,21 @@ pub(crate) fn calc_adaptive_stake( nanotons_to_tons_f64(min_eff_stake), nanotons_to_tons_f64(free_balance), ); - Ok(free_balance) + Ok(AdaptiveStakeResult::Stake(free_balance)) } } #[cfg(test)] mod tests { use super::*; + + fn stake_amount(r: AdaptiveStakeResult) -> u64 { + match r { + AdaptiveStakeResult::Stake(s) => s, + other => panic!("expected stake, got {other:?}"), + } + } + use ton_block::{ Coins, Number16, config_params::{ConfigParam16, ConfigParam17}, @@ -312,7 +342,7 @@ mod tests { .unwrap(); let half = total_balance / 2; - assert_eq!(result, half, "should stake half"); + assert_eq!(stake_amount(result), half, "should stake half"); } // ---- half < min_eff → stake all ---- @@ -346,7 +376,11 @@ mod tests { ) .unwrap(); - assert_eq!(result, free_balance, "should stake all free_balance when half < min_eff"); + assert_eq!( + stake_amount(result), + free_balance, + "should stake all free_balance when half < min_eff" + ); } // ---- current_stake >= min_eff → no top-up ---- @@ -375,7 +409,7 @@ mod tests { ) .unwrap(); - assert_eq!(result, 0, "should return 0 when current_stake >= min_eff"); + assert_eq!(result, AdaptiveStakeResult::Zero(AdaptiveStakeZero::NoTopUpNeeded)); } // ---- insufficient funds guard ---- @@ -405,7 +439,10 @@ mod tests { ) .unwrap(); - assert_eq!(result, 0, "should skip when free_balance < min_eff_stake"); + assert!(matches!( + result, + AdaptiveStakeResult::Zero(AdaptiveStakeZero::InsufficientFree { .. }) + )); } // ---- cap to free_balance when half > free_balance ---- @@ -438,7 +475,10 @@ mod tests { ) .unwrap(); - assert_eq!(result, 0, "should skip when free_balance < half (operator should top up pool)"); + assert!(matches!( + result, + AdaptiveStakeResult::Zero(AdaptiveStakeZero::InsufficientFree { .. }) + )); } // ---- curr vs prev selection ---- @@ -469,7 +509,7 @@ mod tests { .unwrap(); let half = total_balance / 2; - assert_eq!(result, half, "should use curr_min_eff and stake half"); + assert_eq!(stake_amount(result), half, "should use curr_min_eff and stake half"); } #[test] @@ -500,7 +540,8 @@ mod tests { .unwrap(); assert_eq!( - result, free_balance, + stake_amount(result), + free_balance, "should use curr_min_eff (not prev) and stake all when half < curr" ); } @@ -533,7 +574,11 @@ mod tests { .unwrap(); let half = total_balance / 2; - assert_eq!(result, half, "should fallback to prev_min_eff when not enough participants"); + assert_eq!( + stake_amount(result), + half, + "should fallback to prev_min_eff when not enough participants" + ); } // ---- both None → error ---- @@ -593,6 +638,6 @@ mod tests { .unwrap(); let expected = total_balance / 2 - current_stake; - assert_eq!(result, expected, "should top up to half"); + assert_eq!(stake_amount(result), expected, "should top up to half"); } } diff --git a/src/node-control/service/src/elections/runner.rs b/src/node-control/service/src/elections/runner.rs index f17928d9..e7689257 100644 --- a/src/node-control/service/src/elections/runner.rs +++ b/src/node-control/service/src/elections/runner.rs @@ -377,9 +377,10 @@ impl ElectionRunner { .await; } + /// Classifies stake==0 when [`calc_stake`] did not already supply an adaptive reason + /// (defer windows only; adaptive zero cases come from [`calc_adaptive_stake`]). async fn classify_stake_zero( - node: &mut Node, - elections_stake: u64, + node: &Node, configs: &ConfigParams<'_>, ctx: &StakeContext<'_>, ) -> (StakeSkipReason, Option, Option) { @@ -401,18 +402,22 @@ impl ElectionRunner { } }; } - let min_stake = configs.elections_info.min_stake; - let fee = ELECTOR_STAKE_FEE + NPOOL_COMPUTE_FEE; - let pool_free = node.stake_balance(fee).await.unwrap_or(0); - return ( - StakeSkipReason::InsufficientStakeFunds, - Some(min_stake), - Some(pool_free.saturating_add(elections_stake)), - ); } (StakeSkipReason::InsufficientStakeFunds, None, None) } + fn adaptive_zero_to_skip( + zero: adaptive_strategy::AdaptiveStakeZero, + ) -> Option<(StakeSkipReason, Option, Option)> { + use adaptive_strategy::AdaptiveStakeZero::*; + match zero { + NoTopUpNeeded => None, + InsufficientFree { required, available } => { + Some((StakeSkipReason::InsufficientStakeFunds, Some(required), Some(available))) + } + } + } + pub(crate) fn new( elections_config: &ElectionsConfig, bindings: &HashMap, @@ -979,37 +984,32 @@ impl ElectionRunner { node.accepted_stake_amount = Some(participant.stake); } let elections_stake = participant.as_ref().map(|p| p.stake).unwrap_or(0); - let stake = match Self::calc_stake(node, node_id, elections_stake, params, &stake_ctx).await - { - Ok(stake) => stake, - Err(e) => { - let (required, available) = parse_balance_error(&e.to_string()); + let (stake, adaptive_zero) = + match Self::calc_stake(node, node_id, elections_stake, params, &stake_ctx).await { + Ok(v) => v, + Err(e) => { + elections_audit_low_balance_if_parsed(&audit, election_id, node_id, &e).await; + return Err(e).context("stake calculation error"); + } + }; + + if stake == 0 { + tracing::info!("node [{}] skipping elections this tick (stake=0)", node_id); + let skip = match adaptive_zero { + Some(z) => Self::adaptive_zero_to_skip(z), + None => Some(Self::classify_stake_zero(node, params, &stake_ctx).await), + }; + if let Some((reason, required, available)) = skip { elections_audit_stake_skipped( &audit, election_id, node_id, - StakeSkipReason::LowWalletBalance, + reason, required, available, ) .await; - return Err(e).context("stake calculation error"); } - }; - - if stake == 0 { - tracing::info!("node [{}] skipping elections this tick (stake=0)", node_id); - let (reason, required, available) = - Self::classify_stake_zero(node, elections_stake, params, &stake_ctx).await; - elections_audit_stake_skipped( - &audit, - election_id, - node_id, - reason, - required, - available, - ) - .await; return Ok(()); } @@ -1077,16 +1077,7 @@ impl ElectionRunner { )) .await; if let Err(e) = Self::send_stake(node_id, node, stake, to_addr).await { - let (required, available) = parse_balance_error(&e.to_string()); - elections_audit_stake_skipped( - &audit, - election_id, - node_id, - StakeSkipReason::LowWalletBalance, - required, - available, - ) - .await; + elections_audit_low_balance_if_parsed(&audit, election_id, node_id, &e).await; return Err(e); } elections_audit_stake_submitted(&audit, node_id, node, stake).await; @@ -1116,14 +1107,11 @@ impl ElectionRunner { nanotons_to_tons_f64(stake), ); if let Err(e) = Self::send_stake(node_id, node, stake, to_addr).await { - let (required, available) = parse_balance_error(&e.to_string()); - elections_audit_stake_skipped( + elections_audit_low_balance_if_parsed( &audit, election_id, node_id, - StakeSkipReason::LowWalletBalance, - required, - available, + &e, ) .await; return Err(e); @@ -1138,16 +1126,8 @@ impl ElectionRunner { p.stake = stake; } if let Err(e) = Self::send_stake(node_id, node, stake, to_addr).await { - let (required, available) = parse_balance_error(&e.to_string()); - elections_audit_stake_skipped( - &audit, - election_id, - node_id, - StakeSkipReason::LowWalletBalance, - required, - available, - ) - .await; + elections_audit_low_balance_if_parsed(&audit, election_id, node_id, &e) + .await; return Err(e); } elections_audit_stake_submitted(&audit, node_id, node, stake).await; @@ -1468,13 +1448,16 @@ impl ElectionRunner { } /// Calculate stake for a node according to the stake policy. + /// + /// Returns `(amount, adaptive_zero)` where `adaptive_zero` is set when + /// AdaptiveSplit50 chose zero with a reason other than defer (sleep/waiting). async fn calc_stake( node: &mut Node, node_id: &str, elections_stake: u64, configs: &ConfigParams<'_>, ctx: &StakeContext<'_>, - ) -> anyhow::Result { + ) -> anyhow::Result<(u64, Option)> { let min_stake = configs.elections_info.min_stake; tracing::info!("node [{}] calc stake", node_id); let fee = ELECTOR_STAKE_FEE + NPOOL_COMPUTE_FEE; @@ -1532,7 +1515,7 @@ impl ElectionRunner { node_id, node.stake_policy.to_string() ); - return Ok(total_balance); + return Ok((total_balance, None)); } match &node.stake_policy { @@ -1546,7 +1529,7 @@ impl ElectionRunner { ctx.sleep_pct, ctx.waiting_pct, ) { - return Ok(0); + return Ok((0, None)); } let current_stake = if node.stake_accepted { elections_stake } else { 0 }; let stakes: Vec<_> = configs @@ -1563,7 +1546,7 @@ impl ElectionRunner { }) .map(|p| ParticipantStake { stake: p.stake, max_factor: p.max_factor }) .collect(); - adaptive_strategy::calc_adaptive_stake( + let outcome = adaptive_strategy::calc_adaptive_stake( node_id, total_balance, pool_free_balance, @@ -1573,9 +1556,13 @@ impl ElectionRunner { configs.cfg16, configs.cfg17, ctx.prev_min_eff_stake, - ) + )?; + Ok(match outcome { + adaptive_strategy::AdaptiveStakeResult::Stake(s) => (s, None), + adaptive_strategy::AdaptiveStakeResult::Zero(z) => (0, Some(z)), + }) } - other => other.calculate_stake(min_stake, total_balance), + other => Ok((other.calculate_stake(min_stake, total_balance)?, None)), } } @@ -2050,6 +2037,28 @@ async fn elections_audit_stake_skipped( .await; } +/// Emits `LowWalletBalance` only when `err` matches the `required=` / `available=` format +/// (calc_stake or send_stake balance guards). Other failures are not audited as skip. +async fn elections_audit_low_balance_if_parsed( + audit: &Arc, + election_id: u64, + node_id: &str, + err: &(impl std::fmt::Display + ?Sized), +) { + let (required, available) = parse_balance_error(&err.to_string()); + if required.is_some() || available.is_some() { + elections_audit_stake_skipped( + audit, + election_id, + node_id, + StakeSkipReason::LowWalletBalance, + required, + available, + ) + .await; + } +} + async fn elections_audit_stake_submitted( audit: &Arc, node_id: &str, @@ -2077,7 +2086,7 @@ async fn elections_audit_stake_submitted( fn parse_balance_error(msg: &str) -> (Option, Option) { fn tons_to_nanotons(fragment: &str) -> Option { let tons: f64 = fragment.trim().parse().ok()?; - Some((tons * 1_000_000_000.0) as u64) + Some((tons * 1_000_000_000.0).round() as u64) } let required = msg .split("required=") From 705480be818af11168c1161efdd0a889caafb2fd Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Thu, 4 Jun 2026 18:23:11 +0300 Subject: [PATCH 13/30] fix:fmt --- src/node-control/service/src/audit/jsonl_writer.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/node-control/service/src/audit/jsonl_writer.rs b/src/node-control/service/src/audit/jsonl_writer.rs index 7f931f08..5b6ea1d2 100644 --- a/src/node-control/service/src/audit/jsonl_writer.rs +++ b/src/node-control/service/src/audit/jsonl_writer.rs @@ -7,8 +7,7 @@ * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ use crate::audit::{ - AuditEvent, AuditFileHeader, AuditLogConfig, - enums::AuditEventPayload, + AuditEvent, AuditFileHeader, AuditLogConfig, enums::AuditEventPayload, jsonl_log::AuditInitError, }; use chrono::Utc; From a9d6557a9d44598fee33acb7aa3057d0a7f97c3d Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Thu, 4 Jun 2026 19:35:32 +0300 Subject: [PATCH 14/30] fix: delete unused --- src/node-control/service/src/audit/enums.rs | 9 --------- src/node-control/service/src/audit/participant.rs | 10 ---------- 2 files changed, 19 deletions(-) diff --git a/src/node-control/service/src/audit/enums.rs b/src/node-control/service/src/audit/enums.rs index 7ede2ea3..d655ed8f 100644 --- a/src/node-control/service/src/audit/enums.rs +++ b/src/node-control/service/src/audit/enums.rs @@ -8,17 +8,10 @@ */ use serde::{Deserialize, Serialize}; -/// Typed event payload. The wire shape is -/// `{ "event_type": "elections.stake_submitted", "data": { ... } }`. -/// -/// `election_id` is intentionally absent from the variants: it lives on the -/// event `target` (`AuditTarget::Node { election_id }` / -/// `AuditTarget::Elections { election_id }`), so it is never duplicated. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "event_type", content = "data", rename_all = "snake_case")] #[non_exhaustive] pub enum AuditEventPayload { - // ── elections ────────────────────────────────────────────────────────── #[serde(rename = "elections.tick_failed")] ElectionsTickFailed { reason: String }, @@ -137,8 +130,6 @@ pub enum AuditOutcome { Skipped, } -// ── Derived properties — kept in code, never serialized onto the wire ──────── - /// Log-level-like severity, derived from the event type at the display layer. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AuditSeverity { diff --git a/src/node-control/service/src/audit/participant.rs b/src/node-control/service/src/audit/participant.rs index fe054ef0..c3acae67 100644 --- a/src/node-control/service/src/audit/participant.rs +++ b/src/node-control/service/src/audit/participant.rs @@ -8,18 +8,12 @@ */ use serde::{Deserialize, Serialize}; -/// Who initiated the action. Internally tagged: the wire shape is -/// `{ "kind": "...", ...variant fields }`, so illegal field combinations -/// (e.g. a `role` on a scheduler) are unrepresentable. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum AuditActor { Service { id: String, }, - Scheduler { - id: String, - }, User { id: String, #[serde(skip_serializing_if = "Option::is_none")] @@ -35,10 +29,6 @@ impl AuditActor { Self::Service { id: id.into() } } - pub fn scheduler(id: impl Into) -> Self { - Self::Scheduler { id: id.into() } - } - pub fn user(id: impl Into, role: Option, ip: Option) -> Self { Self::User { id: id.into(), role, ip } } From 9f435847e3c0716732ce317818361a972526d891 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Thu, 4 Jun 2026 20:14:46 +0300 Subject: [PATCH 15/30] fix: delete scheduler as an actor --- src/node-control/service/src/audit/event.rs | 7 ++----- src/node-control/service/src/audit/jsonl_writer.rs | 5 +---- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index 6d9f050e..446e2309 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -427,11 +427,8 @@ mod tests { ); assert_eq!(skipped.outcome, AuditOutcome::Skipped); - let failed = AuditEvent::elections_tick_failed( - AuditActor::scheduler("elections-task"), - None, - "boom", - ); + let failed = + AuditEvent::elections_tick_failed(AuditActor::service("elections-task"), None, "boom"); assert_eq!(failed.outcome, AuditOutcome::Failure); assert_eq!(failed.target, AuditTarget::System); } diff --git a/src/node-control/service/src/audit/jsonl_writer.rs b/src/node-control/service/src/audit/jsonl_writer.rs index 5b6ea1d2..9ffecfb6 100644 --- a/src/node-control/service/src/audit/jsonl_writer.rs +++ b/src/node-control/service/src/audit/jsonl_writer.rs @@ -6,10 +6,7 @@ * * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ -use crate::audit::{ - AuditEvent, AuditFileHeader, AuditLogConfig, enums::AuditEventPayload, - jsonl_log::AuditInitError, -}; +use crate::audit::{AuditEvent, AuditFileHeader, AuditLogConfig, jsonl_log::AuditInitError}; use chrono::Utc; use std::{ sync::{ From ccea7b1459a99a600efbb63491177645ee7f9cec Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Thu, 4 Jun 2026 20:40:46 +0300 Subject: [PATCH 16/30] feat(nodectl): emit REST API audit events with AuditActorBuilder --- .../service/src/audit/actor_builder.rs | 159 +++++++++ src/node-control/service/src/audit/enums.rs | 10 +- src/node-control/service/src/audit/event.rs | 88 +++-- .../service/src/audit/in_memory.rs | 9 +- .../service/src/audit/jsonl_log.rs | 28 +- .../service/src/audit/jsonl_writer.rs | 24 +- src/node-control/service/src/audit/log.rs | 6 +- src/node-control/service/src/audit/mod.rs | 2 + .../service/src/audit/participant.rs | 6 +- .../service/src/auth/middleware.rs | 35 +- src/node-control/service/src/auth/mod.rs | 17 + .../service/src/http/auth_tests.rs | 78 ++++- .../service/src/http/config_handlers.rs | 313 +++++++++++++++++- .../service/src/http/config_handlers_tests.rs | 42 ++- .../src/http/entity_crud_handlers_tests.rs | 6 +- .../service/src/http/http_server_task.rs | 76 ++++- src/node-control/service/src/http/mod.rs | 1 + .../service/src/http/rest_audit.rs | 214 ++++++++++++ 18 files changed, 1031 insertions(+), 83 deletions(-) create mode 100644 src/node-control/service/src/audit/actor_builder.rs create mode 100644 src/node-control/service/src/http/rest_audit.rs diff --git a/src/node-control/service/src/audit/actor_builder.rs b/src/node-control/service/src/audit/actor_builder.rs new file mode 100644 index 00000000..f44b6e67 --- /dev/null +++ b/src/node-control/service/src/audit/actor_builder.rs @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use super::participant::AuditActor; +use crate::runtime_config::{RuntimeConfig, RuntimeConfigStore}; +use std::{net::IpAddr, sync::Arc}; + +/// Builds [`AuditActor`] values with REST PII policy (`record_client_ip`, `ip_anonymize`) +/// applied in one place. Handlers must not construct [`AuditActor::User`] directly. +pub struct AuditActorBuilder { + runtime_cfg: Arc, +} + +impl AuditActorBuilder { + pub fn new(runtime_cfg: Arc) -> Self { + Self { runtime_cfg } + } + + /// REST-authenticated user; client IP is attached only when enabled in live config. + /// This is the sole entry point for `actor.ip`, so the PII policy lives in one place. + pub fn rest_user( + &self, + username: impl Into, + role: impl Into, + client_ip: Option, + ) -> AuditActor { + AuditActor::user(username, Some(role.into()), client_ip.and_then(|ip| self.record_ip(ip))) + } + + fn record_ip(&self, ip: IpAddr) -> Option { + let cfg = self.runtime_cfg.get(); + if !cfg.audit_log.record_client_ip { + return None; + } + Some(if cfg.audit_log.ip_anonymize { anonymize_ip(ip) } else { ip.to_string() }) + } +} + +/// Parses the client IP from `x-forwarded-for` (first hop) when present and valid. +pub fn client_ip_from_headers(headers: &axum::http::HeaderMap) -> Option { + let ip_str = headers + .get("x-forwarded-for") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.split(',').next()) + .map(str::trim) + .filter(|v| !v.is_empty())?; + ip_str.parse().ok() +} + +fn anonymize_ip(ip: IpAddr) -> String { + match ip { + IpAddr::V4(v4) => { + let o = v4.octets(); + format!("{}.{}.{}.0", o[0], o[1], o[2]) + } + IpAddr::V6(v6) => { + let segs = v6.segments(); + format!("{:x}:{:x}:{:x}:0:0:0:0:0", segs[0], segs[1], segs[2]) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use common::app_config::AppConfig; + use std::collections::HashMap; + + fn test_app_cfg(record_client_ip: bool, ip_anonymize: bool) -> Arc { + Arc::new(AppConfig { + nodes: HashMap::new(), + wallets: HashMap::new(), + pools: HashMap::new(), + bindings: HashMap::new(), + ton_http_api: Default::default(), + http: Default::default(), + elections: None, + voting: None, + master_wallet: None, + tick_interval: 30, + automation: Default::default(), + log: None, + audit_log: common::app_config::AuditLogConfig { + record_client_ip, + ip_anonymize, + ..common::app_config::AuditLogConfig::default() + }, + }) + } + + fn builder(record_client_ip: bool, ip_anonymize: bool) -> AuditActorBuilder { + AuditActorBuilder::new(Arc::new(RuntimeConfigStore::from_app_config(test_app_cfg( + record_client_ip, + ip_anonymize, + )))) + } + + fn user_ip(actor: &AuditActor) -> Option<&str> { + let AuditActor::User { ip, .. } = actor else { + panic!("expected user actor"); + }; + ip.as_deref() + } + + #[test] + fn service_actor_has_no_ip() { + // Non-REST actors never carry an IP; they bypass the builder entirely. + assert_eq!( + AuditActor::service("http-task"), + AuditActor::Service { id: "http-task".into() } + ); + } + + #[test] + fn rest_user_omits_ip_when_record_disabled() { + let b = builder(false, true); + let actor = b.rest_user("alice", "operator", Some("203.0.113.10".parse().unwrap())); + assert_eq!(user_ip(&actor), None); + } + + #[test] + fn rest_user_keeps_ip_when_record_enabled_no_anonymize() { + let b = builder(true, false); + let actor = b.rest_user("alice", "operator", Some("203.0.113.10".parse().unwrap())); + assert_eq!(user_ip(&actor), Some("203.0.113.10")); + } + + #[test] + fn rest_user_anonymizes_ipv4_last_octet() { + let b = builder(true, true); + let actor = b.rest_user("alice", "operator", Some("203.0.113.10".parse().unwrap())); + assert_eq!(user_ip(&actor), Some("203.0.113.0")); + } + + #[test] + fn rest_user_anonymizes_ipv6_last_segments() { + let b = builder(true, true); + let ip: IpAddr = "2001:db8:85a3:8d3:1319:8a2e:370:7348".parse().unwrap(); + let actor = b.rest_user("alice", "operator", Some(ip)); + assert_eq!(user_ip(&actor), Some("2001:db8:85a3:0:0:0:0:0")); + } + + #[test] + fn client_ip_from_headers_parses_forwarded_for() { + let mut headers = axum::http::HeaderMap::new(); + headers.insert("x-forwarded-for", "10.0.0.5, 10.0.0.6".parse().unwrap()); + assert_eq!(client_ip_from_headers(&headers), Some("10.0.0.5".parse().unwrap())); + } + + #[test] + fn client_ip_from_headers_missing_returns_none() { + assert_eq!(client_ip_from_headers(&axum::http::HeaderMap::new()), None); + } +} diff --git a/src/node-control/service/src/audit/enums.rs b/src/node-control/service/src/audit/enums.rs index d655ed8f..245a65ef 100644 --- a/src/node-control/service/src/audit/enums.rs +++ b/src/node-control/service/src/audit/enums.rs @@ -6,9 +6,7 @@ * * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] #[serde(tag = "event_type", content = "data", rename_all = "snake_case")] #[non_exhaustive] pub enum AuditEventPayload { @@ -98,7 +96,7 @@ pub enum AuditEventPayload { SystemAuditEventsDropped { dropped_events: u64, reason: String }, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "snake_case")] #[non_exhaustive] pub enum StakeSkipReason { @@ -114,7 +112,7 @@ pub enum StakeSkipReason { /// A single typed field change for `rest_api.config_updated`. Replaces the /// previous free-form `serde_json::Value` diff. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct ConfigFieldChange { /// Dotted path, e.g. `elections.sleep_period_pct`. pub field: String, @@ -122,7 +120,7 @@ pub struct ConfigFieldChange { pub new: serde_json::Value, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "snake_case")] pub enum AuditOutcome { Success, diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index 446e2309..410836fd 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -10,30 +10,29 @@ use crate::audit::{ enums::{AuditEventPayload, AuditOutcome, StakeSkipReason}, participant::{AuditActor, AuditTarget}, }; -use chrono::{DateTime, SecondsFormat, Utc}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use uuid::Uuid; - /// Renders timestamps as RFC3339 with millisecond precision and a trailing `Z` /// (e.g. `2026-05-22T12:10:30.123Z`), used for `ts` and `started_at`. mod ts_millis_rfc3339 { - use super::*; - - pub fn serialize(ts: &DateTime, s: S) -> Result { - s.serialize_str(&ts.to_rfc3339_opts(SecondsFormat::Millis, true)) + pub fn serialize( + ts: &chrono::DateTime, + s: S, + ) -> Result { + s.serialize_str(&ts.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)) } - pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { - let raw = String::deserialize(d)?; - DateTime::parse_from_rfc3339(&raw) - .map(|dt| dt.with_timezone(&Utc)) + pub fn deserialize<'de, D: serde::Deserializer<'de>>( + d: D, + ) -> Result, D::Error> { + let raw = ::deserialize(d)?; + chrono::DateTime::parse_from_rfc3339(&raw) + .map(|dt| dt.with_timezone(&chrono::Utc)) .map_err(serde::de::Error::custom) } } /// First JSONL line of every (rotated) audit file. Readers distinguish it from /// events by the absence of an `event_type` field. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct AuditFileHeader { pub schema_version: u16, /// Logical service name, e.g. `"nodectl"`. @@ -42,7 +41,7 @@ pub struct AuditFileHeader { pub service_version: String, pub host: String, #[serde(with = "ts_millis_rfc3339")] - pub started_at: DateTime, + pub started_at: chrono::DateTime, } /// A single audit record. @@ -51,12 +50,12 @@ pub struct AuditFileHeader { /// (`event_type` + `data`), `actor`, `target`. `severity`/`source` are derived /// from the payload at the display layer and `schema_version` lives in /// [`AuditFileHeader`], so none of them are stored per event. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct AuditEvent { /// UUID v7 — sortable by creation time. - pub id: Uuid, + pub id: uuid::Uuid, #[serde(with = "ts_millis_rfc3339")] - pub ts: DateTime, + pub ts: chrono::DateTime, pub outcome: AuditOutcome, #[serde(flatten)] pub payload: AuditEventPayload, @@ -74,7 +73,7 @@ impl AuditEvent { outcome: AuditOutcome, payload: AuditEventPayload, ) -> Self { - Self { id: Uuid::now_v7(), ts: Utc::now(), outcome, payload, actor, target } + Self { id: uuid::Uuid::now_v7(), ts: chrono::Utc::now(), outcome, payload, actor, target } } /// `target` for a per-node election event: always `Node { election_id }`. @@ -223,6 +222,55 @@ impl AuditEvent { }, ) } + + pub fn rest_api_config_updated( + actor: AuditActor, + config_id: impl Into, + operation: impl Into, + changes: Vec, + ) -> Self { + Self::new( + actor, + AuditTarget::Config { id: config_id.into() }, + AuditOutcome::Success, + AuditEventPayload::RestApiConfigUpdated { operation: operation.into(), changes }, + ) + } + + pub fn rest_api_auth_login_success(actor: AuditActor, user_id: impl Into) -> Self { + Self::new( + actor, + AuditTarget::User { id: user_id.into() }, + AuditOutcome::Success, + AuditEventPayload::RestApiAuthLoginSuccess {}, + ) + } + + pub fn rest_api_auth_login_rejected( + actor: AuditActor, + user_id: impl Into, + reason: impl Into, + ) -> Self { + Self::new( + actor, + AuditTarget::User { id: user_id.into() }, + AuditOutcome::Failure, + AuditEventPayload::RestApiAuthLoginRejected { reason: reason.into() }, + ) + } + + pub fn rest_api_token_rejected( + actor: AuditActor, + user_id: impl Into, + reason: impl Into, + ) -> Self { + Self::new( + actor, + AuditTarget::User { id: user_id.into() }, + AuditOutcome::Failure, + AuditEventPayload::RestApiTokenRejected { reason: reason.into() }, + ) + } } #[cfg(test)] @@ -235,11 +283,11 @@ mod tests { const FIXTURE_ID: &str = "9b6c2b5a-9f9d-4a9f-bc31-9a89b0e9d111"; const FIXTURE_TS: &str = "2026-05-22T12:10:30.123Z"; - fn fixture_id() -> Uuid { + fn fixture_id() -> uuid::Uuid { FIXTURE_ID.parse().unwrap() } - fn fixture_ts() -> DateTime { + fn fixture_ts() -> chrono::DateTime { FIXTURE_TS.parse().unwrap() } diff --git a/src/node-control/service/src/audit/in_memory.rs b/src/node-control/service/src/audit/in_memory.rs index 2de9cbba..01bb88f0 100644 --- a/src/node-control/service/src/audit/in_memory.rs +++ b/src/node-control/service/src/audit/in_memory.rs @@ -7,17 +7,14 @@ * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ use crate::audit::{AuditEvent, log::AuditLog}; -use async_trait::async_trait; -use std::sync::Mutex; - /// Captures audit events in memory for unit tests. pub struct InMemoryAuditLog { - pub events: Mutex>, + pub events: std::sync::Mutex>, } impl InMemoryAuditLog { pub fn new() -> Self { - Self { events: Mutex::new(Vec::new()) } + Self { events: std::sync::Mutex::new(Vec::new()) } } pub fn drain(&self) -> Vec { @@ -31,7 +28,7 @@ impl Default for InMemoryAuditLog { } } -#[async_trait] +#[async_trait::async_trait] impl AuditLog for InMemoryAuditLog { async fn record(&self, event: AuditEvent) { self.events.lock().expect("in-memory audit lock").push(event); diff --git a/src/node-control/service/src/audit/jsonl_log.rs b/src/node-control/service/src/audit/jsonl_log.rs index 397f32b0..79b06f15 100644 --- a/src/node-control/service/src/audit/jsonl_log.rs +++ b/src/node-control/service/src/audit/jsonl_log.rs @@ -11,7 +11,6 @@ use crate::audit::{ jsonl_writer::{AuditCommand, AuditWriter}, log::AuditLog, }; -use async_trait::async_trait; use std::{ sync::{ Arc, Mutex, @@ -19,15 +18,10 @@ use std::{ }, time::Duration, }; -use thiserror::Error; -use tokio::{ - sync::{Mutex as AsyncMutex, mpsc, oneshot}, - task::JoinHandle, -}; const WRITER_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10); -#[derive(Debug, Error)] +#[derive(Debug, thiserror::Error)] pub enum AuditInitError { #[error("failed to create audit directory: {0}")] DirCreate(#[source] std::io::Error), @@ -40,15 +34,15 @@ pub enum AuditInitError { } pub struct JsonlAuditLog { - sender: mpsc::Sender, - shutdown_tx: Mutex>>, + sender: tokio::sync::mpsc::Sender, + shutdown_tx: Mutex>>, /// Serializes `shutdown()` so all concurrent callers observe completion. - shutdown_gate: AsyncMutex<()>, + shutdown_gate: tokio::sync::Mutex<()>, dropped_events: Arc, config: Arc, /// Writer task handle, consumed by the first [`AuditLog::shutdown`] call so /// callers can await the final drain/flush. `None` after shutdown. - writer: Mutex>>, + writer: Mutex>>, } impl JsonlAuditLog { @@ -97,14 +91,14 @@ impl JsonlAuditLog { dropped_events: Arc, writer: AuditWriter, ) -> Arc { - let (tx, rx) = mpsc::channel(config.queue_capacity.max(1)); - let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let (tx, rx) = tokio::sync::mpsc::channel(config.queue_capacity.max(1)); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); let handle = tokio::spawn(writer.run(rx, shutdown_rx)); Arc::new(Self { sender: tx, shutdown_tx: Mutex::new(Some(shutdown_tx)), - shutdown_gate: AsyncMutex::new(()), + shutdown_gate: tokio::sync::Mutex::new(()), dropped_events, config, writer: Mutex::new(Some(handle)), @@ -117,7 +111,7 @@ impl JsonlAuditLog { } } -#[async_trait] +#[async_trait::async_trait] impl AuditLog for JsonlAuditLog { async fn shutdown(&self) { let _shutdown_guard = self.shutdown_gate.lock().await; @@ -159,7 +153,7 @@ impl AuditLog for JsonlAuditLog { match self.sender.try_send(cmd) { Ok(()) => return, - Err(mpsc::error::TrySendError::Full(cmd)) => { + Err(tokio::sync::mpsc::error::TrySendError::Full(cmd)) => { let timeout = Duration::from_millis(self.config.queue_full_timeout_ms); match tokio::time::timeout(timeout, self.sender.send(cmd)).await { Ok(Ok(())) => return, @@ -176,7 +170,7 @@ impl AuditLog for JsonlAuditLog { } } } - Err(mpsc::error::TrySendError::Closed(_)) => { + Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { self.dropped_events.fetch_add(1, Ordering::Relaxed); tracing::error!("audit log channel closed; service likely shutting down"); } diff --git a/src/node-control/service/src/audit/jsonl_writer.rs b/src/node-control/service/src/audit/jsonl_writer.rs index 9ffecfb6..d0b12680 100644 --- a/src/node-control/service/src/audit/jsonl_writer.rs +++ b/src/node-control/service/src/audit/jsonl_writer.rs @@ -7,7 +7,6 @@ * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ use crate::audit::{AuditEvent, AuditFileHeader, AuditLogConfig, jsonl_log::AuditInitError}; -use chrono::Utc; use std::{ sync::{ Arc, @@ -15,7 +14,6 @@ use std::{ }, time::Duration, }; -use tokio::sync::{mpsc, oneshot}; /// Schema version stamped into the per-file [`AuditFileHeader`]. const AUDIT_SCHEMA_VERSION: u16 = 1; @@ -134,7 +132,7 @@ impl AuditWriter { service: "nodectl".into(), service_version: env!("CARGO_PKG_VERSION").into(), host: Self::hostname(), - started_at: Utc::now(), + started_at: chrono::Utc::now(), } } @@ -168,7 +166,7 @@ impl AuditWriter { async fn drain_pending_commands( &mut self, - rx: &mut mpsc::Receiver, + rx: &mut tokio::sync::mpsc::Receiver, buffered: &mut Vec, ) { let batch_max_events = self.config.batch_max_events.max(1); @@ -192,7 +190,7 @@ impl AuditWriter { async fn finish_shutdown( &mut self, - rx: &mut mpsc::Receiver, + rx: &mut tokio::sync::mpsc::Receiver, buffered: &mut Vec, ) { self.drain_pending_commands(rx, buffered).await; @@ -202,8 +200,8 @@ impl AuditWriter { pub(crate) async fn run( mut self, - mut rx: mpsc::Receiver, - mut shutdown_rx: oneshot::Receiver<()>, + mut rx: tokio::sync::mpsc::Receiver, + mut shutdown_rx: tokio::sync::oneshot::Receiver<()>, ) { let interval_ms = self.config.batch_interval_ms.max(1); let batch_max_events = self.config.batch_max_events.max(1); @@ -478,17 +476,17 @@ mod tests { f: F, ) -> (Arc, PathBuf) where - F: FnOnce(mpsc::Sender, PathBuf, Arc) -> Fut, + F: FnOnce(tokio::sync::mpsc::Sender, PathBuf, Arc) -> Fut, Fut: std::future::Future, { let config = Arc::new(config); let dropped = Arc::new(AtomicU64::new(0)); let path = config.path.clone(); - let (tx, rx) = mpsc::channel(config.queue_capacity); + let (tx, rx) = tokio::sync::mpsc::channel(config.queue_capacity); // Keep test shutdown signal unsent; tests stop the writer via // `AuditCommand::Shutdown` on the mpsc channel to avoid races with // producer commands. - let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>(); let writer = AuditWriter::open_with_write_delay(config, dropped.clone(), write_delay).await.unwrap(); @@ -505,15 +503,15 @@ mod tests { (dropped_out, path_out) } - async fn send_event(tx: &mpsc::Sender, event: AuditEvent) { + async fn send_event(tx: &tokio::sync::mpsc::Sender, event: AuditEvent) { tx.send(AuditCommand::Event(Box::new(event))).await.unwrap(); } - async fn flush(tx: &mpsc::Sender) { + async fn flush(tx: &tokio::sync::mpsc::Sender) { tx.send(AuditCommand::Flush).await.unwrap(); } - async fn stop(tx: &mpsc::Sender) { + async fn stop(tx: &tokio::sync::mpsc::Sender) { tx.send(AuditCommand::Shutdown).await.unwrap(); } diff --git a/src/node-control/service/src/audit/log.rs b/src/node-control/service/src/audit/log.rs index 9c95f684..3ef705c0 100644 --- a/src/node-control/service/src/audit/log.rs +++ b/src/node-control/service/src/audit/log.rs @@ -7,9 +7,7 @@ * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ use crate::audit::AuditEvent; -use async_trait::async_trait; - -#[async_trait] +#[async_trait::async_trait] pub trait AuditLog: Send + Sync { async fn record(&self, event: AuditEvent); async fn shutdown(&self) {} @@ -17,7 +15,7 @@ pub trait AuditLog: Send + Sync { pub struct NoopAuditLog; -#[async_trait] +#[async_trait::async_trait] impl AuditLog for NoopAuditLog { async fn record(&self, _event: AuditEvent) {} } diff --git a/src/node-control/service/src/audit/mod.rs b/src/node-control/service/src/audit/mod.rs index 11256d9e..0edbfaec 100644 --- a/src/node-control/service/src/audit/mod.rs +++ b/src/node-control/service/src/audit/mod.rs @@ -6,6 +6,7 @@ * * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ +pub mod actor_builder; pub mod enums; pub mod event; pub mod factory; @@ -16,6 +17,7 @@ pub mod jsonl_writer; pub mod log; pub mod participant; +pub use actor_builder::{AuditActorBuilder, client_ip_from_headers}; pub use common::app_config::AuditLogConfig; pub use enums::{ AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, ConfigFieldChange, StakeSkipReason, diff --git a/src/node-control/service/src/audit/participant.rs b/src/node-control/service/src/audit/participant.rs index c3acae67..c9b67a63 100644 --- a/src/node-control/service/src/audit/participant.rs +++ b/src/node-control/service/src/audit/participant.rs @@ -6,9 +6,7 @@ * * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum AuditActor { Service { @@ -41,7 +39,7 @@ impl AuditActor { /// What the action was applied to. Internally tagged so per-variant required /// fields are enforced by the type system; `#[non_exhaustive]` because new /// target kinds are expected as more producers are wired. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] #[non_exhaustive] pub enum AuditTarget { diff --git a/src/node-control/service/src/auth/middleware.rs b/src/node-control/service/src/auth/middleware.rs index 1da10b06..9bc3f15a 100644 --- a/src/node-control/service/src/auth/middleware.rs +++ b/src/node-control/service/src/auth/middleware.rs @@ -6,8 +6,11 @@ * * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ -use super::Role; -use crate::{http::http_server_task::AppState, runtime_config::RuntimeConfig}; +use super::{Claims, Role}; +use crate::{ + http::{http_server_task::AppState, rest_audit}, + runtime_config::RuntimeConfig, +}; use axum::{ body::Body, extract::State, @@ -49,11 +52,21 @@ async fn require_role_impl( next: Next, min_role: Role, ) -> Response { + let headers = req.headers().clone(); + // Check live config: when auth is not configured, pass through. // This allows auth to be enabled/disabled at runtime via config reload. { let cfg = state.runtime_cfg.get(); if cfg.http.auth.is_none() { + // Auth-disabled mode: inject a synthetic identity so mutation handlers + // can still require `Extension` without enabling JWT checks. + req.extensions_mut().insert(Claims { + sub: "anonymous".into(), + role: min_role, + iat: 0, + exp: u64::MAX, + }); return next.run(req).await; } } @@ -65,12 +78,23 @@ async fn require_role_impl( let token = match auth_header.and_then(|h| h.strip_prefix("Bearer ")) { Some(t) => t, - None => return unauthorized_response("missing or invalid Authorization header"), + None => { + rest_audit::record_token_rejected(&state, "unknown", "missing_authorization", &headers) + .await; + return unauthorized_response("missing or invalid Authorization header"); + } }; let claims = match jwt_auth.verify(token) { Ok(c) => c, - Err(_) => return unauthorized_response("invalid or expired token"), + Err(e) => { + let reason = e + .downcast_ref::() + .map(super::token_rejection_reason) + .unwrap_or("invalid_token"); + rest_audit::record_token_rejected(&state, "unknown", reason, &headers).await; + return unauthorized_response("invalid or expired token"); + } }; let user = match state.user_store.find_user(&claims.sub) { @@ -84,6 +108,7 @@ async fn require_role_impl( user = %claims.sub, "token rejected" ); + rest_audit::record_token_rejected(&state, &claims.sub, "missing_user", &headers).await; return unauthorized_response("invalid or expired token"); } }; @@ -99,6 +124,7 @@ async fn require_role_impl( current_role = %user.role, "token rejected" ); + rest_audit::record_token_rejected(&state, &claims.sub, "role_mismatch", &headers).await; return unauthorized_response("invalid or expired token"); } @@ -114,6 +140,7 @@ async fn require_role_impl( revoked_after = revoked_after, "token rejected" ); + rest_audit::record_token_rejected(&state, &claims.sub, "revoked", &headers).await; return unauthorized_response("invalid or expired token"); } } diff --git a/src/node-control/service/src/auth/mod.rs b/src/node-control/service/src/auth/mod.rs index d1479abb..e21f72cb 100644 --- a/src/node-control/service/src/auth/mod.rs +++ b/src/node-control/service/src/auth/mod.rs @@ -12,6 +12,23 @@ pub mod user_store; pub use common::app_config::Role; +/// Maps JWT verification failures to stable audit reason strings (no token material). +pub fn token_rejection_reason(err: &jsonwebtoken::errors::Error) -> &'static str { + use jsonwebtoken::errors::ErrorKind; + match err.kind() { + ErrorKind::ExpiredSignature => "expired", + ErrorKind::InvalidSignature => "signature_mismatch", + ErrorKind::InvalidAlgorithm => "invalid_algorithm", + ErrorKind::InvalidToken => "invalid_token", + ErrorKind::InvalidIssuer => "invalid_issuer", + ErrorKind::InvalidAudience => "invalid_audience", + ErrorKind::ImmatureSignature => "immature_signature", + ErrorKind::InvalidSubject => "invalid_subject", + ErrorKind::MissingRequiredClaim(_) => "missing_required_claim", + _ => "invalid_token", + } +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct Claims { /// Subject: authenticated username. diff --git a/src/node-control/service/src/http/auth_tests.rs b/src/node-control/service/src/http/auth_tests.rs index 0a24d8e4..4961028d 100644 --- a/src/node-control/service/src/http/auth_tests.rs +++ b/src/node-control/service/src/http/auth_tests.rs @@ -7,6 +7,7 @@ * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ use crate::{ + audit::{AuditActorBuilder, AuditEventPayload, in_memory::InMemoryAuditLog, log::AuditLog}, auth::{Role, jwt::JwtAuth, user_store::UserStore}, http::http_server_task::*, runtime_config::{RuntimeConfig, RuntimeConfigStore}, @@ -170,13 +171,33 @@ async fn state_with_auth() -> AppState { runtime_cfg: rt.clone(), elections_task: elections_task(rt.clone()), jwt_auth: test_jwt_auth().await, - user_store: Arc::new(UserStore::new(rt as Arc)), + user_store: Arc::new(UserStore::new(rt.clone() as Arc)), login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), audit: Arc::new(crate::audit::log::NoopAuditLog), + actor_builder: Arc::new(AuditActorBuilder::new(rt.clone())), } } +async fn state_with_auth_audited() -> (AppState, Arc) { + let cfg = auth_config(); + let rt = Arc::new(RuntimeConfigStore::from_app_config(app_cfg_with_auth(cfg.clone()))); + let audit_mem = Arc::new(InMemoryAuditLog::new()); + let audit: Arc = audit_mem.clone(); + let state = AppState { + store: Arc::new(SnapshotStore::new()), + runtime_cfg: rt.clone(), + elections_task: elections_task(rt.clone()), + jwt_auth: test_jwt_auth().await, + user_store: Arc::new(UserStore::new(rt.clone() as Arc)), + login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), + config_changed: Arc::new(tokio::sync::Notify::new()), + audit, + actor_builder: Arc::new(AuditActorBuilder::new(rt)), + }; + (state, audit_mem) +} + async fn state_no_auth() -> AppState { let rt = Arc::new(RuntimeConfigStore::from_app_config(app_cfg_no_auth())); AppState { @@ -188,6 +209,7 @@ async fn state_no_auth() -> AppState { login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), audit: Arc::new(crate::audit::log::NoopAuditLog), + actor_builder: Arc::new(AuditActorBuilder::new(rt)), } } @@ -681,3 +703,57 @@ async fn delete_user_via_rest_returns_404() { let resp = app(st).oneshot(req).await.unwrap(); assert_eq!(resp.status(), 404); } + +// --- Audit emission --- + +#[tokio::test] +async fn login_success_emits_audit_event() { + let (st, audit) = state_with_auth_audited().await; + let resp = app(st.clone()) + .oneshot(post_json( + "/auth/login", + &LoginRequest { username: "op".into(), password: "pass1".into() }, + )) + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let events = audit.drain(); + assert_eq!(events.len(), 1); + assert!(matches!(events[0].payload, AuditEventPayload::RestApiAuthLoginSuccess {})); + let body = serde_json::to_string(&events[0]).unwrap(); + assert!(!body.contains("pass1")); +} + +#[tokio::test] +async fn login_failure_emits_rejected_event_without_password() { + let (st, audit) = state_with_auth_audited().await; + let resp = app(st) + .oneshot(post_json( + "/auth/login", + &LoginRequest { username: "op".into(), password: "wrong".into() }, + )) + .await + .unwrap(); + assert_eq!(resp.status(), 401); + + let events = audit.drain(); + assert_eq!(events.len(), 1); + let AuditEventPayload::RestApiAuthLoginRejected { reason } = &events[0].payload else { + panic!("expected login rejected payload"); + }; + assert_eq!(reason, "invalid_credentials"); + let body = serde_json::to_string(&events[0]).unwrap(); + assert!(!body.contains("wrong")); +} + +#[tokio::test] +async fn invalid_token_emits_token_rejected_event() { + let (st, audit) = state_with_auth_audited().await; + let resp = app(st).oneshot(get_bearer("/v1/elections", "not.a.jwt")).await.unwrap(); + assert_eq!(resp.status(), 401); + + let events = audit.drain(); + assert_eq!(events.len(), 1); + assert!(matches!(events[0].payload, AuditEventPayload::RestApiTokenRejected { .. })); +} diff --git a/src/node-control/service/src/http/config_handlers.rs b/src/node-control/service/src/http/config_handlers.rs index 514a25bd..dd1e8561 100644 --- a/src/node-control/service/src/http/config_handlers.rs +++ b/src/node-control/service/src/http/config_handlers.rs @@ -6,8 +6,12 @@ * * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ -use super::http_server_task::{AppError, AppState}; +use super::{ + http_server_task::{AppError, AppState}, + rest_audit, +}; use crate::{ + auth::Claims, elections::providers::{DefaultElectionsProvider, ElectionsProvider}, runtime_config::{RuntimeConfig, TryUpdateSaveError, open_wallet}, }; @@ -1171,6 +1175,8 @@ fn merge_contracts_automation_update( )] pub async fn v1_contracts_automation_settings_update_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { let req = req.0; @@ -1178,9 +1184,11 @@ pub async fn v1_contracts_automation_settings_update_handler( return Err(AppError::bad_request("at least one setting is required")); } + let before = state.runtime_cfg.get().automation.clone(); + state .runtime_cfg - .try_update_and_save(move |cfg| { + .try_update_and_save(|cfg| { let next = merge_contracts_automation_update(&cfg.automation, &req); next.validate().map_err(|e| e.to_string())?; cfg.automation = next; @@ -1188,6 +1196,17 @@ pub async fn v1_contracts_automation_settings_update_handler( }) .map_err(map_try_update_save)?; + let changes = rest_audit::automation_settings_changes(&before, &req); + rest_audit::record_config_updated( + &state, + &claims, + &headers, + "automation", + "automation.settings_updated", + changes, + ) + .await; + state.config_changed.notify_one(); let config = state.runtime_cfg.get(); @@ -1453,6 +1472,8 @@ pub async fn v1_voting_proposals_inspect_handler( )] pub async fn v1_voting_proposals_add_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { let req = req.0; @@ -1495,6 +1516,18 @@ pub async fn v1_voting_proposals_add_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "voting", + "voting.proposals.add", + format!("voting.proposals.{hash_hex}"), + serde_json::Value::Null, + serde_json::json!({ "hash": hash_hex }), + ) + .await; + Ok(axum::Json(EntityRefResponse { ok: true, result: EntityRefDto { name: hash_hex } })) } @@ -1514,6 +1547,8 @@ pub async fn v1_voting_proposals_add_handler( )] pub async fn v1_voting_proposals_rm_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, axum::extract::Path(hash): axum::extract::Path, ) -> Result, AppError> { let hash_bytes = parse_voting_proposal_hash_hex(&hash)?; @@ -1539,6 +1574,18 @@ pub async fn v1_voting_proposals_rm_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "voting", + "voting.proposals.remove", + format!("voting.proposals.{hash_hex}"), + serde_json::json!({ "hash": hash_hex }), + serde_json::Value::Null, + ) + .await; + Ok(axum::Json(EntityRefResponse { ok: true, result: EntityRefDto { name: hash_hex } })) } @@ -1568,6 +1615,8 @@ pub async fn v1_voting_proposals_rm_handler( )] pub async fn v1_nodes_add_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { let req = req.0; @@ -1598,6 +1647,18 @@ pub async fn v1_nodes_add_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "nodes", + "nodes.add", + format!("nodes.{}", req.name), + serde_json::Value::Null, + serde_json::json!({ "name": req.name }), + ) + .await; + Ok(axum::Json(EntityRefResponse { ok: true, result: EntityRefDto { name: req.name } })) } @@ -1616,6 +1677,8 @@ pub async fn v1_nodes_add_handler( )] pub async fn v1_nodes_rm_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, axum::extract::Path(name): axum::extract::Path, ) -> Result, AppError> { let cfg = state.runtime_cfg.get(); @@ -1636,6 +1699,18 @@ pub async fn v1_nodes_rm_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "nodes", + "nodes.remove", + format!("nodes.{name}"), + serde_json::json!({ "name": name }), + serde_json::Value::Null, + ) + .await; + Ok(axum::Json(EntityRefResponse { ok: true, result: EntityRefDto { name } })) } @@ -1654,6 +1729,8 @@ pub async fn v1_nodes_rm_handler( )] pub async fn v1_wallets_add_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { let req = req.0; @@ -1686,6 +1763,18 @@ pub async fn v1_wallets_add_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "wallets", + "wallets.add", + format!("wallets.{}", req.name), + serde_json::Value::Null, + serde_json::json!({ "name": req.name }), + ) + .await; + Ok(axum::Json(EntityRefResponse { ok: true, result: EntityRefDto { name: req.name } })) } @@ -1705,6 +1794,8 @@ pub async fn v1_wallets_add_handler( )] pub async fn v1_wallets_rm_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, axum::extract::Path(name): axum::extract::Path, ) -> Result, AppError> { if name == MASTER_WALLET_RESERVED_NAME { @@ -1729,6 +1820,18 @@ pub async fn v1_wallets_rm_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "wallets", + "wallets.remove", + format!("wallets.{name}"), + serde_json::json!({ "name": name }), + serde_json::Value::Null, + ) + .await; + Ok(axum::Json(EntityRefResponse { ok: true, result: EntityRefDto { name } })) } @@ -1747,6 +1850,8 @@ pub async fn v1_wallets_rm_handler( )] pub async fn v1_pools_add_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { let req = req.0; @@ -1782,6 +1887,18 @@ pub async fn v1_pools_add_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "pools", + "pools.add", + format!("pools.{}", req.name), + serde_json::Value::Null, + serde_json::json!({ "name": req.name }), + ) + .await; + Ok(axum::Json(EntityRefResponse { ok: true, result: EntityRefDto { name: req.name } })) } @@ -1800,6 +1917,8 @@ pub async fn v1_pools_add_handler( )] pub async fn v1_pools_add_core_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { let req = req.0; @@ -1914,6 +2033,18 @@ pub async fn v1_pools_add_core_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "pools", + "pools.core.add", + format!("pools.{}.{}", req.name, req.slot), + serde_json::Value::Null, + serde_json::json!({ "name": req.name, "slot": req.slot }), + ) + .await; + Ok(axum::Json(EntityRefResponse { ok: true, result: EntityRefDto { name: req.name } })) } @@ -1933,6 +2064,8 @@ pub async fn v1_pools_add_core_handler( )] pub async fn v1_pools_rm_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, axum::extract::Path(name): axum::extract::Path, ) -> Result, AppError> { let cfg = state.runtime_cfg.get(); @@ -1955,6 +2088,18 @@ pub async fn v1_pools_rm_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "pools", + "pools.remove", + format!("pools.{name}"), + serde_json::json!({ "name": name }), + serde_json::Value::Null, + ) + .await; + Ok(axum::Json(EntityRefResponse { ok: true, result: EntityRefDto { name } })) } @@ -1973,6 +2118,8 @@ pub async fn v1_pools_rm_handler( )] pub async fn v1_bindings_add_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { let req = req.0; @@ -2019,6 +2166,18 @@ pub async fn v1_bindings_add_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "bindings", + "bindings.add", + format!("bindings.{}", req.node), + serde_json::Value::Null, + serde_json::json!({ "node": req.node }), + ) + .await; + Ok(axum::Json(EntityRefResponse { ok: true, result: EntityRefDto { name: req.node } })) } @@ -2038,6 +2197,8 @@ pub async fn v1_bindings_add_handler( )] pub async fn v1_bindings_rm_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, axum::extract::Path(node): axum::extract::Path, ) -> Result, AppError> { let cfg = state.runtime_cfg.get(); @@ -2061,6 +2222,18 @@ pub async fn v1_bindings_rm_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "bindings", + "bindings.remove", + format!("bindings.{node}"), + serde_json::json!({ "node": node }), + serde_json::Value::Null, + ) + .await; + Ok(axum::Json(EntityRefResponse { ok: true, result: EntityRefDto { name: node } })) } @@ -2082,6 +2255,8 @@ pub async fn v1_bindings_rm_handler( )] pub async fn v1_elections_settings_update_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { let req = req.0; @@ -2147,13 +2322,16 @@ pub async fn v1_elections_settings_update_handler( .ok() .and_then(|p| extract_max_factor(p).ok()); + let policy_changed = req.policy.is_some() || req.reset; let policy = req.policy; - let node = req.node; + let node = req.node.clone(); let reset = req.reset; let tick_interval = req.tick_interval; let max_factor = req.max_factor; let sleep_period_pct = req.sleep_period_pct; let waiting_period_pct = req.waiting_period_pct; + let before = elections.clone(); + let node_for_audit = node.clone(); state .runtime_cfg @@ -2192,6 +2370,27 @@ pub async fn v1_elections_settings_update_handler( }) .map_err(map_try_update_save)?; + let after = state.runtime_cfg.get().elections.as_ref().expect("validated above").clone(); + let changes = rest_audit::elections_settings_changes( + &before, + &after, + sleep_period_pct, + waiting_period_pct, + tick_interval, + max_factor, + policy_changed, + node_for_audit.as_deref(), + ); + rest_audit::record_config_updated( + &state, + &claims, + &headers, + "elections", + "elections.settings_updated", + changes, + ) + .await; + // Restart elections task so changes take effect immediately. let task = state.elections_task.clone(); tokio::spawn(async move { @@ -2228,6 +2427,8 @@ pub async fn v1_elections_settings_update_handler( )] pub async fn v1_ton_http_api_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { let req = req.0; @@ -2237,6 +2438,7 @@ pub async fn v1_ton_http_api_handler( return Err(AppError::bad_request("at least one non-empty url is required")); } + let before_count = state.runtime_cfg.get().ton_http_api.endpoints().len(); let api_key = req.api_key; let append = req.append; state @@ -2265,6 +2467,21 @@ pub async fn v1_ton_http_api_handler( state.config_changed.notify_one(); let endpoints = state.runtime_cfg.get().ton_http_api.endpoints(); + let operation = if append { "ton_http_api.append" } else { "ton_http_api.replace" }; + rest_audit::record_config_updated( + &state, + &claims, + &headers, + "ton_http_api", + operation, + vec![rest_audit::config_field( + "ton_http_api.endpoint_count", + serde_json::json!(before_count), + serde_json::json!(endpoints.len()), + )], + ) + .await; + Ok(axum::Json(TonHttpApiResponse { ok: true, result: TonHttpApiResult { endpoints } })) } @@ -2282,6 +2499,8 @@ pub async fn v1_ton_http_api_handler( )] pub async fn v1_log_set_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { let req = req.0; @@ -2324,6 +2543,12 @@ pub async fn v1_log_set_handler( } } + let before = state.runtime_cfg.get().log.clone().unwrap_or_default(); + let rotation_set = req.rotation.is_some(); + let output_set = req.output.is_some(); + let max_size_set = req.max_size_mb.is_some(); + let max_files_set = req.max_files.is_some(); + state .runtime_cfg .update_and_save(|cfg| { @@ -2349,6 +2574,60 @@ pub async fn v1_log_set_handler( }) .map_err(|e| AppError::internal(e.to_string()))?; + let after = state.runtime_cfg.get().log.clone().unwrap_or_default(); + let mut changes = Vec::new(); + if level_str.is_some() { + changes.push(rest_audit::config_field( + "log.level", + serde_json::json!(before.level.to_string()), + serde_json::json!(after.level.to_string()), + )); + } + if path_str.is_some() { + changes.push(rest_audit::config_field( + "log.path", + serde_json::json!(before.path), + serde_json::json!(after.path), + )); + } + if rotation_set { + changes.push(rest_audit::config_field( + "log.rotation", + serde_json::json!(before.rotation), + serde_json::json!(after.rotation), + )); + } + if output_set { + changes.push(rest_audit::config_field( + "log.output", + serde_json::json!(before.output), + serde_json::json!(after.output), + )); + } + if max_size_set { + changes.push(rest_audit::config_field( + "log.max_size_mb", + serde_json::json!(before.max_size_mb), + serde_json::json!(after.max_size_mb), + )); + } + if max_files_set { + changes.push(rest_audit::config_field( + "log.max_files", + serde_json::json!(before.max_files), + serde_json::json!(after.max_files), + )); + } + rest_audit::record_config_updated( + &state, + &claims, + &headers, + "log", + "log.settings_updated", + changes, + ) + .await; + let config = state.runtime_cfg.get(); let log = config.log.as_ref().cloned().unwrap_or_default(); let dto = log_config_to_dto(&log); @@ -2373,6 +2652,8 @@ pub async fn v1_log_set_handler( )] pub async fn v1_elections_static_adnl_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { let node_name = req.0.node; @@ -2411,6 +2692,18 @@ pub async fn v1_elections_static_adnl_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "elections", + "elections.static_adnl.set", + format!("elections.static_adnl.{node_name}"), + serde_json::Value::Null, + serde_json::json!({ "node": node_name }), + ) + .await; + tracing::info!("node [{}] static ADNL address set: {}", node_name, adnl_b64); Ok(axum::Json(StaticAdnlResponse { ok: true, result: StaticAdnlDto { adnl_addr: adnl_b64 } })) } @@ -2429,6 +2722,8 @@ pub async fn v1_elections_static_adnl_handler( )] pub async fn v1_elections_static_adnl_disable_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, axum::extract::Path(node_name): axum::extract::Path, ) -> Result, AppError> { let cfg = state.runtime_cfg.get(); @@ -2451,6 +2746,18 @@ pub async fn v1_elections_static_adnl_disable_handler( .map_err(|e| AppError::internal(e.to_string()))?; state.config_changed.notify_one(); + rest_audit::record_entity_mutation( + &state, + &claims, + &headers, + "elections", + "elections.static_adnl.disable", + format!("elections.static_adnl.{node_name}"), + serde_json::json!({ "node": node_name }), + serde_json::Value::Null, + ) + .await; + tracing::info!("node [{}] static ADNL disabled (ephemeral per cycle)", node_name); Ok(axum::Json(OkResponse { ok: true })) } diff --git a/src/node-control/service/src/http/config_handlers_tests.rs b/src/node-control/service/src/http/config_handlers_tests.rs index a335560c..1af4af06 100644 --- a/src/node-control/service/src/http/config_handlers_tests.rs +++ b/src/node-control/service/src/http/config_handlers_tests.rs @@ -12,6 +12,7 @@ //! "not deployed" path (no address) and the "error" path (RPC unreachable), //! plus the SNP shape and slot-ordering invariants. use crate::{ + audit::{AuditActorBuilder, AuditEventPayload, in_memory::InMemoryAuditLog, log::AuditLog}, auth::{jwt::JwtAuth, user_store::UserStore}, http::http_server_task::*, runtime_config::{RuntimeConfig, RuntimeConfigStore}, @@ -71,13 +72,32 @@ async fn state_from_cfg(cfg: AppConfig) -> AppState { runtime_cfg: rt.clone(), elections_task: Arc::new(TaskController::new("elections", Noop, rt.clone())), jwt_auth, - user_store: Arc::new(UserStore::new(rt as Arc)), + user_store: Arc::new(UserStore::new(rt.clone() as Arc)), login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), audit: Arc::new(crate::audit::log::NoopAuditLog), + actor_builder: Arc::new(AuditActorBuilder::new(rt)), } } +async fn state_audited(cfg: AppConfig) -> (AppState, Arc) { + let rt = Arc::new(RuntimeConfigStore::from_app_config(Arc::new(cfg))); + let audit_mem = Arc::new(InMemoryAuditLog::new()); + let audit: Arc = audit_mem.clone(); + let state = AppState { + store: Arc::new(SnapshotStore::new()), + runtime_cfg: rt.clone(), + elections_task: Arc::new(TaskController::new("elections", Noop, rt.clone())), + jwt_auth: Arc::new(JwtAuth::new(None, Some(TEST_JWT_SECRET)).await.unwrap()), + user_store: Arc::new(UserStore::new(rt.clone() as Arc)), + login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), + config_changed: Arc::new(tokio::sync::Notify::new()), + audit, + actor_builder: Arc::new(AuditActorBuilder::new(rt)), + }; + (state, audit_mem) +} + async fn state_with_pools(pools: HashMap) -> AppState { let mut cfg = (*empty_app_cfg()).clone(); cfg.pools = pools; @@ -570,3 +590,23 @@ async fn pools_add_core_rejects_max_nominators_zero() { let v = json(resp).await; assert!(v["error"]["message"].as_str().unwrap().contains("max_nominators")); } + +#[tokio::test] +async fn config_update_emits_audit_event_with_diff() { + let mut cfg = (*empty_app_cfg()).clone(); + cfg.elections = Some(Default::default()); + let (st, audit) = state_audited(cfg).await; + + let body = serde_json::json!({ "sleep_period_pct": 0.12 }); + let resp = routes(false, st).oneshot(post_json("/v1/elections/settings", &body)).await.unwrap(); + assert_eq!(resp.status(), 200); + + let events = audit.drain(); + assert_eq!(events.len(), 1); + let AuditEventPayload::RestApiConfigUpdated { operation, changes } = &events[0].payload else { + panic!("expected config_updated payload"); + }; + assert_eq!(operation, "elections.settings_updated"); + assert_eq!(changes.len(), 1); + assert_eq!(changes[0].field, "elections.sleep_period_pct"); +} diff --git a/src/node-control/service/src/http/entity_crud_handlers_tests.rs b/src/node-control/service/src/http/entity_crud_handlers_tests.rs index 557f36dc..1203d291 100644 --- a/src/node-control/service/src/http/entity_crud_handlers_tests.rs +++ b/src/node-control/service/src/http/entity_crud_handlers_tests.rs @@ -136,10 +136,11 @@ async fn app_state(cfg: AppConfig) -> AppState { runtime_cfg: rt.clone(), elections_task: Arc::new(TaskController::new("elections", Noop, rt.clone())), jwt_auth, - user_store: Arc::new(UserStore::new(rt as Arc)), + user_store: Arc::new(UserStore::new(rt.clone() as Arc)), login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), audit: Arc::new(crate::audit::log::NoopAuditLog), + actor_builder: Arc::new(crate::audit::AuditActorBuilder::new(rt.clone())), } } @@ -155,10 +156,11 @@ async fn app_state_with_path(cfg: AppConfig, path: std::path::PathBuf) -> AppSta runtime_cfg: rt.clone(), elections_task: Arc::new(TaskController::new("elections", Noop, rt.clone())), jwt_auth, - user_store: Arc::new(UserStore::new(rt as Arc)), + user_store: Arc::new(UserStore::new(rt.clone() as Arc)), login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), audit: Arc::new(crate::audit::log::NoopAuditLog), + actor_builder: Arc::new(crate::audit::AuditActorBuilder::new(rt)), } } diff --git a/src/node-control/service/src/http/http_server_task.rs b/src/node-control/service/src/http/http_server_task.rs index d4a58e04..66635da2 100644 --- a/src/node-control/service/src/http/http_server_task.rs +++ b/src/node-control/service/src/http/http_server_task.rs @@ -23,7 +23,7 @@ use super::{ login_rate_limiter::{LoginRateLimiter, login_limiter_key}, }; use crate::{ - audit::log::AuditLog, + audit::{AuditActorBuilder, log::AuditLog}, auth::{ Claims, jwt::JwtAuth, @@ -55,6 +55,7 @@ pub struct AppState { /// (entity CRUD, ton-http-api) so the service loop can rebuild caches. pub config_changed: Arc, pub audit: Arc, + pub actor_builder: Arc, } pub async fn run( @@ -113,6 +114,7 @@ pub async fn run( let elections_task = tasks.get("elections").cloned().expect("elections task is not registered"); let login_rate_limiter = Arc::new(tokio::sync::Mutex::new(LoginRateLimiter::default())); + let actor_builder = Arc::new(AuditActorBuilder::new(runtime_cfg.clone())); let state = AppState { store, runtime_cfg, @@ -122,6 +124,7 @@ pub async fn run( login_rate_limiter, config_changed, audit, + actor_builder, }; let app = routes(enable_swagger, state); @@ -508,6 +511,8 @@ pub async fn v1_elections_handler( )] pub async fn v1_elections_exclude_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { if state.runtime_cfg.get().elections.is_none() { @@ -541,6 +546,26 @@ pub async fn v1_elections_exclude_handler( .collect(); tracing::info!("elections excluded: {}", excluded.join(", ")); + let changes: Vec<_> = to_exclude + .iter() + .map(|node| { + super::rest_audit::config_field( + format!("bindings.{node}.enable"), + serde_json::json!(true), + serde_json::json!(false), + ) + }) + .collect(); + super::rest_audit::record_config_updated( + &state, + &claims, + &headers, + "elections", + "elections.exclude", + changes, + ) + .await; + let applied = ElectionsExcludeResult { excluded, updated_at: state.runtime_cfg.updated_at() }; Ok(axum::Json(ElectionsExcludeResponse { ok: true, result: applied })) } @@ -559,6 +584,8 @@ pub async fn v1_elections_exclude_handler( )] pub async fn v1_elections_include_handler( state: axum::extract::State, + claims: axum::Extension, + headers: axum::http::HeaderMap, req: axum::Json, ) -> Result, AppError> { if state.runtime_cfg.get().elections.is_none() { @@ -592,6 +619,26 @@ pub async fn v1_elections_include_handler( .collect(); tracing::info!("elections excluded: {}", excluded.join(", ")); + let changes: Vec<_> = to_include + .iter() + .map(|node| { + super::rest_audit::config_field( + format!("bindings.{node}.enable"), + serde_json::json!(false), + serde_json::json!(true), + ) + }) + .collect(); + super::rest_audit::record_config_updated( + &state, + &claims, + &headers, + "elections", + "elections.include", + changes, + ) + .await; + let applied = ElectionsExcludeResult { excluded, updated_at: state.runtime_cfg.updated_at() }; Ok(axum::Json(ElectionsExcludeResponse { ok: true, result: applied })) } @@ -728,6 +775,13 @@ pub async fn login_handler( rate_limit_key = %limiter_key, "login rejected" ); + super::rest_audit::record_login_rejected( + &state, + &req.username, + "rate_limited", + &headers, + ) + .await; return Err(AppError::too_many_requests("too many login attempts, try again later")); } } @@ -765,6 +819,13 @@ pub async fn login_handler( rate_limit_key = %limiter_key, "login rejected" ); + super::rest_audit::record_login_rejected( + &state, + &req.username, + "rate_limited", + &headers, + ) + .await; return Err(AppError::too_many_requests( "too many login attempts, try again later", )); @@ -778,6 +839,13 @@ pub async fn login_handler( rate_limit_key = %limiter_key, "login rejected" ); + super::rest_audit::record_login_rejected( + &state, + &req.username, + "invalid_credentials", + &headers, + ) + .await; return Err(AppError::unauthorized("invalid username or password")); } }; @@ -799,6 +867,9 @@ pub async fn login_handler( AppError::internal("token generation failed") })?; + super::rest_audit::record_login_success(&state, &req.username, &role.to_string(), &headers) + .await; + Ok(axum::Json(LoginResponse { ok: true, token, expires_in, role: role.to_string() })) } @@ -1058,13 +1129,14 @@ mod tests { let user_store = Arc::new(UserStore::new(runtime_cfg.clone() as Arc)); AppState { store, - runtime_cfg, + runtime_cfg: runtime_cfg.clone(), elections_task, jwt_auth: test_jwt_auth().await, user_store, login_rate_limiter: Arc::new(tokio::sync::Mutex::new(LoginRateLimiter::default())), config_changed: Arc::new(tokio::sync::Notify::new()), audit: Arc::new(NoopAuditLog), + actor_builder: Arc::new(AuditActorBuilder::new(runtime_cfg)), } } diff --git a/src/node-control/service/src/http/mod.rs b/src/node-control/service/src/http/mod.rs index 9dc2cee9..915650d7 100644 --- a/src/node-control/service/src/http/mod.rs +++ b/src/node-control/service/src/http/mod.rs @@ -9,6 +9,7 @@ pub mod config_handlers; pub mod http_server_task; pub(crate) mod login_rate_limiter; +pub(crate) mod rest_audit; pub use http_server_task::run; diff --git a/src/node-control/service/src/http/rest_audit.rs b/src/node-control/service/src/http/rest_audit.rs new file mode 100644 index 00000000..c7a2bbf8 --- /dev/null +++ b/src/node-control/service/src/http/rest_audit.rs @@ -0,0 +1,214 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use super::http_server_task::AppState; +use crate::{ + audit::{AuditEvent, ConfigFieldChange, actor_builder::client_ip_from_headers}, + auth::Claims, +}; +use common::app_config::{ContractsAutomationConfig, ElectionsConfig}; + +pub fn config_field( + field: impl Into, + old: serde_json::Value, + new: serde_json::Value, +) -> ConfigFieldChange { + ConfigFieldChange { field: field.into(), old, new } +} + +pub async fn record_login_success( + state: &AppState, + username: &str, + role: &str, + headers: &axum::http::HeaderMap, +) { + let actor = state.actor_builder.rest_user(username, role, client_ip_from_headers(headers)); + state.audit.record(AuditEvent::rest_api_auth_login_success(actor, username)).await; +} + +pub async fn record_login_rejected( + state: &AppState, + username: &str, + reason: &str, + headers: &axum::http::HeaderMap, +) { + let actor = state.actor_builder.rest_user(username, "unknown", client_ip_from_headers(headers)); + state.audit.record(AuditEvent::rest_api_auth_login_rejected(actor, username, reason)).await; +} + +pub async fn record_token_rejected( + state: &AppState, + user_id: &str, + reason: &str, + headers: &axum::http::HeaderMap, +) { + let actor = state.actor_builder.rest_user(user_id, "unknown", client_ip_from_headers(headers)); + state.audit.record(AuditEvent::rest_api_token_rejected(actor, user_id, reason)).await; +} + +pub async fn record_config_updated( + state: &AppState, + claims: &Claims, + headers: &axum::http::HeaderMap, + config_id: &str, + operation: &str, + changes: Vec, +) { + if changes.is_empty() { + return; + } + let actor = state.actor_builder.rest_user( + &claims.sub, + claims.role.to_string(), + client_ip_from_headers(headers), + ); + state + .audit + .record(AuditEvent::rest_api_config_updated(actor, config_id, operation, changes)) + .await; +} + +pub async fn record_entity_mutation( + state: &AppState, + claims: &Claims, + headers: &axum::http::HeaderMap, + config_id: &str, + operation: &str, + field: impl Into, + old: serde_json::Value, + new: serde_json::Value, +) { + record_config_updated( + state, + claims, + headers, + config_id, + operation, + vec![config_field(field, old, new)], + ) + .await; +} + +pub fn elections_settings_changes( + before: &ElectionsConfig, + after: &ElectionsConfig, + sleep_period_pct: Option, + waiting_period_pct: Option, + tick_interval: Option, + max_factor: Option, + policy_changed: bool, + policy_node: Option<&str>, +) -> Vec { + let mut changes = Vec::new(); + if let Some(v) = sleep_period_pct { + changes.push(config_field( + "elections.sleep_period_pct", + serde_json::json!(before.sleep_period_pct), + serde_json::json!(v), + )); + } + if let Some(v) = waiting_period_pct { + changes.push(config_field( + "elections.waiting_period_pct", + serde_json::json!(before.waiting_period_pct), + serde_json::json!(v), + )); + } + if let Some(v) = tick_interval { + changes.push(config_field( + "elections.tick_interval", + serde_json::json!(before.tick_interval), + serde_json::json!(v), + )); + } + if let Some(v) = max_factor { + changes.push(config_field( + "elections.max_factor", + serde_json::json!(before.max_factor), + serde_json::json!(v), + )); + } + if policy_changed { + let field = policy_node + .map(|n| format!("elections.policy_overrides.{n}")) + .unwrap_or_else(|| "elections.policy".into()); + let old = + policy_node.and_then(|n| before.policy_overrides.get(n)).unwrap_or(&before.policy); + let new = policy_node.and_then(|n| after.policy_overrides.get(n)).unwrap_or(&after.policy); + changes.push(config_field(field, serde_json::json!(old), serde_json::json!(new))); + } + changes +} + +pub fn automation_settings_changes( + before: &ContractsAutomationConfig, + req: &super::config_handlers::ContractsAutomationSettingsUpdateRequest, +) -> Vec { + let mut changes = Vec::new(); + if let Some(v) = req.tick_interval_sec { + changes.push(config_field( + "automation.tick_interval_sec", + serde_json::json!(before.tick_interval_sec), + serde_json::json!(v), + )); + } + if let Some(v) = req.auto_deploy { + changes.push(config_field( + "automation.auto_deploy", + serde_json::json!(before.auto_deploy), + serde_json::json!(v), + )); + } + if let Some(v) = req.auto_topup { + changes.push(config_field( + "automation.auto_topup", + serde_json::json!(before.auto_topup), + serde_json::json!(v), + )); + } + if let Some(ref patch) = req.wallet { + if let Some(v) = patch.deploy { + changes.push(config_field( + "automation.wallet.deploy", + serde_json::json!(before.wallet.deploy), + serde_json::json!(v), + )); + } + if let Some(v) = patch.topup { + changes.push(config_field( + "automation.wallet.topup", + serde_json::json!(before.wallet.topup), + serde_json::json!(v), + )); + } + if let Some(v) = patch.threshold { + changes.push(config_field( + "automation.wallet.threshold", + serde_json::json!(before.wallet.threshold), + serde_json::json!(v), + )); + } + } + if let Some(ref patch) = req.pool { + if let Some(v) = patch.snp { + changes.push(config_field( + "automation.pool.snp", + serde_json::json!(before.pool.snp), + serde_json::json!(v), + )); + } + if let Some(v) = patch.ton_core { + changes.push(config_field( + "automation.pool.ton_core", + serde_json::json!(before.pool.ton_core), + serde_json::json!(v), + )); + } + } + changes +} From de547520419b900bb2e0abfd26e2d9fd14ef1a37 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Mon, 8 Jun 2026 14:22:14 +0300 Subject: [PATCH 17/30] fix: review comments --- src/node-control/service/src/audit/enums.rs | 41 ++-- src/node-control/service/src/audit/event.rs | 101 ++++---- .../service/src/audit/in_memory.rs | 2 +- .../service/src/audit/jsonl_writer.rs | 39 ++- src/node-control/service/src/audit/mod.rs | 2 +- .../src/elections/adaptive_strategy.rs | 48 +--- .../service/src/elections/runner.rs | 230 +++++++----------- .../service/src/elections/runner_tests.rs | 143 +++++------ 8 files changed, 268 insertions(+), 338 deletions(-) diff --git a/src/node-control/service/src/audit/enums.rs b/src/node-control/service/src/audit/enums.rs index d655ed8f..5777ec89 100644 --- a/src/node-control/service/src/audit/enums.rs +++ b/src/node-control/service/src/audit/enums.rs @@ -12,9 +12,6 @@ use serde::{Deserialize, Serialize}; #[serde(tag = "event_type", content = "data", rename_all = "snake_case")] #[non_exhaustive] pub enum AuditEventPayload { - #[serde(rename = "elections.tick_failed")] - ElectionsTickFailed { reason: String }, - #[serde(rename = "elections.key_generated")] ElectionsKeyGenerated { #[serde(skip_serializing_if = "Option::is_none")] @@ -41,18 +38,24 @@ pub enum AuditEventPayload { available_nanotons: Option, }, + #[serde(rename = "elections.stake_failed")] + ElectionsStakeFailed { reason: String }, + #[serde(rename = "elections.stake_recovered")] ElectionsStakeRecovered { amount_nanotons: String, #[serde(skip_serializing_if = "Option::is_none")] - tx_hash: Option, + msg_hash: Option, }, + #[serde(rename = "elections.stake_recover_failed")] + ElectionsStakeRecoverFailed { reason: String }, + #[serde(rename = "elections.withdraw_processed")] - ElectionsWithdrawProcessed { tx_hash: String }, + ElectionsWithdrawProcessed { msg_hash: String }, - #[serde(rename = "elections.withdraw_process_failed")] - ElectionsWithdrawProcessFailed { reason: String }, + #[serde(rename = "elections.withdraw_failed")] + ElectionsWithdrawFailed { reason: String }, // ── rewards (reserved; producers not wired yet) ───────────────────────── #[serde(rename = "rewards.distribution_started")] @@ -71,8 +74,8 @@ pub enum AuditEventPayload { #[serde(rename = "rest_api.config_updated")] RestApiConfigUpdated { operation: String, changes: Vec }, - #[serde(rename = "rest_api.auth_login_success")] - RestApiAuthLoginSuccess {}, + #[serde(rename = "rest_api.auth_login_succeeded")] + RestApiAuthLoginSucceeded {}, #[serde(rename = "rest_api.auth_login_rejected")] RestApiAuthLoginRejected { reason: String }, @@ -105,9 +108,9 @@ pub enum StakeSkipReason { LowWalletBalance, WithdrawRequestsPending, PoolNotReady, - AdaptiveSleepPeriod, + AdaptiveSleepingPeriod, AdaptiveWaitingPeriod, - NodeExcluded, + ElectionsDisabled, RecoverPending, InsufficientStakeFunds, } @@ -161,7 +164,7 @@ impl AuditEventPayload { | RewardsDistributionStarted { .. } | RewardsDistributionCompleted { .. } | RestApiConfigUpdated { .. } - | RestApiAuthLoginSuccess {} + | RestApiAuthLoginSucceeded {} | VaultKeyCreated {} | VaultKeyRemoved {} | SystemServiceStarted { .. } @@ -173,8 +176,9 @@ impl AuditEventPayload { | RestApiTokenRejected { .. } | SystemAuditEventsDropped { .. } => Warn, - ElectionsTickFailed { .. } - | ElectionsWithdrawProcessFailed { .. } + ElectionsStakeFailed { .. } + | ElectionsStakeRecoverFailed { .. } + | ElectionsWithdrawFailed { .. } | RewardsDistributionFailed { .. } => Error, } } @@ -183,14 +187,15 @@ impl AuditEventPayload { use AuditEventPayload::*; use AuditSource::*; match self { - ElectionsTickFailed { .. } - | ElectionsKeyGenerated { .. } + ElectionsKeyGenerated { .. } | ElectionsStakeSubmitted { .. } | ElectionsStakeAccepted { .. } | ElectionsStakeSkipped { .. } + | ElectionsStakeFailed { .. } | ElectionsStakeRecovered { .. } + | ElectionsStakeRecoverFailed { .. } | ElectionsWithdrawProcessed { .. } - | ElectionsWithdrawProcessFailed { .. } => Elections, + | ElectionsWithdrawFailed { .. } => Elections, RewardsDistributionStarted { .. } | RewardsDistributionCompleted { .. } @@ -198,7 +203,7 @@ impl AuditEventPayload { | RewardsRecipientSkipped { .. } => Rewards, RestApiConfigUpdated { .. } - | RestApiAuthLoginSuccess {} + | RestApiAuthLoginSucceeded {} | RestApiAuthLoginRejected { .. } | RestApiTokenRejected { .. } => RestApi, diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index 446e2309..1f552036 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -64,6 +64,15 @@ pub struct AuditEvent { pub target: AuditTarget, } +/// Named payload fields for [`AuditEvent::elections_stake_submitted`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ElectionsStakeSubmittedParams { + pub stake_nanotons: String, + pub max_factor: u32, + pub policy: String, + pub submission_time: u64, +} + impl AuditEvent { /// Internal constructor that stamps `id`/`ts`. Crate-private so call sites /// must go through the typed constructors below, which bake the canonical @@ -82,24 +91,6 @@ impl AuditEvent { AuditTarget::Node { id: node_id.into(), election_id: Some(election_id) } } - pub fn elections_tick_failed( - actor: AuditActor, - election_id: Option, - reason: impl Into, - ) -> Self { - // A tick can fail before the active election id is known; fall back to a - // system target in that case (source still resolves to `elections`). - let target = election_id - .map(|election_id| AuditTarget::Elections { election_id }) - .unwrap_or(AuditTarget::System); - Self::new( - actor, - target, - AuditOutcome::Failure, - AuditEventPayload::ElectionsTickFailed { reason: reason.into() }, - ) - } - pub fn elections_key_generated( actor: AuditActor, node_id: impl Into, @@ -114,25 +105,21 @@ impl AuditEvent { ) } - #[allow(clippy::too_many_arguments)] pub fn elections_stake_submitted( actor: AuditActor, node_id: impl Into, election_id: u64, - stake_nanotons: impl Into, - max_factor: u32, - policy: impl Into, - submission_time: u64, + params: ElectionsStakeSubmittedParams, ) -> Self { Self::new( actor, Self::node_target(node_id, election_id), AuditOutcome::Success, AuditEventPayload::ElectionsStakeSubmitted { - stake_nanotons: stake_nanotons.into(), - max_factor, - policy: policy.into(), - submission_time, + stake_nanotons: params.stake_nanotons, + max_factor: params.max_factor, + policy: params.policy, + submission_time: params.submission_time, }, ) } @@ -157,12 +144,26 @@ impl AuditEvent { ) } + pub fn elections_stake_failed( + actor: AuditActor, + node_id: impl Into, + election_id: u64, + reason: impl Into, + ) -> Self { + Self::new( + actor, + Self::node_target(node_id, election_id), + AuditOutcome::Failure, + AuditEventPayload::ElectionsStakeFailed { reason: reason.into() }, + ) + } + pub fn elections_stake_recovered( actor: AuditActor, node_id: impl Into, election_id: u64, amount_nanotons: impl Into, - tx_hash: Option, + msg_hash: Option, ) -> Self { Self::new( actor, @@ -170,26 +171,40 @@ impl AuditEvent { AuditOutcome::Success, AuditEventPayload::ElectionsStakeRecovered { amount_nanotons: amount_nanotons.into(), - tx_hash, + msg_hash, }, ) } + pub fn elections_stake_recover_failed( + actor: AuditActor, + node_id: impl Into, + election_id: u64, + reason: impl Into, + ) -> Self { + Self::new( + actor, + Self::node_target(node_id, election_id), + AuditOutcome::Failure, + AuditEventPayload::ElectionsStakeRecoverFailed { reason: reason.into() }, + ) + } + pub fn elections_withdraw_processed( actor: AuditActor, node_id: impl Into, election_id: u64, - tx_hash: impl Into, + msg_hash: impl Into, ) -> Self { Self::new( actor, Self::node_target(node_id, election_id), AuditOutcome::Success, - AuditEventPayload::ElectionsWithdrawProcessed { tx_hash: tx_hash.into() }, + AuditEventPayload::ElectionsWithdrawProcessed { msg_hash: msg_hash.into() }, ) } - pub fn elections_withdraw_process_failed( + pub fn elections_withdraw_failed( actor: AuditActor, node_id: impl Into, election_id: u64, @@ -199,7 +214,7 @@ impl AuditEvent { actor, Self::node_target(node_id, election_id), AuditOutcome::Failure, - AuditEventPayload::ElectionsWithdrawProcessFailed { reason: reason.into() }, + AuditEventPayload::ElectionsWithdrawFailed { reason: reason.into() }, ) } @@ -356,7 +371,6 @@ mod tests { fn all_payload_variants() -> Vec { vec![ - AuditEventPayload::ElectionsTickFailed { reason: "tick error".into() }, AuditEventPayload::ElectionsKeyGenerated { pubkey: Some("aabb".into()) }, AuditEventPayload::ElectionsStakeSubmitted { stake_nanotons: "1".into(), @@ -370,12 +384,14 @@ mod tests { required_nanotons: None, available_nanotons: None, }, - AuditEventPayload::ElectionsWithdrawProcessed { tx_hash: "abc".into() }, - AuditEventPayload::ElectionsWithdrawProcessFailed { reason: "send failed".into() }, + AuditEventPayload::ElectionsStakeFailed { reason: "send failed".into() }, + AuditEventPayload::ElectionsWithdrawProcessed { msg_hash: "abc".into() }, + AuditEventPayload::ElectionsWithdrawFailed { reason: "send failed".into() }, AuditEventPayload::ElectionsStakeRecovered { amount_nanotons: "50000000000000".into(), - tx_hash: Some("def".into()), + msg_hash: Some("def".into()), }, + AuditEventPayload::ElectionsStakeRecoverFailed { reason: "send failed".into() }, AuditEventPayload::RewardsDistributionStarted { recipients_count: 3 }, AuditEventPayload::RewardsDistributionCompleted { recipients_count: 3, @@ -391,7 +407,7 @@ mod tests { new: json!(2), }], }, - AuditEventPayload::RestApiAuthLoginSuccess {}, + AuditEventPayload::RestApiAuthLoginSucceeded {}, AuditEventPayload::RestApiAuthLoginRejected { reason: "bad password".into() }, AuditEventPayload::RestApiTokenRejected { reason: "expired".into() }, AuditEventPayload::VaultKeyCreated {}, @@ -427,10 +443,13 @@ mod tests { ); assert_eq!(skipped.outcome, AuditOutcome::Skipped); - let failed = - AuditEvent::elections_tick_failed(AuditActor::service("elections-task"), None, "boom"); + let failed = AuditEvent::elections_stake_failed( + AuditActor::service("elections-task"), + "node1", + 1, + "boom", + ); assert_eq!(failed.outcome, AuditOutcome::Failure); - assert_eq!(failed.target, AuditTarget::System); } #[test] diff --git a/src/node-control/service/src/audit/in_memory.rs b/src/node-control/service/src/audit/in_memory.rs index 2de9cbba..fc999ffb 100644 --- a/src/node-control/service/src/audit/in_memory.rs +++ b/src/node-control/service/src/audit/in_memory.rs @@ -12,7 +12,7 @@ use std::sync::Mutex; /// Captures audit events in memory for unit tests. pub struct InMemoryAuditLog { - pub events: Mutex>, + events: Mutex>, } impl InMemoryAuditLog { diff --git a/src/node-control/service/src/audit/jsonl_writer.rs b/src/node-control/service/src/audit/jsonl_writer.rs index 9ffecfb6..7fc685ec 100644 --- a/src/node-control/service/src/audit/jsonl_writer.rs +++ b/src/node-control/service/src/audit/jsonl_writer.rs @@ -10,7 +10,7 @@ use crate::audit::{AuditEvent, AuditFileHeader, AuditLogConfig, jsonl_log::Audit use chrono::Utc; use std::{ sync::{ - Arc, + Arc, Once, atomic::{AtomicU64, Ordering}, }, time::Duration, @@ -20,6 +20,8 @@ use tokio::sync::{mpsc, oneshot}; /// Schema version stamped into the per-file [`AuditFileHeader`]. const AUDIT_SCHEMA_VERSION: u16 = 1; +static HOSTNAME_FALLBACK_WARNED: Once = Once::new(); + #[derive(Debug)] pub(crate) enum AuditCommand { Event(Box), @@ -38,6 +40,8 @@ pub(crate) enum AuditCommand { } pub(crate) struct AuditWriter { + /// Host identity stamped into each new file's [`AuditFileHeader`]. + host: String, config: Arc, /// Live append handle. `None` only transiently during rotation (the old /// handle is closed before the on-disk rename so the swap is portable to @@ -116,6 +120,7 @@ impl AuditWriter { let current_size = file.metadata().await.map_err(AuditInitError::Metadata)?.len(); let mut writer = Self { + host: resolve_hostname(), config, file: Some(file), current_size, @@ -128,30 +133,22 @@ impl AuditWriter { Ok(writer) } - fn file_header() -> AuditFileHeader { + fn file_header(&self) -> AuditFileHeader { AuditFileHeader { schema_version: AUDIT_SCHEMA_VERSION, service: "nodectl".into(), service_version: env!("CARGO_PKG_VERSION").into(), - host: Self::hostname(), + host: self.host.clone(), started_at: Utc::now(), } } - fn hostname() -> String { - std::env::var("HOSTNAME") - .or_else(|_| std::env::var("COMPUTERNAME")) - .ok() - .filter(|h| !h.is_empty()) - .unwrap_or_else(|| "unknown".to_string()) - } - async fn write_header_if_empty(&mut self) -> std::io::Result<()> { if self.current_size != 0 { return Ok(()); } use tokio::io::AsyncWriteExt; - let mut line = serde_json::to_vec(&Self::file_header()) + let mut line = serde_json::to_vec(&self.file_header()) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; line.push(b'\n'); let file = self @@ -443,6 +440,24 @@ impl AuditWriter { } } +/// Resolves the host identity for audit file headers (`HOSTNAME`, then `COMPUTERNAME`). +fn resolve_hostname() -> String { + if let Some(host) = std::env::var("HOSTNAME") + .or_else(|_| std::env::var("COMPUTERNAME")) + .ok() + .filter(|h| !h.is_empty()) + { + return host; + } + HOSTNAME_FALLBACK_WARNED.call_once(|| { + tracing::warn!( + "audit log host identity unavailable (HOSTNAME/COMPUTERNAME unset); \ + file header will use host=\"unknown\" — set HOSTNAME for forensics" + ); + }); + "unknown".to_string() +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/node-control/service/src/audit/mod.rs b/src/node-control/service/src/audit/mod.rs index 11256d9e..91f79940 100644 --- a/src/node-control/service/src/audit/mod.rs +++ b/src/node-control/service/src/audit/mod.rs @@ -20,7 +20,7 @@ pub use common::app_config::AuditLogConfig; pub use enums::{ AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, ConfigFieldChange, StakeSkipReason, }; -pub use event::{AuditEvent, AuditFileHeader}; +pub use event::{AuditEvent, AuditFileHeader, ElectionsStakeSubmittedParams}; pub use factory::AuditLogFactory; #[cfg(test)] pub use in_memory::InMemoryAuditLog; diff --git a/src/node-control/service/src/elections/adaptive_strategy.rs b/src/node-control/service/src/elections/adaptive_strategy.rs index f0385338..8727850a 100644 --- a/src/node-control/service/src/elections/adaptive_strategy.rs +++ b/src/node-control/service/src/elections/adaptive_strategy.rs @@ -22,9 +22,11 @@ pub(crate) enum AdaptiveDeferReason { WaitingForParticipants, } -/// Why AdaptiveSplit50 returns zero stake after the defer window has passed. +/// Why AdaptiveSplit50 returns zero stake. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum AdaptiveStakeZero { + /// Sleep or waiting-for-participants gate has not passed yet. + Defer(AdaptiveDeferReason), /// Stake already meets min effective — no top-up this tick (not an error). NoTopUpNeeded, /// Free pool balance is below the required delta to min effective stake. @@ -38,8 +40,8 @@ pub(crate) enum AdaptiveStakeResult { Zero(AdaptiveStakeZero), } -/// Returns `true` if staking should proceed, `false` if we should defer (return 0). -pub(crate) fn is_adaptive_split50_ready( +/// Returns `None` when staking should proceed; otherwise the wait gate that blocks. +pub(crate) fn adaptive_split50_status( node_id: &str, elections_info: &ElectionsInfo, cfg15_start_before: u32, @@ -47,7 +49,7 @@ pub(crate) fn is_adaptive_split50_ready( cfg16: &ConfigParam16, sleep_pct: f64, waiting_pct: f64, -) -> bool { +) -> Option { let min_validators = cfg16.min_validators.as_u16() as usize; let participants_count = elections_info.participants.len(); let election_duration = cfg15_start_before.saturating_sub(cfg15_end_before) as u64; @@ -57,7 +59,7 @@ pub(crate) fn is_adaptive_split50_ready( "node [{}] adaptive_split50: election_duration=0, skipping wait logic", node_id ); - return true; + return None; } let election_start = elections_info.elect_close.saturating_sub(election_duration); @@ -65,17 +67,15 @@ pub(crate) fn is_adaptive_split50_ready( let wait_deadline = election_start + (election_duration as f64 * waiting_pct) as u64; let now = common::time_format::now(); - // Wait if sleep period hasn't passed yet if now < sleep_deadline { tracing::info!( "node [{}] adaptive_split50: sleep period, now < sleep_deadline={}", node_id, common::time_format::format_ts(sleep_deadline) ); - return false; + return Some(AdaptiveDeferReason::SleepPeriod); } - // Wait if not enough participants and waiting period hasn't expired if participants_count < min_validators && now < wait_deadline { tracing::info!( "node [{}] adaptive_split50: waiting for participants ({}/{}), deadline={}", @@ -84,39 +84,9 @@ pub(crate) fn is_adaptive_split50_ready( min_validators, common::time_format::format_ts(wait_deadline) ); - return false; - } - - true -} - -/// When [`is_adaptive_split50_ready`] would return `false`, reports which wait gate blocked. -pub(crate) fn adaptive_split50_defer_reason( - elections_info: &ElectionsInfo, - cfg15_start_before: u32, - cfg15_end_before: u32, - cfg16: &ConfigParam16, - sleep_pct: f64, - waiting_pct: f64, -) -> Option { - let min_validators = cfg16.min_validators.as_u16() as usize; - let participants_count = elections_info.participants.len(); - let election_duration = cfg15_start_before.saturating_sub(cfg15_end_before) as u64; - if election_duration == 0 { - return None; - } - - let election_start = elections_info.elect_close.saturating_sub(election_duration); - let sleep_deadline = election_start + (election_duration as f64 * sleep_pct) as u64; - let wait_deadline = election_start + (election_duration as f64 * waiting_pct) as u64; - let now = common::time_format::now(); - - if now < sleep_deadline { - return Some(AdaptiveDeferReason::SleepPeriod); - } - if participants_count < min_validators && now < wait_deadline { return Some(AdaptiveDeferReason::WaitingForParticipants); } + None } diff --git a/src/node-control/service/src/elections/runner.rs b/src/node-control/service/src/elections/runner.rs index e7689257..ba9a2a02 100644 --- a/src/node-control/service/src/elections/runner.rs +++ b/src/node-control/service/src/elections/runner.rs @@ -11,7 +11,10 @@ use super::{ election_emulator::ParticipantStake, providers::{ElectionsProvider, ValidatorConfig, ValidatorEntry}, }; -use crate::audit::{AuditEvent, StakeSkipReason, log::AuditLog, participant::AuditActor}; +use crate::audit::{ + AuditEvent, ElectionsStakeSubmittedParams, StakeSkipReason, log::AuditLog, + participant::AuditActor, +}; use anyhow::Context as _; use common::{ app_config::{BindingStatus, ElectionsConfig, NodeBinding, StakePolicy}, @@ -77,6 +80,20 @@ const WITHDRAW_PROCESS_LIMIT: u8 = 10; type OnStatusChange = Arc) + Send + Sync>; +/// Stake balance guard failed: `available` nanotons are below `required`. +/// Carried as a typed [`anyhow::Error`] root cause so audit producers read structured +/// amounts instead of parsing free-form error strings. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error( + "insufficient balance: required={} TON, available={} TON", + *required as f64 / 1_000_000_000.0, + *available as f64 / 1_000_000_000.0 +)] +struct InsufficientBalance { + required: u64, + available: u64, +} + /// Persists a batch of freshly generated static ADNL addresses (node_id → base64) into the /// runtime config under `elections.static_adnls`. Called at most once per tick. pub(crate) type PersistStaticAdnls = @@ -358,59 +375,17 @@ impl ElectionRunner { } } - async fn audit_stake_skipped( - &self, - election_id: u64, - node_id: &str, - reason: StakeSkipReason, - required_nanotons: Option, - available_nanotons: Option, - ) { - elections_audit_stake_skipped( - &self.audit, - election_id, - node_id, - reason, - required_nanotons, - available_nanotons, - ) - .await; - } - - /// Classifies stake==0 when [`calc_stake`] did not already supply an adaptive reason - /// (defer windows only; adaptive zero cases come from [`calc_adaptive_stake`]). - async fn classify_stake_zero( - node: &Node, - configs: &ConfigParams<'_>, - ctx: &StakeContext<'_>, - ) -> (StakeSkipReason, Option, Option) { - if matches!(&node.stake_policy, StakePolicy::AdaptiveSplit50) { - if let Some(defer) = adaptive_strategy::adaptive_split50_defer_reason( - configs.elections_info, - configs.cfg15.elections_start_before, - configs.cfg15.elections_end_before, - configs.cfg16, - ctx.sleep_pct, - ctx.waiting_pct, - ) { - return match defer { - adaptive_strategy::AdaptiveDeferReason::SleepPeriod => { - (StakeSkipReason::AdaptiveSleepPeriod, None, None) - } - adaptive_strategy::AdaptiveDeferReason::WaitingForParticipants => { - (StakeSkipReason::AdaptiveWaitingPeriod, None, None) - } - }; - } - } - (StakeSkipReason::InsufficientStakeFunds, None, None) - } - fn adaptive_zero_to_skip( zero: adaptive_strategy::AdaptiveStakeZero, ) -> Option<(StakeSkipReason, Option, Option)> { - use adaptive_strategy::AdaptiveStakeZero::*; + use adaptive_strategy::{AdaptiveDeferReason, AdaptiveStakeZero::*}; match zero { + Defer(AdaptiveDeferReason::SleepPeriod) => { + Some((StakeSkipReason::AdaptiveSleepingPeriod, None, None)) + } + Defer(AdaptiveDeferReason::WaitingForParticipants) => { + Some((StakeSkipReason::AdaptiveWaitingPeriod, None, None)) + } NoTopUpNeeded => None, InsufficientFree { required, available } => { Some((StakeSkipReason::InsufficientStakeFunds, Some(required), Some(available))) @@ -515,9 +490,6 @@ impl ElectionRunner { tokio::select! { _ = interval.tick() => { tracing::info!("TICK"); - let audit = self.audit.clone(); - let tick_election_id = - self.elector.get_active_election_id().await.ok().filter(|&id| id > 0); // Clear per-node last_error at the start of the tick (best-effort). for node in self.nodes.values_mut() { @@ -529,17 +501,6 @@ impl ElectionRunner { self.refresh_validator_configs().await; if let Err(e) = self.run().await { - let election_id = tick_election_id.or({ - let id = self.past_elections_cache_id; - (id > 0).then_some(id) - }); - audit - .record(AuditEvent::elections_tick_failed( - elections_audit_actor(), - election_id, - format!("{e:#}"), - )) - .await; tracing::error!("runner tick error: {:#}", e); } @@ -713,7 +674,8 @@ impl ElectionRunner { .collect::>(); nodes.sort(); for node_id in &skip_tick_nodes { - self.audit_stake_skipped( + elections_audit_stake_skipped( + &self.audit, election_id, node_id, StakeSkipReason::PoolNotReady, @@ -746,16 +708,18 @@ impl ElectionRunner { recover_amount as f64 / 1_000_000_000.0 ); if excluded { - self.audit_stake_skipped( + elections_audit_stake_skipped( + &self.audit, election_id, &node_id, - StakeSkipReason::NodeExcluded, + StakeSkipReason::ElectionsDisabled, None, None, ) .await; } else { - self.audit_stake_skipped( + elections_audit_stake_skipped( + &self.audit, election_id, &node_id, StakeSkipReason::RecoverPending, @@ -777,7 +741,8 @@ impl ElectionRunner { "node [{}] skip participate this tick: withdraw requests sent, awaiting pool drain", node_id ); - self.audit_stake_skipped( + elections_audit_stake_skipped( + &self.audit, election_id, &node_id, StakeSkipReason::WithdrawRequestsPending, @@ -804,6 +769,7 @@ impl ElectionRunner { cfg17: &cfg17, }; if let Err(e) = self.participate(&node_id, election_id, &config_params).await { + elections_audit_stake_failed(&self.audit, election_id, &node_id, &e).await; if let Some(node) = self.nodes.get_mut(&node_id) { node.last_error = Some(format!("{:#}", e)); } @@ -987,18 +953,12 @@ impl ElectionRunner { let (stake, adaptive_zero) = match Self::calc_stake(node, node_id, elections_stake, params, &stake_ctx).await { Ok(v) => v, - Err(e) => { - elections_audit_low_balance_if_parsed(&audit, election_id, node_id, &e).await; - return Err(e).context("stake calculation error"); - } + Err(e) => return Err(e).context("stake calculation error"), }; if stake == 0 { tracing::info!("node [{}] skipping elections this tick (stake=0)", node_id); - let skip = match adaptive_zero { - Some(z) => Self::adaptive_zero_to_skip(z), - None => Some(Self::classify_stake_zero(node, params, &stake_ctx).await), - }; + let skip = adaptive_zero.and_then(Self::adaptive_zero_to_skip); if let Some((reason, required, available)) = skip { elections_audit_stake_skipped( &audit, @@ -1077,7 +1037,6 @@ impl ElectionRunner { )) .await; if let Err(e) = Self::send_stake(node_id, node, stake, to_addr).await { - elections_audit_low_balance_if_parsed(&audit, election_id, node_id, &e).await; return Err(e); } elections_audit_stake_submitted(&audit, node_id, node, stake).await; @@ -1107,13 +1066,6 @@ impl ElectionRunner { nanotons_to_tons_f64(stake), ); if let Err(e) = Self::send_stake(node_id, node, stake, to_addr).await { - elections_audit_low_balance_if_parsed( - &audit, - election_id, - node_id, - &e, - ) - .await; return Err(e); } node.participant.as_mut().map(|p| p.stake += stake); @@ -1126,8 +1078,6 @@ impl ElectionRunner { p.stake = stake; } if let Err(e) = Self::send_stake(node_id, node, stake, to_addr).await { - elections_audit_low_balance_if_parsed(&audit, election_id, node_id, &e) - .await; return Err(e); } elections_audit_stake_submitted(&audit, node_id, node, stake).await; @@ -1149,19 +1099,16 @@ impl ElectionRunner { let fee = ELECTOR_STAKE_FEE + NPOOL_COMPUTE_FEE; let stake_balance = node.stake_balance(fee).await?; if stake_balance < stake { - anyhow::bail!( - "low stake balance: required={} TON, available={} TON", - stake as f64 / 1_000_000_000.0, - stake_balance as f64 / 1_000_000_000.0 - ); + return Err(InsufficientBalance { required: stake, available: stake_balance }.into()); } let wallet_balance = node.wallet_balance().await?; - if wallet_balance < fee + WALLET_COMPUTE_FEE { - anyhow::bail!( - "low wallet balance: required={} TON, available={} TON", - (fee + WALLET_COMPUTE_FEE) as f64 / 1_000_000_000.0, - wallet_balance as f64 / 1_000_000_000.0 - ); + let wallet_required = fee + WALLET_COMPUTE_FEE; + if wallet_balance < wallet_required { + return Err(InsufficientBalance { + required: wallet_required, + available: wallet_balance, + } + .into()); } let send_value = node.pool.as_ref().map(|_| fee).unwrap_or(stake + fee); @@ -1293,7 +1240,7 @@ impl ElectionRunner { .await .context("build process_withdraw_requests message")?; let msg_boc = write_boc(&msg).context("encode process_withdraw_requests boc")?; - let tx_hash = format!("{:x}", msg.repr_hash()); + let msg_hash = format!("{:x}", msg.repr_hash()); let send_result = { let node = self.nodes.get_mut(node_id).expect("node not found"); node.api.send_boc(&msg_boc).await.context("send process_withdraw_requests boc") @@ -1301,7 +1248,7 @@ impl ElectionRunner { if let Err(e) = send_result { self.audit - .record(AuditEvent::elections_withdraw_process_failed( + .record(AuditEvent::elections_withdraw_failed( elections_audit_actor(), node_id, election_id, @@ -1322,7 +1269,7 @@ impl ElectionRunner { elections_audit_actor(), node_id, election_id, - tx_hash, + msg_hash, )) .await; Ok(true) @@ -1357,16 +1304,30 @@ impl ElectionRunner { .wallet .message(to_addr, RECOVER_FEE, Self::build_recover_stake_payload().await?) .await?; - let tx_hash = format!("{:x}", msg.repr_hash()); + let msg_hash = format!("{:x}", msg.repr_hash()); let msg_boc = write_boc(&msg)?; - node.api.send_boc(&msg_boc).await?; + let send_result = { + let node = self.nodes.get_mut(node_id).expect("node not found"); + node.api.send_boc(&msg_boc).await.context("send recover stake boc") + }; + if let Err(e) = send_result { + self.audit + .record(AuditEvent::elections_stake_recover_failed( + elections_audit_actor(), + node_id, + election_id, + format!("{e:#}"), + )) + .await; + return Err(e); + } self.audit .record(AuditEvent::elections_stake_recovered( elections_audit_actor(), node_id, election_id, nanotons_to_dec_string(amount), - Some(tx_hash), + Some(msg_hash), )) .await; } @@ -1496,10 +1457,8 @@ impl ElectionRunner { total_balance as f64 / 1_000_000_000.0 ); if total_balance < min_stake { - anyhow::bail!( - "not enough funds: available={} TON, required={} TON", - total_balance as f64 / 1_000_000_000.0, - min_stake as f64 / 1_000_000_000.0 + return Err( + InsufficientBalance { required: min_stake, available: total_balance }.into() ); } @@ -1520,7 +1479,7 @@ impl ElectionRunner { match &node.stake_policy { StakePolicy::AdaptiveSplit50 => { - if !adaptive_strategy::is_adaptive_split50_ready( + if let Some(defer) = adaptive_strategy::adaptive_split50_status( node_id, configs.elections_info, configs.cfg15.elections_start_before, @@ -1529,7 +1488,7 @@ impl ElectionRunner { ctx.sleep_pct, ctx.waiting_pct, ) { - return Ok((0, None)); + return Ok((0, Some(adaptive_strategy::AdaptiveStakeZero::Defer(defer)))); } let current_stake = if node.stake_accepted { elections_stake } else { 0 }; let stakes: Vec<_> = configs @@ -2037,26 +1996,20 @@ async fn elections_audit_stake_skipped( .await; } -/// Emits `LowWalletBalance` only when `err` matches the `required=` / `available=` format -/// (calc_stake or send_stake balance guards). Other failures are not audited as skip. -async fn elections_audit_low_balance_if_parsed( +async fn elections_audit_stake_failed( audit: &Arc, election_id: u64, node_id: &str, - err: &(impl std::fmt::Display + ?Sized), + err: &anyhow::Error, ) { - let (required, available) = parse_balance_error(&err.to_string()); - if required.is_some() || available.is_some() { - elections_audit_stake_skipped( - audit, - election_id, + audit + .record(AuditEvent::elections_stake_failed( + elections_audit_actor(), node_id, - StakeSkipReason::LowWalletBalance, - required, - available, - ) + election_id, + format!("{err:#}"), + )) .await; - } } async fn elections_audit_stake_submitted( @@ -2074,33 +2027,16 @@ async fn elections_audit_stake_submitted( elections_audit_actor(), node_id, participant.election_id, - nanotons_to_dec_string(stake), - participant.max_factor, - node.stake_policy.to_string(), - submission_time, + ElectionsStakeSubmittedParams { + stake_nanotons: nanotons_to_dec_string(stake), + max_factor: participant.max_factor, + policy: node.stake_policy.to_string(), + submission_time, + }, )) .await; } -/// Parses `required=` / `available=` TON amounts from runner balance error strings. -fn parse_balance_error(msg: &str) -> (Option, Option) { - fn tons_to_nanotons(fragment: &str) -> Option { - let tons: f64 = fragment.trim().parse().ok()?; - Some((tons * 1_000_000_000.0).round() as u64) - } - let required = msg - .split("required=") - .nth(1) - .and_then(|s| s.split(" TON").next()) - .and_then(tons_to_nanotons); - let available = msg - .split("available=") - .nth(1) - .and_then(|s| s.split(" TON").next()) - .and_then(tons_to_nanotons); - (required, available) -} - async fn find_validator_entries( node: &mut Node, current_vset: Option<&ValidatorSet>, diff --git a/src/node-control/service/src/elections/runner_tests.rs b/src/node-control/service/src/elections/runner_tests.rs index 368aeda5..d32864f9 100644 --- a/src/node-control/service/src/elections/runner_tests.rs +++ b/src/node-control/service/src/elections/runner_tests.rs @@ -915,6 +915,46 @@ async fn test_recover_stake_returns_funds() { assert!(node.participant.is_none(), "should not participate when recovering stake"); } +#[tokio::test] +async fn recover_stake_send_failed_emits_audit_event() { + let node_id = "node-1"; + let mut harness = TestHarness::new(); + + let returned_amount: u64 = 20_000_000_000_000; + setup_default_elector(&mut harness.elector_mock, ELECTION_ID, returned_amount); + setup_wallet(&mut harness.wallet_mock); + + let provider = &mut harness.provider_mock; + provider.expect_election_parameters().returning(|| Ok(default_cfg15())); + provider.expect_validator_config().returning(|| Ok(ValidatorConfig::new())); + provider.expect_account().returning(|_| Ok(fake_account(WALLET_BALANCE))); + provider + .expect_send_boc() + .times(1) + .returning(|_| Err(anyhow::anyhow!("simulated recover send_boc failure"))); + provider.expect_config_param_16().returning(|| Ok(default_cfg16())); + provider.expect_config_param_17().returning(|| Ok(default_cfg17())); + provider.expect_shutdown().returning(|| Ok(())); + + let audit = harness.audit.clone(); + let mut runner = harness.build(node_id).await; + runner.run().await.unwrap(); + + let events = audit.drain(); + let ev = find_audit_event(&events, |p| { + matches!(p, AuditEventPayload::ElectionsStakeRecoverFailed { .. }) + }); + + assert_eq!(ev.payload.severity(), AuditSeverity::Error); + assert_eq!(ev.outcome, AuditOutcome::Failure); + assert_node_target(&ev.target, node_id, ELECTION_ID); + + let AuditEventPayload::ElectionsStakeRecoverFailed { reason } = &ev.payload else { + unreachable!(); + }; + assert!(reason.contains("simulated recover send_boc failure")); +} + // ===================================================== // TEST: recover stake — low wallet balance // ===================================================== @@ -1321,6 +1361,7 @@ async fn test_low_stake_balance() { provider.expect_shutdown().returning(|| Ok(())); + let audit = harness.audit.clone(); let mut runner = harness.build(node_id).await; let result = runner.run().await; // run() itself is Ok, but the node should have an error @@ -1330,10 +1371,23 @@ async fn test_low_stake_balance() { assert!(node.last_error.is_some(), "should have an error for low balance"); let err = node.last_error.as_ref().unwrap(); assert!( - err.contains("not enough") || err.contains("low stake"), + err.contains("insufficient balance"), "error should mention insufficient balance, got: {}", err ); + + let events = audit.drain(); + let ev = + find_audit_event(&events, |p| matches!(p, AuditEventPayload::ElectionsStakeFailed { .. })); + + assert_eq!(ev.payload.severity(), AuditSeverity::Error); + assert_eq!(ev.outcome, AuditOutcome::Failure); + assert_node_target(&ev.target, node_id, ELECTION_ID); + + let AuditEventPayload::ElectionsStakeFailed { reason } = &ev.payload else { + unreachable!(); + }; + assert!(reason.contains("insufficient balance")); } // ===================================================== @@ -3814,78 +3868,6 @@ async fn cached_prev_min_eff_updates_on_refresh() { ); } -#[tokio::test] -async fn tick_success_does_not_emit_tick_audit_events() { - let node_id = "node-1"; - let mut harness = TestHarness::new(); - - setup_default_elector(&mut harness.elector_mock, ELECTION_ID, 0); - setup_default_provider(&mut harness.provider_mock, WALLET_BALANCE, None); - setup_wallet(&mut harness.wallet_mock); - harness.wallet_mock.expect_message().returning(|_dest, _value, _payload| Ok(dummy_cell())); - - let audit = harness.audit.clone(); - let mut runner = harness.build(node_id).await; - let mut ctx = CancellationCtx::new(); - let cancel_ctx = ctx.clone(); - let store = Arc::new(SnapshotStore::new()); - - tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(200)).await; - ctx.cancel(CancellationReason::GracefullyShutdown()); - }); - - runner.run_loop(Duration::from_millis(50), cancel_ctx, store, None).await.unwrap(); - - let events = audit.drain(); - assert!( - !events.iter().any(|e| matches!(e.payload, AuditEventPayload::ElectionsTickFailed { .. })), - "successful ticks must not emit tick_failed audit events" - ); -} - -#[tokio::test] -async fn tick_emits_failed_on_error() { - let node_id = "node-1"; - let mut harness = TestHarness::new(); - - harness.elector_mock.expect_address().returning(|| Ok(elector_address())); - harness.elector_mock.expect_get_active_election_id().returning(|| Ok(ELECTION_ID)); - harness - .elector_mock - .expect_elections_info() - .returning(|| Err(anyhow::anyhow!("simulated elections_info failure"))); - harness.elector_mock.expect_past_elections().returning(|| Ok(vec![])); - harness.elector_mock.expect_compute_returned_stake().returning(|_| Ok(0)); - setup_default_provider_without_account(&mut harness.provider_mock, WALLET_BALANCE); - setup_wallet(&mut harness.wallet_mock); - - let audit = harness.audit.clone(); - let mut runner = harness.build(node_id).await; - let mut ctx = CancellationCtx::new(); - let cancel_ctx = ctx.clone(); - let store = Arc::new(SnapshotStore::new()); - - tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(200)).await; - ctx.cancel(CancellationReason::GracefullyShutdown()); - }); - - runner.run_loop(Duration::from_millis(50), cancel_ctx, store, None).await.unwrap(); - - let events = audit.drain(); - let failed = - find_audit_event(&events, |p| matches!(p, AuditEventPayload::ElectionsTickFailed { .. })); - - assert_eq!(failed.payload.severity(), AuditSeverity::Error); - assert_eq!(failed.outcome, AuditOutcome::Failure); - assert_eq!(failed.target, AuditTarget::Elections { election_id: ELECTION_ID }); - let AuditEventPayload::ElectionsTickFailed { reason } = &failed.payload else { - unreachable!(); - }; - assert!(reason.contains("simulated elections_info failure")); -} - #[tokio::test] async fn stake_submitted_event_contains_correct_payload() { let node_id = "node-1"; @@ -3953,7 +3935,10 @@ async fn stake_skipped_event_has_skipped_outcome_and_warn_severity() { let ev = find_audit_event(&events, |p| { matches!( p, - AuditEventPayload::ElectionsStakeSkipped { reason: StakeSkipReason::NodeExcluded, .. } + AuditEventPayload::ElectionsStakeSkipped { + reason: StakeSkipReason::ElectionsDisabled, + .. + } ) }); payload_stake_skipped(&ev.payload); @@ -3964,7 +3949,7 @@ async fn stake_skipped_event_has_skipped_outcome_and_warn_severity() { } #[tokio::test] -async fn withdraw_processed_emits_tx_hash() { +async fn withdraw_processed_emits_msg_hash() { let node_id = "node-1"; let mut harness = TestHarness::new().with_toncore_nominator_pair(); @@ -3999,10 +3984,10 @@ async fn withdraw_processed_emits_tx_hash() { assert_eq!(ev.outcome, AuditOutcome::Success); assert_node_target(&ev.target, node_id, ELECTION_ID); - let AuditEventPayload::ElectionsWithdrawProcessed { tx_hash } = &ev.payload else { + let AuditEventPayload::ElectionsWithdrawProcessed { msg_hash } = &ev.payload else { unreachable!(); }; - assert!(!tx_hash.is_empty(), "tx_hash must be the sent message cell hash"); + assert!(!msg_hash.is_empty(), "msg_hash must be the outbound message cell repr_hash"); } #[tokio::test] @@ -4043,14 +4028,14 @@ async fn withdraw_failed_emits_error_string() { let events = audit.drain(); let ev = find_audit_event(&events, |p| { - matches!(p, AuditEventPayload::ElectionsWithdrawProcessFailed { .. }) + matches!(p, AuditEventPayload::ElectionsWithdrawFailed { .. }) }); assert_eq!(ev.payload.severity(), AuditSeverity::Error); assert_eq!(ev.outcome, AuditOutcome::Failure); assert_node_target(&ev.target, node_id, ELECTION_ID); - let AuditEventPayload::ElectionsWithdrawProcessFailed { reason } = &ev.payload else { + let AuditEventPayload::ElectionsWithdrawFailed { reason } = &ev.payload else { unreachable!(); }; assert!(reason.contains("simulated withdraw send_boc failure")); From fd6c66e295fde955bc0ac5287651e9b1b88932e6 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Mon, 8 Jun 2026 17:39:44 +0300 Subject: [PATCH 18/30] feat(audit): ring buffer, dedup stake_skipped, rename payload fields --- src/Cargo.lock | 1 + src/node-control/common/src/app_config.rs | 2 +- src/node-control/service/Cargo.toml | 1 + src/node-control/service/src/audit/enums.rs | 17 +- src/node-control/service/src/audit/event.rs | 63 +++--- src/node-control/service/src/audit/factory.rs | 28 ++- .../service/src/audit/jsonl_log.rs | 58 +++++ src/node-control/service/src/audit/mod.rs | 2 + .../service/src/audit/ring_buffer.rs | 207 ++++++++++++++++++ .../service/src/elections/runner.rs | 10 +- .../service/src/elections/runner_tests.rs | 10 +- .../service/src/http/auth_tests.rs | 2 + .../service/src/http/config_handlers_tests.rs | 1 + .../src/http/entity_crud_handlers_tests.rs | 2 + .../service/src/http/http_server_task.rs | 8 +- .../service/src/service_main_task.rs | 5 +- 16 files changed, 358 insertions(+), 59 deletions(-) create mode 100644 src/node-control/service/src/audit/ring_buffer.rs diff --git a/src/Cargo.lock b/src/Cargo.lock index ea8145f5..831ff6fe 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -4939,6 +4939,7 @@ dependencies = [ "http-body-util", "jsonwebtoken", "mockall", + "parking_lot", "secrets-vault", "serde", "serde_json", diff --git a/src/node-control/common/src/app_config.rs b/src/node-control/common/src/app_config.rs index 0ffa7cd4..68cb0455 100644 --- a/src/node-control/common/src/app_config.rs +++ b/src/node-control/common/src/app_config.rs @@ -960,7 +960,7 @@ fn default_audit_include_payload() -> bool { } fn default_audit_ring_buffer_capacity() -> usize { - 10_000 + 100 } fn default_audit_enabled() -> bool { diff --git a/src/node-control/service/Cargo.toml b/src/node-control/service/Cargo.toml index 9e9de3bc..cb96b782 100644 --- a/src/node-control/service/Cargo.toml +++ b/src/node-control/service/Cargo.toml @@ -29,6 +29,7 @@ base64 = "0.22" chrono = { version = "0.4", features = ["serde"] } thiserror = "1" uuid = { version = "1", features = ["serde", "v4", "v7"] } +parking_lot = "0.12.5" [dev-dependencies] mockall = "0.13" diff --git a/src/node-control/service/src/audit/enums.rs b/src/node-control/service/src/audit/enums.rs index 5777ec89..727b95ea 100644 --- a/src/node-control/service/src/audit/enums.rs +++ b/src/node-control/service/src/audit/enums.rs @@ -19,23 +19,18 @@ pub enum AuditEventPayload { }, #[serde(rename = "elections.stake_submitted")] - ElectionsStakeSubmitted { - stake_nanotons: String, - max_factor: u32, - policy: String, - submission_time: u64, - }, + ElectionsStakeSubmitted { stake: String, max_factor: u32, policy: String, submission_time: u64 }, #[serde(rename = "elections.stake_accepted")] - ElectionsStakeAccepted { stake_nanotons: String }, + ElectionsStakeAccepted { stake: String }, #[serde(rename = "elections.stake_skipped")] ElectionsStakeSkipped { reason: StakeSkipReason, #[serde(skip_serializing_if = "Option::is_none")] - required_nanotons: Option, + required: Option, #[serde(skip_serializing_if = "Option::is_none")] - available_nanotons: Option, + available: Option, }, #[serde(rename = "elections.stake_failed")] @@ -43,7 +38,7 @@ pub enum AuditEventPayload { #[serde(rename = "elections.stake_recovered")] ElectionsStakeRecovered { - amount_nanotons: String, + amount: String, #[serde(skip_serializing_if = "Option::is_none")] msg_hash: Option, }, @@ -62,7 +57,7 @@ pub enum AuditEventPayload { RewardsDistributionStarted { recipients_count: u32 }, #[serde(rename = "rewards.distribution_completed")] - RewardsDistributionCompleted { recipients_count: u32, total_nanotons: String }, + RewardsDistributionCompleted { recipients_count: u32, total: String }, #[serde(rename = "rewards.distribution_failed")] RewardsDistributionFailed { reason: String }, diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index 1f552036..7f6a8a95 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -67,13 +67,29 @@ pub struct AuditEvent { /// Named payload fields for [`AuditEvent::elections_stake_submitted`]. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ElectionsStakeSubmittedParams { - pub stake_nanotons: String, + pub stake: String, pub max_factor: u32, pub policy: String, pub submission_time: u64, } impl AuditEvent { + /// Returns a stable deduplication key for events that should appear at most once + /// per election in the audit log (e.g. `elections.stake_skipped` with a persistent + /// reason like `ElectionsDisabled`). + /// + /// Returns `None` for events that are always recorded without deduplication. + /// The key encodes `(event_category, node_id, election_id, reason)` so that + /// the same skip reason for the same node in the same election is written only once. + pub fn dedup_key(&self) -> Option { + if let AuditEventPayload::ElectionsStakeSkipped { reason, .. } = &self.payload + && let AuditTarget::Node { id: node_id, election_id: Some(election_id) } = &self.target + { + return Some(format!("stake_skipped:{node_id}:{election_id}:{reason:?}")); + } + None + } + /// Internal constructor that stamps `id`/`ts`. Crate-private so call sites /// must go through the typed constructors below, which bake the canonical /// outcome per event type. @@ -116,7 +132,7 @@ impl AuditEvent { Self::node_target(node_id, election_id), AuditOutcome::Success, AuditEventPayload::ElectionsStakeSubmitted { - stake_nanotons: params.stake_nanotons, + stake: params.stake, max_factor: params.max_factor, policy: params.policy, submission_time: params.submission_time, @@ -129,18 +145,14 @@ impl AuditEvent { node_id: impl Into, election_id: u64, reason: StakeSkipReason, - required_nanotons: Option, - available_nanotons: Option, + required: Option, + available: Option, ) -> Self { Self::new( actor, Self::node_target(node_id, election_id), AuditOutcome::Skipped, - AuditEventPayload::ElectionsStakeSkipped { - reason, - required_nanotons, - available_nanotons, - }, + AuditEventPayload::ElectionsStakeSkipped { reason, required, available }, ) } @@ -162,17 +174,14 @@ impl AuditEvent { actor: AuditActor, node_id: impl Into, election_id: u64, - amount_nanotons: impl Into, + amount: impl Into, msg_hash: Option, ) -> Self { Self::new( actor, Self::node_target(node_id, election_id), AuditOutcome::Success, - AuditEventPayload::ElectionsStakeRecovered { - amount_nanotons: amount_nanotons.into(), - msg_hash, - }, + AuditEventPayload::ElectionsStakeRecovered { amount: amount.into(), msg_hash }, ) } @@ -279,7 +288,7 @@ mod tests { AuditActor::service("elections-task"), AuditTarget::Node { id: "node1".into(), election_id: Some(1_779_265_552) }, AuditEventPayload::ElectionsStakeSubmitted { - stake_nanotons: "50000000000000".into(), + stake: "50000000000000".into(), max_factor: 196_608, policy: "adaptive_split50".into(), submission_time: 1_779_265_400, @@ -294,7 +303,7 @@ mod tests { "outcome": "success", "event_type": "elections.stake_submitted", "data": { - "stake_nanotons": "50000000000000", + "stake": "50000000000000", "max_factor": 196608, "policy": "adaptive_split50", "submission_time": 1779265400 @@ -313,8 +322,8 @@ mod tests { AuditTarget::Node { id: "node6".into(), election_id: Some(1_779_265_552) }, AuditEventPayload::ElectionsStakeSkipped { reason: StakeSkipReason::LowWalletBalance, - required_nanotons: Some("1200000000".into()), - available_nanotons: Some("900000000".into()), + required: Some("1200000000".into()), + available: Some("900000000".into()), }, ); @@ -327,8 +336,8 @@ mod tests { "event_type": "elections.stake_skipped", "data": { "reason": "low_wallet_balance", - "required_nanotons": "1200000000", - "available_nanotons": "900000000" + "required": "1200000000", + "available": "900000000" }, "actor": { "kind": "service", "id": "elections-task" }, "target": { "kind": "node", "id": "node6", "election_id": 1779265552 } @@ -373,29 +382,29 @@ mod tests { vec![ AuditEventPayload::ElectionsKeyGenerated { pubkey: Some("aabb".into()) }, AuditEventPayload::ElectionsStakeSubmitted { - stake_nanotons: "1".into(), + stake: "1".into(), max_factor: 1, policy: "all".into(), submission_time: 1, }, - AuditEventPayload::ElectionsStakeAccepted { stake_nanotons: "50000000000000".into() }, + AuditEventPayload::ElectionsStakeAccepted { stake: "50000000000000".into() }, AuditEventPayload::ElectionsStakeSkipped { reason: StakeSkipReason::WithdrawRequestsPending, - required_nanotons: None, - available_nanotons: None, + required: None, + available: None, }, AuditEventPayload::ElectionsStakeFailed { reason: "send failed".into() }, AuditEventPayload::ElectionsWithdrawProcessed { msg_hash: "abc".into() }, AuditEventPayload::ElectionsWithdrawFailed { reason: "send failed".into() }, AuditEventPayload::ElectionsStakeRecovered { - amount_nanotons: "50000000000000".into(), + amount: "50000000000000".into(), msg_hash: Some("def".into()), }, AuditEventPayload::ElectionsStakeRecoverFailed { reason: "send failed".into() }, AuditEventPayload::RewardsDistributionStarted { recipients_count: 3 }, AuditEventPayload::RewardsDistributionCompleted { recipients_count: 3, - total_nanotons: "9".into(), + total: "9".into(), }, AuditEventPayload::RewardsDistributionFailed { reason: "rpc".into() }, AuditEventPayload::RewardsRecipientSkipped { reason: "below_min".into() }, @@ -467,6 +476,6 @@ mod tests { assert!(cfg.include_payload); assert!(!cfg.record_client_ip); assert!(!cfg.ip_anonymize); - assert_eq!(cfg.ring_buffer_capacity, 10_000); + assert_eq!(cfg.ring_buffer_capacity, 100); } } diff --git a/src/node-control/service/src/audit/factory.rs b/src/node-control/service/src/audit/factory.rs index 6f5b9c7f..135c6edd 100644 --- a/src/node-control/service/src/audit/factory.rs +++ b/src/node-control/service/src/audit/factory.rs @@ -10,18 +10,31 @@ use crate::audit::{ AuditLogConfig, jsonl_log::{AuditInitError, JsonlAuditLog}, log::{AuditLog, NoopAuditLog}, + ring_buffer::AuditEventBuffer, }; use std::sync::Arc; +/// Output of [`AuditLogFactory::from_config`]: the write handle and a separate ring +/// buffer for the REST read-path. Both are always present; the ring is empty (capacity 0 +/// is normalised to 1) when `config.enabled` is false. +pub struct AuditComponents { + pub log: Arc, + pub ring: Arc, +} + pub struct AuditLogFactory; impl AuditLogFactory { - pub async fn from_config(config: &AuditLogConfig) -> Result, AuditInitError> { + pub async fn from_config(config: &AuditLogConfig) -> Result { if !config.enabled { - return Ok(Arc::new(NoopAuditLog)); + return Ok(AuditComponents { + log: Arc::new(NoopAuditLog), + ring: AuditEventBuffer::new(0), + }); } let log = JsonlAuditLog::start(config.clone()).await?; - Ok(log) + let ring = log.ring(); + Ok(AuditComponents { log, ring }) } } @@ -38,8 +51,11 @@ mod tests { #[tokio::test] async fn factory_returns_noop_when_disabled() { let cfg = AuditLogConfig { enabled: false, ..AuditLogConfig::default() }; - let log = AuditLogFactory::from_config(&cfg).await.expect("factory init"); + let AuditComponents { log, ring } = + AuditLogFactory::from_config(&cfg).await.expect("factory init"); log.record(sample_event()).await; + // Noop ring: event not pushed (NoopAuditLog doesn't touch the ring). + let _ = ring; } #[tokio::test] @@ -47,8 +63,10 @@ mod tests { let dir = tempdir().unwrap(); let mut cfg = AuditLogConfig { enabled: true, ..AuditLogConfig::default() }; cfg.path = dir.path().join("audit.jsonl"); - let log = AuditLogFactory::from_config(&cfg).await.expect("factory init"); + let AuditComponents { log, ring } = + AuditLogFactory::from_config(&cfg).await.expect("factory init"); log.record(sample_event()).await; + assert_eq!(ring.len(), 1, "ring should contain the recorded event"); log.shutdown().await; } } diff --git a/src/node-control/service/src/audit/jsonl_log.rs b/src/node-control/service/src/audit/jsonl_log.rs index 397f32b0..eec1314f 100644 --- a/src/node-control/service/src/audit/jsonl_log.rs +++ b/src/node-control/service/src/audit/jsonl_log.rs @@ -10,6 +10,7 @@ use crate::audit::{ AuditEvent, AuditLogConfig, jsonl_writer::{AuditCommand, AuditWriter}, log::AuditLog, + ring_buffer::AuditEventBuffer, }; use async_trait::async_trait; use std::{ @@ -46,6 +47,9 @@ pub struct JsonlAuditLog { shutdown_gate: AsyncMutex<()>, dropped_events: Arc, config: Arc, + /// In-memory ring buffer for the REST read-path. Populated in `record()` before + /// the channel send so events appear immediately and survive queue overflow. + ring: Arc, /// Writer task handle, consumed by the first [`AuditLog::shutdown`] call so /// callers can await the final drain/flush. `None` after shutdown. writer: Mutex>>, @@ -91,6 +95,11 @@ impl JsonlAuditLog { } } + /// Returns a handle to the ring buffer for the REST read-path. + pub fn ring(&self) -> Arc { + self.ring.clone() + } + /// Wires the channels and spawns the writer task for an already-opened writer. fn spawn_writer( config: Arc, @@ -100,12 +109,14 @@ impl JsonlAuditLog { let (tx, rx) = mpsc::channel(config.queue_capacity.max(1)); let (shutdown_tx, shutdown_rx) = oneshot::channel(); let handle = tokio::spawn(writer.run(rx, shutdown_rx)); + let ring = AuditEventBuffer::new(config.ring_buffer_capacity); Arc::new(Self { sender: tx, shutdown_tx: Mutex::new(Some(shutdown_tx)), shutdown_gate: AsyncMutex::new(()), dropped_events, + ring, config, writer: Mutex::new(Some(handle)), }) @@ -153,6 +164,18 @@ impl AuditLog for JsonlAuditLog { } async fn record(&self, event: AuditEvent) { + // Deduplication: drop events whose key already exists in the ring. + // Prevents e.g. repeated elections.stake_skipped for the same (node, election, reason). + if let Some(key) = event.dedup_key() + && self.ring.contains_dedup_key(&key) + { + return; + } + + // Push into ring first: readers see the event immediately and even + // queue-dropped events remain accessible on the REST read-path. + self.ring.push(event.clone()); + let event_id = event.id; let source = event.payload.source(); let cmd = AuditCommand::Event(Box::new(event)); @@ -224,4 +247,39 @@ mod tests { "shutdown should not block indefinitely behind queue backpressure" ); } + + /// Ring buffer must contain the event even when the writer queue is full and + /// the event is dropped from the channel (dropped_events increments). + /// + /// Uses `write_delay = 500ms` so the writer is slow relative to the 10ms + /// channel timeout, guaranteeing that most events are dropped from the channel + /// while still being captured in the ring (populated before the send attempt). + #[tokio::test(flavor = "current_thread")] + async fn record_pushes_to_ring_even_when_queue_full() { + let dir = tempdir().unwrap(); + let cfg = AuditLogConfig { + path: dir.path().join("audit.jsonl"), + queue_capacity: 1, + queue_full_timeout_ms: 10, + ring_buffer_capacity: 200, + batch_interval_ms: 60_000, + batch_max_events: 1, + ..AuditLogConfig::default() + }; + let log = + JsonlAuditLog::start_with_write_delay(cfg, Duration::from_millis(500)).await.unwrap(); + + for i in 0..50 { + log.record(sample_event(&format!("ev-{i}"))).await; + } + // Let the runtime settle so the dropped_events counter is up to date. + tokio::time::sleep(Duration::from_millis(100)).await; + + // Ring captures every record() call regardless of channel state. + assert_eq!(log.ring().len(), 50, "ring must contain every record() call"); + // Writer is ~500 ms/event; timeout is 10 ms → most events dropped from channel. + assert!(log.dropped_events() > 0, "some events must have been dropped from channel"); + + log.shutdown().await; + } } diff --git a/src/node-control/service/src/audit/mod.rs b/src/node-control/service/src/audit/mod.rs index 91f79940..80878c63 100644 --- a/src/node-control/service/src/audit/mod.rs +++ b/src/node-control/service/src/audit/mod.rs @@ -15,6 +15,7 @@ pub mod jsonl_log; pub mod jsonl_writer; pub mod log; pub mod participant; +pub mod ring_buffer; pub use common::app_config::AuditLogConfig; pub use enums::{ @@ -27,3 +28,4 @@ pub use in_memory::InMemoryAuditLog; pub use jsonl_log::AuditInitError; pub use log::{AuditLog, NoopAuditLog}; pub use participant::{AuditActor, AuditTarget}; +pub use ring_buffer::AuditEventBuffer; diff --git a/src/node-control/service/src/audit/ring_buffer.rs b/src/node-control/service/src/audit/ring_buffer.rs new file mode 100644 index 00000000..68e87be4 --- /dev/null +++ b/src/node-control/service/src/audit/ring_buffer.rs @@ -0,0 +1,207 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use crate::audit::AuditEvent; +use parking_lot::RwLock; +use std::{collections::VecDeque, sync::Arc}; + +/// Fixed-capacity FIFO buffer of recent audit events for the REST read-path. +/// +/// Populated on every [`crate::audit::log::AuditLog::record`] call — independent of the +/// JSONL writer task. Events are available immediately and remain visible even when the +/// writer queue is full. +/// +/// `parking_lot::RwLock` is used intentionally: the critical section is purely in-memory +/// (no IO, no `.await`), so a synchronous lock is correct and avoids the overhead of an +/// async lock. The lock is **never held across an `.await` point**. +pub struct AuditEventBuffer { + inner: RwLock>, + capacity: usize, +} + +impl AuditEventBuffer { + /// Creates a new buffer wrapped in `Arc`. `capacity` is the maximum number of events + /// retained; when full, the oldest event is evicted (FIFO). + pub fn new(capacity: usize) -> Arc { + Arc::new(Self { + inner: RwLock::new(VecDeque::with_capacity(capacity.max(1))), + capacity: capacity.max(1), + }) + } + + /// Appends `event`, evicting the oldest entry when the buffer is at capacity. + pub fn push(&self, event: AuditEvent) { + let mut buf = self.inner.write(); + if buf.len() == self.capacity { + buf.pop_front(); + } + buf.push_back(event); + } + + /// Returns a point-in-time snapshot of all buffered events (oldest first). + pub fn snapshot(&self) -> Vec { + self.inner.read().iter().cloned().collect() + } + + /// Filters under the read lock — avoids cloning events that don't match `predicate`. + pub fn filter_collect(&self, predicate: F) -> Vec + where + F: Fn(&AuditEvent) -> bool, + { + self.inner.read().iter().filter(|e| predicate(e)).cloned().collect() + } + + pub fn len(&self) -> usize { + self.inner.read().len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn capacity(&self) -> usize { + self.capacity + } + + /// Returns `true` if the buffer already contains an event with the given + /// deduplication key. Used by [`crate::audit::jsonl_log::JsonlAuditLog`] to + /// suppress repeated identical `elections.stake_skipped` events within one election. + pub fn contains_dedup_key(&self, key: &str) -> bool { + self.inner.read().iter().any(|e| e.dedup_key().as_deref() == Some(key)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::audit::{AuditEvent, AuditSource}; + use std::sync::Arc; + + fn ev(tag: &str) -> AuditEvent { + AuditEvent::system_service_started(tag) + } + + // ── original tests ──────────────────────────────────────────────────────── + + #[test] + fn len_and_is_empty() { + let buf = AuditEventBuffer::new(5); + assert!(buf.is_empty()); + buf.push(ev("x")); + assert_eq!(buf.len(), 1); + assert!(!buf.is_empty()); + } + + // ── required new tests ──────────────────────────────────────────────────── + + #[test] + fn push_below_capacity_keeps_all() { + let buf = AuditEventBuffer::new(5); + for i in 0..4 { + buf.push(ev(&format!("{i}"))); + } + assert_eq!(buf.len(), 4); + assert_eq!(buf.snapshot().len(), 4); + } + + #[test] + fn push_at_capacity_evicts_oldest() { + let buf = AuditEventBuffer::new(3); + let a = ev("a"); + let b = ev("b"); + let c = ev("c"); + let d = ev("d"); + let a_id = a.id; + let d_id = d.id; + buf.push(a); + buf.push(b); + buf.push(c); + // at capacity, push d — must evict a + buf.push(d); + + let snap = buf.snapshot(); + assert_eq!(snap.len(), 3, "len must stay at capacity"); + assert!(!snap.iter().any(|e| e.id == a_id), "oldest (a) must be evicted"); + assert!(snap.iter().any(|e| e.id == d_id), "newest (d) must be present"); + } + + #[test] + fn snapshot_returns_in_order() { + let buf = AuditEventBuffer::new(10); + let ids: Vec<_> = (0..5) + .map(|i| { + let e = ev(&format!("{i}")); + let id = e.id; + buf.push(e); + id + }) + .collect(); + + let snap = buf.snapshot(); + assert_eq!(snap.len(), 5); + for (i, expected_id) in ids.iter().enumerate() { + assert_eq!(&snap[i].id, expected_id, "event at index {i} is out of order"); + } + } + + #[test] + fn filter_collect_only_matching_events() { + let buf = AuditEventBuffer::new(20); + for _ in 0..5 { + buf.push(ev("system")); + } + let matched = buf.filter_collect(|e| e.payload.source() == AuditSource::System); + assert_eq!(matched.len(), 5); + + let unmatched = buf.filter_collect(|e| e.payload.source() == AuditSource::Elections); + assert!(unmatched.is_empty()); + } + + #[test] + fn concurrent_push_and_snapshot_no_panic() { + let buf = Arc::new(AuditEventBuffer::new(50)); + let mut handles = vec![]; + + // 8 producer threads each push 100 events + for t in 0..8u8 { + let b = buf.clone(); + handles.push(std::thread::spawn(move || { + for i in 0..100u32 { + b.push(AuditEvent::system_service_started(&format!("t{t}-{i}"))); + } + })); + } + + // 1 reader thread continuously snapshots while producers are active + let b = buf.clone(); + handles.push(std::thread::spawn(move || { + for _ in 0..200 { + let snap = b.snapshot(); + // capacity is 50 — snapshot must never exceed it + assert!(snap.len() <= 50); + } + })); + + for h in handles { + h.join().expect("thread panicked"); + } + // Final state: buffer should be full (800 pushes into cap=50) + assert_eq!(buf.len(), 50); + } + + #[test] + fn zero_capacity_buffer_silently_drops() { + // capacity=0 is normalised to 1 internally; no crash, snapshot is non-empty after push + let buf = AuditEventBuffer::new(0); + assert_eq!(buf.capacity(), 1, "capacity normalised to 1"); + buf.push(ev("a")); + buf.push(ev("b")); // evicts first, keeps second + let snap = buf.snapshot(); + assert_eq!(snap.len(), 1, "only the latest event is retained"); + } +} diff --git a/src/node-control/service/src/elections/runner.rs b/src/node-control/service/src/elections/runner.rs index ba9a2a02..b6c112b6 100644 --- a/src/node-control/service/src/elections/runner.rs +++ b/src/node-control/service/src/elections/runner.rs @@ -1981,8 +1981,8 @@ async fn elections_audit_stake_skipped( election_id: u64, node_id: &str, reason: StakeSkipReason, - required_nanotons: Option, - available_nanotons: Option, + required: Option, + available: Option, ) { audit .record(AuditEvent::elections_stake_skipped( @@ -1990,8 +1990,8 @@ async fn elections_audit_stake_skipped( node_id, election_id, reason, - required_nanotons.map(nanotons_to_dec_string), - available_nanotons.map(nanotons_to_dec_string), + required.map(nanotons_to_dec_string), + available.map(nanotons_to_dec_string), )) .await; } @@ -2028,7 +2028,7 @@ async fn elections_audit_stake_submitted( node_id, participant.election_id, ElectionsStakeSubmittedParams { - stake_nanotons: nanotons_to_dec_string(stake), + stake: nanotons_to_dec_string(stake), max_factor: participant.max_factor, policy: node.stake_policy.to_string(), submission_time, diff --git a/src/node-control/service/src/elections/runner_tests.rs b/src/node-control/service/src/elections/runner_tests.rs index d32864f9..8d184094 100644 --- a/src/node-control/service/src/elections/runner_tests.rs +++ b/src/node-control/service/src/elections/runner_tests.rs @@ -3895,16 +3895,12 @@ async fn stake_submitted_event_contains_correct_payload() { assert_eq!(ev.outcome, AuditOutcome::Success); assert_node_target(&ev.target, node_id, ELECTION_ID); - let AuditEventPayload::ElectionsStakeSubmitted { - stake_nanotons, - max_factor, - policy, - submission_time, - } = &ev.payload + let AuditEventPayload::ElectionsStakeSubmitted { stake, max_factor, policy, submission_time } = + &ev.payload else { unreachable!(); }; - assert_eq!(stake_nanotons, &expected_stake.to_string()); + assert_eq!(stake, &expected_stake.to_string()); assert_eq!(*max_factor, 196608); assert_eq!(policy, "split50"); assert!(*submission_time > 0); diff --git a/src/node-control/service/src/http/auth_tests.rs b/src/node-control/service/src/http/auth_tests.rs index 0a24d8e4..3da2298c 100644 --- a/src/node-control/service/src/http/auth_tests.rs +++ b/src/node-control/service/src/http/auth_tests.rs @@ -174,6 +174,7 @@ async fn state_with_auth() -> AppState { login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), audit: Arc::new(crate::audit::log::NoopAuditLog), + audit_ring: crate::audit::AuditEventBuffer::new(0), } } @@ -188,6 +189,7 @@ async fn state_no_auth() -> AppState { login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), audit: Arc::new(crate::audit::log::NoopAuditLog), + audit_ring: crate::audit::AuditEventBuffer::new(0), } } diff --git a/src/node-control/service/src/http/config_handlers_tests.rs b/src/node-control/service/src/http/config_handlers_tests.rs index a335560c..3856d3d8 100644 --- a/src/node-control/service/src/http/config_handlers_tests.rs +++ b/src/node-control/service/src/http/config_handlers_tests.rs @@ -75,6 +75,7 @@ async fn state_from_cfg(cfg: AppConfig) -> AppState { login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), audit: Arc::new(crate::audit::log::NoopAuditLog), + audit_ring: crate::audit::AuditEventBuffer::new(0), } } diff --git a/src/node-control/service/src/http/entity_crud_handlers_tests.rs b/src/node-control/service/src/http/entity_crud_handlers_tests.rs index 557f36dc..0cd76c06 100644 --- a/src/node-control/service/src/http/entity_crud_handlers_tests.rs +++ b/src/node-control/service/src/http/entity_crud_handlers_tests.rs @@ -140,6 +140,7 @@ async fn app_state(cfg: AppConfig) -> AppState { login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), audit: Arc::new(crate::audit::log::NoopAuditLog), + audit_ring: crate::audit::AuditEventBuffer::new(0), } } @@ -159,6 +160,7 @@ async fn app_state_with_path(cfg: AppConfig, path: std::path::PathBuf) -> AppSta login_rate_limiter: Arc::new(tokio::sync::Mutex::new(Default::default())), config_changed: Arc::new(tokio::sync::Notify::new()), audit: Arc::new(crate::audit::log::NoopAuditLog), + audit_ring: crate::audit::AuditEventBuffer::new(0), } } diff --git a/src/node-control/service/src/http/http_server_task.rs b/src/node-control/service/src/http/http_server_task.rs index d4a58e04..45ed06f8 100644 --- a/src/node-control/service/src/http/http_server_task.rs +++ b/src/node-control/service/src/http/http_server_task.rs @@ -23,7 +23,7 @@ use super::{ login_rate_limiter::{LoginRateLimiter, login_limiter_key}, }; use crate::{ - audit::log::AuditLog, + audit::{AuditEventBuffer, log::AuditLog}, auth::{ Claims, jwt::JwtAuth, @@ -55,6 +55,9 @@ pub struct AppState { /// (entity CRUD, ton-http-api) so the service loop can rebuild caches. pub config_changed: Arc, pub audit: Arc, + /// In-memory ring buffer for the REST read-path (e.g. GET /v1/elections). + /// Never read from disk on the hot path. + pub audit_ring: Arc, } pub async fn run( @@ -64,6 +67,7 @@ pub async fn run( tasks: HashMap<&'static str, Arc>, config_changed: Arc, audit: Arc, + audit_ring: Arc, ) { tracing::info!("http-server task started"); @@ -122,6 +126,7 @@ pub async fn run( login_rate_limiter, config_changed, audit, + audit_ring, }; let app = routes(enable_swagger, state); @@ -1065,6 +1070,7 @@ mod tests { login_rate_limiter: Arc::new(tokio::sync::Mutex::new(LoginRateLimiter::default())), config_changed: Arc::new(tokio::sync::Notify::new()), audit: Arc::new(NoopAuditLog), + audit_ring: crate::audit::AuditEventBuffer::new(0), } } diff --git a/src/node-control/service/src/service_main_task.rs b/src/node-control/service/src/service_main_task.rs index 498fbecd..53842281 100644 --- a/src/node-control/service/src/service_main_task.rs +++ b/src/node-control/service/src/service_main_task.rs @@ -7,7 +7,7 @@ * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ use crate::{ - audit::AuditLogFactory, + audit::{AuditLogFactory, factory::AuditComponents}, elections::election_task::BindingStatusCallback, http::http_server_task, runtime_config::RuntimeConfigStore, @@ -54,7 +54,7 @@ pub async fn run_with_config( .context("initialize runtime config store")?; let runtime_cfg = Arc::new(runtime_cfg); - let audit = + let AuditComponents { log: audit, ring: audit_ring } = AuditLogFactory::from_config(&app_cfg.audit_log).await.context("audit log init failed")?; let store = Arc::new(SnapshotStore::new()); @@ -124,6 +124,7 @@ pub async fn run_with_config( tasks.clone(), config_changed.clone(), audit.clone(), + audit_ring, )); let max_wait = std::time::Duration::from_secs(10); From 7f5ad771cc2b3aa4fc298ca1e7ede7242d132322 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Mon, 8 Jun 2026 18:23:48 +0300 Subject: [PATCH 19/30] fix:fmt --- .../src/commands/nodectl/service_api_cmd.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/node-control/commands/src/commands/nodectl/service_api_cmd.rs b/src/node-control/commands/src/commands/nodectl/service_api_cmd.rs index d865d350..4c7cdee8 100644 --- a/src/node-control/commands/src/commands/nodectl/service_api_cmd.rs +++ b/src/node-control/commands/src/commands/nodectl/service_api_cmd.rs @@ -796,10 +796,8 @@ fn print_recent_events_table(body: &serde_json::Value) { }; let target = ev.get("target"); - let node_id = target - .and_then(|t| t.get("id")) - .and_then(serde_json::Value::as_str) - .unwrap_or("-"); + let node_id = + target.and_then(|t| t.get("id")).and_then(serde_json::Value::as_str).unwrap_or("-"); let data = ev.get("data"); let details = format_event_details(short_event, data); @@ -831,7 +829,12 @@ fn format_event_details(event_type: &str, data: Option<&serde_json::Value>) -> S data.get("required").and_then(serde_json::Value::as_str), data.get("available").and_then(serde_json::Value::as_str), ) { - format!("reason={} req={} avail={}", reason, display_tons_from_str(req), display_tons_from_str(avail)) + format!( + "reason={} req={} avail={}", + reason, + display_tons_from_str(req), + display_tons_from_str(avail) + ) } else { format!("reason={}", reason) } From b65c3b508cecdd9cf3cc259885cf461a0549cce1 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Tue, 9 Jun 2026 11:11:00 +0300 Subject: [PATCH 20/30] fix: resolve post-merge compilation errors after integrating SMA-103 - Add `use serde::{Deserialize, Serialize}` to enums.rs (bare derives on StakeSkipReason/ConfigFieldChange/AuditOutcome broke after SMA-103 merge) - Add ElectionsTickFailed arm to severity() and source() match blocks - Restore REST event constructors to event.rs (rest_api_auth_login_success, rest_api_auth_login_rejected, rest_api_token_rejected, rest_api_config_updated) removed by SMA-103 merge; required by rest_audit.rs (SMA-104) - Fix RestApiAuthLoginSuccess -> RestApiAuthLoginSucceeded in auth_tests.rs Co-authored-by: Cursor --- src/node-control/service/src/audit/enums.rs | 8 ++- src/node-control/service/src/audit/event.rs | 52 +++++++++++++++++++ .../service/src/http/auth_tests.rs | 2 +- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/src/node-control/service/src/audit/enums.rs b/src/node-control/service/src/audit/enums.rs index 5ec29d04..6a2b39d3 100644 --- a/src/node-control/service/src/audit/enums.rs +++ b/src/node-control/service/src/audit/enums.rs @@ -6,6 +6,8 @@ * * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ +use serde::{Deserialize, Serialize}; + #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] #[serde(tag = "event_type", content = "data", rename_all = "snake_case")] #[non_exhaustive] @@ -177,7 +179,8 @@ impl AuditEventPayload { | RestApiTokenRejected { .. } | SystemAuditEventsDropped { .. } => Warn, - ElectionsStakeFailed { .. } + ElectionsTickFailed { .. } + | ElectionsStakeFailed { .. } | ElectionsStakeRecoverFailed { .. } | ElectionsWithdrawFailed { .. } | RewardsDistributionFailed { .. } => Error, @@ -188,7 +191,8 @@ impl AuditEventPayload { use AuditEventPayload::*; use AuditSource::*; match self { - ElectionsKeyGenerated { .. } + ElectionsTickFailed { .. } + | ElectionsKeyGenerated { .. } | ElectionsStakeSubmitted { .. } | ElectionsStakeAccepted { .. } | ElectionsStakeSkipped { .. } diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index f56bb1cd..fb1e465d 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -218,6 +218,58 @@ impl AuditEvent { ) } + pub fn rest_api_auth_login_success(actor: AuditActor, user_id: impl Into) -> Self { + Self::new( + actor, + AuditTarget::User { id: user_id.into() }, + AuditOutcome::Success, + AuditEventPayload::RestApiAuthLoginSucceeded {}, + ) + } + + pub fn rest_api_auth_login_rejected( + actor: AuditActor, + user_id: impl Into, + reason: impl Into, + ) -> Self { + Self::new( + actor, + AuditTarget::User { id: user_id.into() }, + AuditOutcome::Failure, + AuditEventPayload::RestApiAuthLoginRejected { reason: reason.into() }, + ) + } + + pub fn rest_api_token_rejected( + actor: AuditActor, + user_id: impl Into, + reason: impl Into, + ) -> Self { + Self::new( + actor, + AuditTarget::User { id: user_id.into() }, + AuditOutcome::Failure, + AuditEventPayload::RestApiTokenRejected { reason: reason.into() }, + ) + } + + pub fn rest_api_config_updated( + actor: AuditActor, + config_id: impl Into, + operation: impl Into, + changes: Vec, + ) -> Self { + Self::new( + actor, + AuditTarget::Config { id: config_id.into() }, + AuditOutcome::Success, + AuditEventPayload::RestApiConfigUpdated { + operation: operation.into(), + changes, + }, + ) + } + pub fn system_service_started(version: impl Into) -> Self { Self::new( AuditActor::System, diff --git a/src/node-control/service/src/http/auth_tests.rs b/src/node-control/service/src/http/auth_tests.rs index 4961028d..a719b573 100644 --- a/src/node-control/service/src/http/auth_tests.rs +++ b/src/node-control/service/src/http/auth_tests.rs @@ -720,7 +720,7 @@ async fn login_success_emits_audit_event() { let events = audit.drain(); assert_eq!(events.len(), 1); - assert!(matches!(events[0].payload, AuditEventPayload::RestApiAuthLoginSuccess {})); + assert!(matches!(events[0].payload, AuditEventPayload::RestApiAuthLoginSucceeded {})); let body = serde_json::to_string(&events[0]).unwrap(); assert!(!body.contains("pass1")); } From a2d19bfeff78f9c3127a2cb938b7fad2ca1dce72 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Tue, 9 Jun 2026 11:12:19 +0300 Subject: [PATCH 21/30] fix:fmt --- src/node-control/service/src/audit/event.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index fb1e465d..2f3d5a36 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -263,10 +263,7 @@ impl AuditEvent { actor, AuditTarget::Config { id: config_id.into() }, AuditOutcome::Success, - AuditEventPayload::RestApiConfigUpdated { - operation: operation.into(), - changes, - }, + AuditEventPayload::RestApiConfigUpdated { operation: operation.into(), changes }, ) } From 7d6edf775e7406f51b3233e756699dfea7b9e9a5 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Tue, 9 Jun 2026 11:17:21 +0300 Subject: [PATCH 22/30] remove: drop unused ElectionsTickFailed variant from AuditEventPayload Co-authored-by: Cursor --- src/node-control/service/src/audit/enums.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/node-control/service/src/audit/enums.rs b/src/node-control/service/src/audit/enums.rs index 6a2b39d3..d9507e7b 100644 --- a/src/node-control/service/src/audit/enums.rs +++ b/src/node-control/service/src/audit/enums.rs @@ -12,9 +12,6 @@ use serde::{Deserialize, Serialize}; #[serde(tag = "event_type", content = "data", rename_all = "snake_case")] #[non_exhaustive] pub enum AuditEventPayload { - #[serde(rename = "elections.tick_failed")] - ElectionsTickFailed { reason: String }, - #[serde(rename = "elections.key_generated")] ElectionsKeyGenerated { #[serde(skip_serializing_if = "Option::is_none")] @@ -179,8 +176,7 @@ impl AuditEventPayload { | RestApiTokenRejected { .. } | SystemAuditEventsDropped { .. } => Warn, - ElectionsTickFailed { .. } - | ElectionsStakeFailed { .. } + ElectionsStakeFailed { .. } | ElectionsStakeRecoverFailed { .. } | ElectionsWithdrawFailed { .. } | RewardsDistributionFailed { .. } => Error, @@ -191,8 +187,7 @@ impl AuditEventPayload { use AuditEventPayload::*; use AuditSource::*; match self { - ElectionsTickFailed { .. } - | ElectionsKeyGenerated { .. } + ElectionsKeyGenerated { .. } | ElectionsStakeSubmitted { .. } | ElectionsStakeAccepted { .. } | ElectionsStakeSkipped { .. } From c8085e0d5e46c523a2958eef35ea56f643f82dcf Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Tue, 9 Jun 2026 11:37:34 +0300 Subject: [PATCH 23/30] fix: imports --- src/node-control/service/src/audit/enums.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node-control/service/src/audit/enums.rs b/src/node-control/service/src/audit/enums.rs index d9507e7b..5777ec89 100644 --- a/src/node-control/service/src/audit/enums.rs +++ b/src/node-control/service/src/audit/enums.rs @@ -8,7 +8,7 @@ */ use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "event_type", content = "data", rename_all = "snake_case")] #[non_exhaustive] pub enum AuditEventPayload { From f6a28f0b1b7f600637c1791d26924fa429f8ded5 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Tue, 9 Jun 2026 15:45:42 +0300 Subject: [PATCH 24/30] fix: post-merge fixups after integrating remote sma-104 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename ElectionsStakeSubmittedParams.stake_nanotons → stake and elections_stake_recovered param amount_nanotons → amount to match the enum field names introduced by sma-104 (which dropped _nanotons suffixes from all payload fields). - Remove dead duplicate function adaptive_split50_defer_reason that was synthesised by the ort merge strategy from both SMA-103 and SMA-104 versions of adaptive_strategy.rs; the identical logic with node_id is already in adaptive_split50_status. - Update runner.rs and event.rs test fixtures to use the renamed fields. All 310 lib tests pass. Co-authored-by: Cursor --- src/node-control/service/src/audit/event.rs | 10 +-- .../src/elections/adaptive_strategy.rs | 49 ----------- .../service/src/elections/runner.rs | 2 +- .../scripts/add-nominators-to-pool.ts | 85 ++++++++++++++----- 4 files changed, 70 insertions(+), 76 deletions(-) diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index 4772cf49..5bbf653d 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -67,7 +67,7 @@ pub struct AuditEvent { /// Named payload fields for [`AuditEvent::elections_stake_submitted`]. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ElectionsStakeSubmittedParams { - pub stake_nanotons: String, + pub stake: String, pub max_factor: u32, pub policy: String, pub submission_time: u64, @@ -132,7 +132,7 @@ impl AuditEvent { Self::node_target(node_id, election_id), AuditOutcome::Success, AuditEventPayload::ElectionsStakeSubmitted { - stake_nanotons: params.stake_nanotons, + stake: params.stake, max_factor: params.max_factor, policy: params.policy, submission_time: params.submission_time, @@ -174,7 +174,7 @@ impl AuditEvent { actor: AuditActor, node_id: impl Into, election_id: u64, - amount_nanotons: impl Into, + amount: impl Into, msg_hash: Option, ) -> Self { Self::new( @@ -182,7 +182,7 @@ impl AuditEvent { Self::node_target(node_id, election_id), AuditOutcome::Success, AuditEventPayload::ElectionsStakeRecovered { - amount_nanotons: amount_nanotons.into(), + amount: amount.into(), msg_hash, }, ) @@ -449,7 +449,7 @@ mod tests { AuditEventPayload::ElectionsWithdrawProcessed { msg_hash: "abc".into() }, AuditEventPayload::ElectionsWithdrawFailed { reason: "send failed".into() }, AuditEventPayload::ElectionsStakeRecovered { - amount_nanotons: "50000000000000".into(), + amount: "50000000000000".into(), msg_hash: Some("def".into()), }, AuditEventPayload::ElectionsStakeRecoverFailed { reason: "send failed".into() }, diff --git a/src/node-control/service/src/elections/adaptive_strategy.rs b/src/node-control/service/src/elections/adaptive_strategy.rs index 9a5176c8..8727850a 100644 --- a/src/node-control/service/src/elections/adaptive_strategy.rs +++ b/src/node-control/service/src/elections/adaptive_strategy.rs @@ -90,55 +90,6 @@ pub(crate) fn adaptive_split50_status( None } -/// When [`is_adaptive_split50_ready`] would return `false`, reports which wait gate blocked. -pub(crate) fn adaptive_split50_defer_reason( - elections_info: &ElectionsInfo, - cfg15_start_before: u32, - cfg15_end_before: u32, - cfg16: &ConfigParam16, - sleep_pct: f64, - waiting_pct: f64, -) -> Option { - let min_validators = cfg16.min_validators.as_u16() as usize; - let participants_count = elections_info.participants.len(); - let election_duration = cfg15_start_before.saturating_sub(cfg15_end_before) as u64; - - if election_duration == 0 { - tracing::warn!( - "node [{}] adaptive_split50: election_duration=0, skipping wait logic", - node_id - ); - return None; - } - - let election_start = elections_info.elect_close.saturating_sub(election_duration); - let sleep_deadline = election_start + (election_duration as f64 * sleep_pct) as u64; - let wait_deadline = election_start + (election_duration as f64 * waiting_pct) as u64; - let now = common::time_format::now(); - - if now < sleep_deadline { - tracing::info!( - "node [{}] adaptive_split50: sleep period, now < sleep_deadline={}", - node_id, - common::time_format::format_ts(sleep_deadline) - ); - return Some(AdaptiveDeferReason::SleepPeriod); - } - - if participants_count < min_validators && now < wait_deadline { - tracing::info!( - "node [{}] adaptive_split50: waiting for participants ({}/{}), deadline={}", - node_id, - participants_count, - min_validators, - common::time_format::format_ts(wait_deadline) - ); - return Some(AdaptiveDeferReason::WaitingForParticipants); - } - - None -} - /// Calculate stake for AdaptiveSplit50 policy. /// /// Determines min_eff_stake from current emulation and/or past elections, diff --git a/src/node-control/service/src/elections/runner.rs b/src/node-control/service/src/elections/runner.rs index ce296d71..b6c112b6 100644 --- a/src/node-control/service/src/elections/runner.rs +++ b/src/node-control/service/src/elections/runner.rs @@ -2028,7 +2028,7 @@ async fn elections_audit_stake_submitted( node_id, participant.election_id, ElectionsStakeSubmittedParams { - stake_nanotons: nanotons_to_dec_string(stake), + stake: nanotons_to_dec_string(stake), max_factor: participant.max_factor, policy: node.stake_policy.to_string(), submission_time, diff --git a/src/node/tests/test_load_net/scripts/add-nominators-to-pool.ts b/src/node/tests/test_load_net/scripts/add-nominators-to-pool.ts index b94769aa..247dafdc 100644 --- a/src/node/tests/test_load_net/scripts/add-nominators-to-pool.ts +++ b/src/node/tests/test_load_net/scripts/add-nominators-to-pool.ts @@ -19,6 +19,8 @@ * MASTER_WALLET_ID (default 42) * NOMINATOR_SUBWALLET_BASE (default 10000) * NOMINATOR_WORKCHAIN (default **0** — required by pool; do not use -1 for nominators) + * HIGHLOAD_WORKCHAIN (default **0** — must match nominators; orchestrator deploys nominator wallets in-batch on the same wc) + * HIGHLOAD_BATCH_CHUNK (default 10) — max messages per sendBatch (same wc; avoids huge BOC on singlehost) * NOMINATOR_FUND_EXTRA_TON (default 0.15) * POOL_INFO_DELAY_MS — wait before get_pool_data dump (default 5000) * TONCORE_POOL_INFO_EXTRA — comma-separated extra pool addresses to print after (same API) @@ -38,6 +40,10 @@ const DEFAULT_COUNT = 40; const DEFAULT_SUBWALLET_BASE = 10_000; /** Basechain — required by pool.fc for nominator deposit/withdraw. */ const DEFAULT_NOMINATOR_WORKCHAIN = 0; +/** Highload orchestrator wc — same as nominators so sendBatch does not cross workchains (MC→0 breaks on singlehost). */ +const DEFAULT_HIGHLOAD_WORKCHAIN = 0; +/** Max outgoing messages per sendBatch (same wc). */ +const DEFAULT_HIGHLOAD_BATCH_CHUNK = 10; /** Attached TON per msg; must be >= on-chain min_nominator_stake + 1 TON pool fee (default min stake 10k → 10001). */ const DEFAULT_STAKE_TON = "10001"; const DEFAULT_POOL_INFO_DELAY_MS = 5000; @@ -145,7 +151,8 @@ async function run() { console.error( `Default stake: ${DEFAULT_STAKE_TON} TON (override argv or NOMINATOR_STAKE_TON). ` + `Optional: MASTER_WALLET_ID (default 42), NOMINATOR_SUBWALLET_BASE (default ${DEFAULT_SUBWALLET_BASE}), ` + - `NOMINATOR_WORKCHAIN (default ${DEFAULT_NOMINATOR_WORKCHAIN}), NOMINATOR_FUND_EXTRA_TON (default 0.15), ` + + `NOMINATOR_WORKCHAIN (default ${DEFAULT_NOMINATOR_WORKCHAIN}), HIGHLOAD_WORKCHAIN (default ${DEFAULT_HIGHLOAD_WORKCHAIN}), ` + + `HIGHLOAD_BATCH_CHUNK (default ${DEFAULT_HIGHLOAD_BATCH_CHUNK}), NOMINATOR_FUND_EXTRA_TON (default 0.15), ` + `POOL_INFO_DELAY_MS, TONCORE_POOL_INFO_EXTRA`, ); process.exit(1); @@ -173,6 +180,19 @@ async function run() { `NOMINATOR_WORKCHAIN must be 0 (basechain). Pool rejects nominator ops from masterchain (see pool.fc throw 61). Got ${nominatorWorkchain}`, ); } + const highloadWorkchain = Number.parseInt( + process.env.HIGHLOAD_WORKCHAIN ?? String(DEFAULT_HIGHLOAD_WORKCHAIN), + 10, + ); + const batchChunk = Number.parseInt(process.env.HIGHLOAD_BATCH_CHUNK ?? String(DEFAULT_HIGHLOAD_BATCH_CHUNK), 10); + if (highloadWorkchain !== nominatorWorkchain) { + throw new Error( + `HIGHLOAD_WORKCHAIN (${highloadWorkchain}) must match NOMINATOR_WORKCHAIN (${nominatorWorkchain}) so batch deploy stays on one workchain`, + ); + } + if (!Number.isFinite(batchChunk) || batchChunk < 1 || batchChunk > 40) { + throw new Error(`HIGHLOAD_BATCH_CHUNK must be 1..40, got ${batchChunk}`); + } const amountPer = toNano(amountTon); const masterKey = Buffer.from(process.env.MASTER_WALLET_KEY!, "hex"); @@ -219,10 +239,19 @@ async function run() { const highloadTopup = (deployAndFees + amountPer + toNano("1")) * BigInt(count); - const highloadWallet = new HighloadWalletV3(HighloadWalletV3.newSequence(), publicKey, HIGHLOAD_TIMEOUT_SEC, HighloadWalletV3.DEFAULT_SUBWALLET_ID, -1); + const highloadWallet = new HighloadWalletV3( + HighloadWalletV3.newSequence(), + publicKey, + HIGHLOAD_TIMEOUT_SEC, + HighloadWalletV3.DEFAULT_SUBWALLET_ID, + highloadWorkchain, + ); - console.log(`Master wallet: ${masterWallet.address.toString()} (walletId=${masterWalletId})`); - console.log(`Highload wallet (orchestrator, mc): ${highloadWallet.address.toString()} (subwalletId=${HighloadWalletV3.DEFAULT_SUBWALLET_ID}, DEFAULT_SUBWALLET_ID)`); + console.log(`Master wallet: ${masterWallet.address.toString()} (walletId=${masterWalletId}, wc=-1)`); + console.log( + `Highload wallet (orchestrator, wc=${highloadWorkchain}): ${highloadWallet.address.toString()} ` + + `(subwalletId=${HighloadWalletV3.DEFAULT_SUBWALLET_ID})`, + ); console.log( `Pool: ${poolAddr.toString()}, ${count} nominators × ${amountTon} TON, ` + `body op=0 action=${TONCORE_ACTION_NOMINATOR_DEPOSIT} ('d'), nominator wc=${nominatorWorkchain}, subwallet base=${subwalletBase}`, @@ -251,10 +280,10 @@ async function run() { }); } - const totalOutFromHighload = (deployAndFees + amountPer) * BigInt(count); - const valuePerBatch = totalOutFromHighload + toNano("1"); - /** Enough for `sendBatch` attach + headroom for MC deploy/import fees (not full nominal `highloadTopup`). */ - const highloadFundedMinBalance = valuePerBatch + toNano("5"); + const perWalletOut = deployAndFees + amountPer; + /** Enough for largest chunk sendBatch + headroom (master→highload cross-wc deploy uses separate topup). */ + const largestChunk = Math.min(count, batchChunk); + const highloadFundedMinBalance = perWalletOut * BigInt(largestChunk) + toNano("6"); const hlDeployed = await client.isContractDeployed(highloadWallet.address).catch(() => false); const hlBalance = await client.getBalance(highloadWallet.address); @@ -268,7 +297,8 @@ async function run() { messages: [ internal({ to: highloadWallet.address, - value: highloadTopup + toNano("1"), // 1 TON for fees + // Extra TON for masterchain→basechain deploy/import when highload wc=0. + value: highloadTopup + toNano("2"), bounce: false, init: hlDeployed ? undefined : highloadWallet.init, }), @@ -296,18 +326,31 @@ async function run() { } const highloadContract = client.open(highloadWallet); - const createdAt = Math.floor(Date.now() / 1000) - 10; - console.log(` Highload deploy+fund: one sendBatch with ${count} messages (single external)…`); - try { - await highloadContract.sendBatch(masterKey, { - messages: batchMessages, - valuePerBatch, - createdAt, - }); - } catch (e) { - throw new Error(e instanceof Error ? e.message : String(e)); + let createdAt = Math.floor(Date.now() / 1000) - 10; + const batchCount = Math.ceil(count / batchChunk); + console.log( + ` Highload deploy+fund: ${batchCount} sendBatch chunk(s), up to ${batchChunk} messages each (wc=${highloadWorkchain})…`, + ); + for (let offset = 0; offset < batchMessages.length; offset += batchChunk) { + const chunk = batchMessages.slice(offset, offset + batchChunk); + const chunkIndex = Math.floor(offset / batchChunk) + 1; + const valuePerBatch = perWalletOut * BigInt(chunk.length) + toNano("1"); + console.log(` sendBatch chunk ${chunkIndex}/${batchCount}: ${chunk.length} message(s), attach ${fromNano(valuePerBatch)} TON…`); + try { + await highloadContract.sendBatch(masterKey, { + messages: chunk, + valuePerBatch, + createdAt, + }); + } catch (e) { + throw new Error(e instanceof Error ? e.message : String(e)); + } + highloadWallet.sequence.next(); + createdAt = Math.floor(Date.now() / 1000) - 10; + if (offset + batchChunk < batchMessages.length) { + await new Promise((r) => setTimeout(r, 3000)); + } } - highloadWallet.sequence.next(); console.log( ` Next highload query id (local sequence after sendBatch; reuse same on-chain highload only with matching query id): ${highloadWallet.sequence.current()}`, ); @@ -341,7 +384,7 @@ async function run() { const lowHl = hlBal < toNano("50"); const hint = lowHl ? `Highload balance ${fromNano(hlBal)} TON is low — top up the master and re-run (intended highload topup this run: ${fromNano(highloadTopup)} TON).` - : `Highload still holds ${fromNano(hlBal)} TON — try lowering count if ton-http-api rejects large BOC (HTTP 500), or wait for the chain.`; + : `Highload still holds ${fromNano(hlBal)} TON — try lowering HIGHLOAD_BATCH_CHUNK or count, or wait for basechain to import blocks.`; throw new Error( `Only ${ready}/${count} nominator wallets became active after ${SCRIPT_CHAIN_POLL_MAX_MS} ms. ${hint} Stuck: ${stuck.join("; ")}`, ); From 7f09c0a0bfabb0fa351a0816a04099c7cc38b9ac Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Tue, 9 Jun 2026 15:49:34 +0300 Subject: [PATCH 25/30] refactor(adaptive): flatten AdaptiveStakeZero into AdaptiveStakeResult MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the intermediate AdaptiveStakeZero enum and move its variants (Defer, NoTopUpNeeded, InsufficientFree) directly into AdaptiveStakeResult. Update all call-sites in runner.rs accordingly, renaming adaptive_zero_to_skip → adaptive_result_to_skip. Co-authored-by: Cursor --- .../src/elections/adaptive_strategy.rs | 34 ++++++------------- .../service/src/elections/runner.rs | 17 +++++----- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/src/node-control/service/src/elections/adaptive_strategy.rs b/src/node-control/service/src/elections/adaptive_strategy.rs index 8727850a..d6834db5 100644 --- a/src/node-control/service/src/elections/adaptive_strategy.rs +++ b/src/node-control/service/src/elections/adaptive_strategy.rs @@ -22,9 +22,10 @@ pub(crate) enum AdaptiveDeferReason { WaitingForParticipants, } -/// Why AdaptiveSplit50 returns zero stake. +/// Outcome of [`calc_adaptive_stake`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum AdaptiveStakeZero { +pub(crate) enum AdaptiveStakeResult { + Stake(u64), /// Sleep or waiting-for-participants gate has not passed yet. Defer(AdaptiveDeferReason), /// Stake already meets min effective — no top-up this tick (not an error). @@ -33,13 +34,6 @@ pub(crate) enum AdaptiveStakeZero { InsufficientFree { required: u64, available: u64 }, } -/// Outcome of [`calc_adaptive_stake`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum AdaptiveStakeResult { - Stake(u64), - Zero(AdaptiveStakeZero), -} - /// Returns `None` when staking should proceed; otherwise the wait gate that blocks. pub(crate) fn adaptive_split50_status( node_id: &str, @@ -178,7 +172,7 @@ pub(crate) fn calc_adaptive_stake( nanotons_to_tons_f64(current_stake), nanotons_to_tons_f64(min_eff_stake) ); - return Ok(AdaptiveStakeResult::Zero(AdaptiveStakeZero::NoTopUpNeeded)); + return Ok(AdaptiveStakeResult::NoTopUpNeeded); } // Insufficient funds guard — if the pool doesn't have enough free @@ -194,10 +188,10 @@ pub(crate) fn calc_adaptive_stake( nanotons_to_tons_f64(required), nanotons_to_tons_f64(min_eff_stake), ); - return Ok(AdaptiveStakeResult::Zero(AdaptiveStakeZero::InsufficientFree { + return Ok(AdaptiveStakeResult::InsufficientFree { required, available: free_balance, - })); + }); } // Decide between staking half or min_eff_stake. @@ -221,10 +215,10 @@ pub(crate) fn calc_adaptive_stake( nanotons_to_tons_f64(stake), nanotons_to_tons_f64(free_balance), ); - return Ok(AdaptiveStakeResult::Zero(AdaptiveStakeZero::InsufficientFree { + return Ok(AdaptiveStakeResult::InsufficientFree { required: stake, available: free_balance, - })); + }); } Ok(AdaptiveStakeResult::Stake(stake)) } else { @@ -379,7 +373,7 @@ mod tests { ) .unwrap(); - assert_eq!(result, AdaptiveStakeResult::Zero(AdaptiveStakeZero::NoTopUpNeeded)); + assert_eq!(result, AdaptiveStakeResult::NoTopUpNeeded); } // ---- insufficient funds guard ---- @@ -409,10 +403,7 @@ mod tests { ) .unwrap(); - assert!(matches!( - result, - AdaptiveStakeResult::Zero(AdaptiveStakeZero::InsufficientFree { .. }) - )); + assert!(matches!(result, AdaptiveStakeResult::InsufficientFree { .. })); } // ---- cap to free_balance when half > free_balance ---- @@ -445,10 +436,7 @@ mod tests { ) .unwrap(); - assert!(matches!( - result, - AdaptiveStakeResult::Zero(AdaptiveStakeZero::InsufficientFree { .. }) - )); + assert!(matches!(result, AdaptiveStakeResult::InsufficientFree { .. })); } // ---- curr vs prev selection ---- diff --git a/src/node-control/service/src/elections/runner.rs b/src/node-control/service/src/elections/runner.rs index b6c112b6..421e73be 100644 --- a/src/node-control/service/src/elections/runner.rs +++ b/src/node-control/service/src/elections/runner.rs @@ -375,11 +375,11 @@ impl ElectionRunner { } } - fn adaptive_zero_to_skip( - zero: adaptive_strategy::AdaptiveStakeZero, + fn adaptive_result_to_skip( + result: adaptive_strategy::AdaptiveStakeResult, ) -> Option<(StakeSkipReason, Option, Option)> { - use adaptive_strategy::{AdaptiveDeferReason, AdaptiveStakeZero::*}; - match zero { + use adaptive_strategy::{AdaptiveDeferReason, AdaptiveStakeResult::*}; + match result { Defer(AdaptiveDeferReason::SleepPeriod) => { Some((StakeSkipReason::AdaptiveSleepingPeriod, None, None)) } @@ -390,6 +390,7 @@ impl ElectionRunner { InsufficientFree { required, available } => { Some((StakeSkipReason::InsufficientStakeFunds, Some(required), Some(available))) } + Stake(_) => unreachable!("caller must not pass Stake variant"), } } @@ -958,7 +959,7 @@ impl ElectionRunner { if stake == 0 { tracing::info!("node [{}] skipping elections this tick (stake=0)", node_id); - let skip = adaptive_zero.and_then(Self::adaptive_zero_to_skip); + let skip = adaptive_zero.and_then(Self::adaptive_result_to_skip); if let Some((reason, required, available)) = skip { elections_audit_stake_skipped( &audit, @@ -1418,7 +1419,7 @@ impl ElectionRunner { elections_stake: u64, configs: &ConfigParams<'_>, ctx: &StakeContext<'_>, - ) -> anyhow::Result<(u64, Option)> { + ) -> anyhow::Result<(u64, Option)> { let min_stake = configs.elections_info.min_stake; tracing::info!("node [{}] calc stake", node_id); let fee = ELECTOR_STAKE_FEE + NPOOL_COMPUTE_FEE; @@ -1488,7 +1489,7 @@ impl ElectionRunner { ctx.sleep_pct, ctx.waiting_pct, ) { - return Ok((0, Some(adaptive_strategy::AdaptiveStakeZero::Defer(defer)))); + return Ok((0, Some(adaptive_strategy::AdaptiveStakeResult::Defer(defer)))); } let current_stake = if node.stake_accepted { elections_stake } else { 0 }; let stakes: Vec<_> = configs @@ -1518,7 +1519,7 @@ impl ElectionRunner { )?; Ok(match outcome { adaptive_strategy::AdaptiveStakeResult::Stake(s) => (s, None), - adaptive_strategy::AdaptiveStakeResult::Zero(z) => (0, Some(z)), + other => (0, Some(other)), }) } other => Ok((other.calculate_stake(min_stake, total_balance)?, None)), From 930524250b09552ee4f37a8b56c9c0b7f6b9a794 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Tue, 9 Jun 2026 15:56:15 +0300 Subject: [PATCH 26/30] fix:fmt --- src/node-control/service/src/audit/event.rs | 5 +---- .../service/src/elections/adaptive_strategy.rs | 10 +++++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index 5bbf653d..6de401af 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -181,10 +181,7 @@ impl AuditEvent { actor, Self::node_target(node_id, election_id), AuditOutcome::Success, - AuditEventPayload::ElectionsStakeRecovered { - amount: amount.into(), - msg_hash, - }, + AuditEventPayload::ElectionsStakeRecovered { amount: amount.into(), msg_hash }, ) } diff --git a/src/node-control/service/src/elections/adaptive_strategy.rs b/src/node-control/service/src/elections/adaptive_strategy.rs index d6834db5..d410189f 100644 --- a/src/node-control/service/src/elections/adaptive_strategy.rs +++ b/src/node-control/service/src/elections/adaptive_strategy.rs @@ -31,7 +31,10 @@ pub(crate) enum AdaptiveStakeResult { /// Stake already meets min effective — no top-up this tick (not an error). NoTopUpNeeded, /// Free pool balance is below the required delta to min effective stake. - InsufficientFree { required: u64, available: u64 }, + InsufficientFree { + required: u64, + available: u64, + }, } /// Returns `None` when staking should proceed; otherwise the wait gate that blocks. @@ -188,10 +191,7 @@ pub(crate) fn calc_adaptive_stake( nanotons_to_tons_f64(required), nanotons_to_tons_f64(min_eff_stake), ); - return Ok(AdaptiveStakeResult::InsufficientFree { - required, - available: free_balance, - }); + return Ok(AdaptiveStakeResult::InsufficientFree { required, available: free_balance }); } // Decide between staking half or min_eff_stake. From be88956c43f6ef5afbd38aa25d16fc88301e0025 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Tue, 9 Jun 2026 16:10:03 +0300 Subject: [PATCH 27/30] feat(audit): project elections audit events into GET /v1/elections --- .../src/commands/nodectl/service_api_cmd.rs | 115 +---- .../service/src/audit/jsonl_writer.rs | 2 - src/node-control/service/src/audit/mod.rs | 5 + .../service/src/audit/projection.rs | 414 ++++++++++++++++++ .../service/src/http/http_server_task.rs | 281 +++++++++++- 5 files changed, 691 insertions(+), 126 deletions(-) create mode 100644 src/node-control/service/src/audit/projection.rs diff --git a/src/node-control/commands/src/commands/nodectl/service_api_cmd.rs b/src/node-control/commands/src/commands/nodectl/service_api_cmd.rs index 4c7cdee8..b6e045ed 100644 --- a/src/node-control/commands/src/commands/nodectl/service_api_cmd.rs +++ b/src/node-control/commands/src/commands/nodectl/service_api_cmd.rs @@ -661,19 +661,17 @@ fn print_elections_table(body: &str) -> anyhow::Result<()> { let Some(participants) = value.get("our_participants").and_then(serde_json::Value::as_array) else { println!("\n {}\n", "No participants in response".yellow()); - print_recent_events_table(&value); return Ok(()); }; if participants.is_empty() { println!("\n {}\n", "No controlled participants".yellow()); - print_recent_events_table(&value); return Ok(()); } println!("\n {} ({})\n", "Our Participants".cyan().bold(), participants.len()); println!( - " {} {} {} {} {} {} {} {} {}", + " {} {} {} {} {} {} {} {} {} {}", format!("{:<14}", "Node").cyan().bold(), format!("{:<13}", "Status").cyan().bold(), format!("{:<5}", "Pos").cyan().bold(), @@ -681,10 +679,11 @@ fn print_elections_table(body: &str) -> anyhow::Result<()> { format!("{:<15}", "Accepted TON").cyan().bold(), format!("{:<24}", "Submitted At").cyan().bold(), format!("{:<6}", "MaxF").cyan().bold(), + format!("{:<30}", "Last error").cyan().bold(), format!("{:<44}", "Pubkey").cyan().bold(), "ADNL".cyan().bold(), ); - println!(" {}", "-".repeat(148).dimmed()); + println!(" {}", "-".repeat(180).dimmed()); for p in participants { let node = binding_str(p, "node_id"); @@ -736,11 +735,14 @@ fn print_elections_table(body: &str) -> anyhow::Result<()> { .unwrap_or_else(|| "-".to_string()); let accepted_stake = binding_str(p, "accepted_stake"); + let last_error = binding_str(p, "last_error"); + let last_error_display = + if last_error == "-" { "-".to_string() } else { last_error.yellow().to_string() }; let pubkey = binding_str(p, "pubkey"); let adnl = binding_str(p, "adnl"); println!( - " {:<14} {} {:<5} {:<15} {:<15} {:<24} {:<6} {:<44} {}", + " {:<14} {} {:<5} {:<15} {:<15} {:<24} {:<6} {:<30} {:<44} {}", node, status, position, @@ -748,117 +750,16 @@ fn print_elections_table(body: &str) -> anyhow::Result<()> { display_tons_from_str(&accepted_stake), submitted_at, max_factor, + last_error_display, pubkey, adnl, ); } println!(); - print_recent_events_table(&value); - Ok(()) } -fn print_recent_events_table(body: &serde_json::Value) { - let Some(events) = body.get("recent_events").and_then(serde_json::Value::as_array) else { - return; - }; - if events.is_empty() { - return; - } - - println!(" {} ({})\n", "Recent Audit Events".cyan().bold(), events.len()); - println!( - " {} {} {} {} {}", - format!("{:<24}", "Time").cyan().bold(), - format!("{:<35}", "Event").cyan().bold(), - format!("{:<14}", "Node").cyan().bold(), - format!("{:<9}", "Outcome").cyan().bold(), - "Details".cyan().bold(), - ); - println!(" {}", "-".repeat(110).dimmed()); - - for ev in events { - let ts = ev.get("ts").and_then(serde_json::Value::as_str).unwrap_or("-"); - // Trim to HH:MM:SS for compactness; full date shown if it fits - let ts_display = if ts.len() >= 19 { &ts[..19] } else { ts }; - let ts_display = ts_display.replace('T', " "); - - let event_type = ev.get("event_type").and_then(serde_json::Value::as_str).unwrap_or("-"); - let short_event = event_type.trim_start_matches("elections."); - - let outcome_raw = ev.get("outcome").and_then(serde_json::Value::as_str).unwrap_or("-"); - let outcome = match outcome_raw { - "success" => format!("{:<9}", "ok").green().to_string(), - "failure" => format!("{:<9}", "fail").red().bold().to_string(), - "skipped" => format!("{:<9}", "skipped").yellow().to_string(), - other => format!("{:<9}", other), - }; - - let target = ev.get("target"); - let node_id = - target.and_then(|t| t.get("id")).and_then(serde_json::Value::as_str).unwrap_or("-"); - - let data = ev.get("data"); - let details = format_event_details(short_event, data); - - println!( - " {:<24} {:<35} {:<14} {} {}", - ts_display, short_event, node_id, outcome, details - ); - } - println!(); -} - -fn format_event_details(event_type: &str, data: Option<&serde_json::Value>) -> String { - let Some(data) = data else { return String::new() }; - - match event_type { - "stake_submitted" => { - let stake = data.get("stake").and_then(serde_json::Value::as_str).unwrap_or("-"); - let policy = data.get("policy").and_then(serde_json::Value::as_str).unwrap_or("-"); - format!("stake={} policy={}", display_tons_from_str(stake), policy) - } - "stake_accepted" => { - let stake = data.get("stake").and_then(serde_json::Value::as_str).unwrap_or("-"); - format!("stake={}", display_tons_from_str(stake)) - } - "stake_skipped" => { - let reason = data.get("reason").and_then(serde_json::Value::as_str).unwrap_or("-"); - if let (Some(req), Some(avail)) = ( - data.get("required").and_then(serde_json::Value::as_str), - data.get("available").and_then(serde_json::Value::as_str), - ) { - format!( - "reason={} req={} avail={}", - reason, - display_tons_from_str(req), - display_tons_from_str(avail) - ) - } else { - format!("reason={}", reason) - } - } - "stake_failed" | "stake_recover_failed" | "withdraw_failed" => { - let reason = data.get("reason").and_then(serde_json::Value::as_str).unwrap_or("-"); - format!("reason={}", reason) - } - "stake_recovered" => { - let amount = data.get("amount").and_then(serde_json::Value::as_str).unwrap_or("-"); - format!("amount={}", display_tons_from_str(amount)) - } - "withdraw_processed" => { - let hash = data.get("msg_hash").and_then(serde_json::Value::as_str).unwrap_or("-"); - format!("msg_hash={}", &hash[..hash.len().min(16)]) - } - "key_generated" => { - let pubkey = data.get("pubkey").and_then(serde_json::Value::as_str).unwrap_or("-"); - format!("pubkey={}…", &pubkey[..pubkey.len().min(12)]) - } - _ => String::new(), - } -} - fn binding_str(value: &serde_json::Value, key: &str) -> String { value .get(key) diff --git a/src/node-control/service/src/audit/jsonl_writer.rs b/src/node-control/service/src/audit/jsonl_writer.rs index a7f77072..a99e0583 100644 --- a/src/node-control/service/src/audit/jsonl_writer.rs +++ b/src/node-control/service/src/audit/jsonl_writer.rs @@ -15,8 +15,6 @@ use std::{ }, time::Duration, }; -use tokio::sync::{mpsc, oneshot}; - /// Schema version stamped into the per-file [`AuditFileHeader`]. const AUDIT_SCHEMA_VERSION: u16 = 1; diff --git a/src/node-control/service/src/audit/mod.rs b/src/node-control/service/src/audit/mod.rs index f038d9b6..e154a9ff 100644 --- a/src/node-control/service/src/audit/mod.rs +++ b/src/node-control/service/src/audit/mod.rs @@ -16,6 +16,7 @@ pub mod jsonl_log; pub mod jsonl_writer; pub mod log; pub mod participant; +pub mod projection; pub mod ring_buffer; pub use actor_builder::{AuditActorBuilder, client_ip_from_headers}; @@ -30,4 +31,8 @@ pub use in_memory::InMemoryAuditLog; pub use jsonl_log::AuditInitError; pub use log::{AuditLog, NoopAuditLog}; pub use participant::{AuditActor, AuditTarget}; +pub use projection::{ + ElectionsProjection, collect_recent_election_ids, merge_projection_into_participants, + project_elections, +}; pub use ring_buffer::AuditEventBuffer; diff --git a/src/node-control/service/src/audit/projection.rs b/src/node-control/service/src/audit/projection.rs new file mode 100644 index 00000000..e3599884 --- /dev/null +++ b/src/node-control/service/src/audit/projection.rs @@ -0,0 +1,414 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +use crate::audit::{ + AuditEvent, AuditEventPayload, AuditOutcome, AuditSource, StakeSkipReason, + participant::AuditTarget, +}; +use chrono::{DateTime, Utc}; +use common::{ + snapshot::{OurElectionParticipant, StakeSubmission}, + time_format, + ton_utils::max_stake_factor_raw_to_multiplier, +}; +use std::collections::BTreeMap; + +/// Projected stake submission from audit events. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectedStakeSubmission { + pub ts: DateTime, + pub node_id: String, + pub stake: String, + pub max_factor: u32, + pub policy: String, + pub submission_time: u64, +} + +/// Projected stake skip from audit events. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectedStakeSkip { + pub ts: DateTime, + pub node_id: String, + pub reason: StakeSkipReason, +} + +/// Projected withdraw outcome from audit events. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectedWithdraw { + pub ts: DateTime, + pub node_id: String, + pub outcome: AuditOutcome, + pub msg_hash: Option, + pub error: Option, +} + +/// Projected stake failure from audit events. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectedStakeFailure { + pub ts: DateTime, + pub node_id: String, + pub reason: String, +} + +/// Per-node elections audit data keyed by election id. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct NodeElectionProjection { + pub stake_submissions: Vec, + pub stake_skips: Vec, + pub withdraws: Vec, + pub stake_failures: Vec, +} + +/// Aggregated elections projection from the in-memory audit ring buffer. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ElectionsProjection { + /// `election_id` → `node_id` → per-node projection. + pub nodes: BTreeMap>, +} + +/// Collects up to `max` recent election ids: the live cycle first, then others seen in `events`. +pub fn collect_recent_election_ids( + current_election_id: Option, + events: &[AuditEvent], + max: usize, +) -> Vec { + let mut ids = Vec::new(); + let mut seen = std::collections::HashSet::new(); + + if let Some(id) = current_election_id { + seen.insert(id); + ids.push(id); + } + + for ev in events.iter().rev() { + let Some(election_id) = election_id_from_event(ev) else { continue }; + if seen.insert(election_id) { + ids.push(election_id); + if ids.len() >= max { + break; + } + } + } + + ids +} + +/// Builds an [`ElectionsProjection`] from audit events, keeping only `recent_election_ids`. +pub fn project_elections( + events: &[AuditEvent], + recent_election_ids: &[u64], +) -> ElectionsProjection { + let mut projection = ElectionsProjection::default(); + + for ev in events { + if ev.payload.source() != AuditSource::Elections { + continue; + } + let Some(election_id) = election_id_from_event(ev) else { continue }; + if !recent_election_ids.contains(&election_id) { + continue; + } + let node_id = node_id_from_event(ev).unwrap_or_default(); + let node = + projection.nodes.entry(election_id).or_default().entry(node_id.clone()).or_default(); + + match &ev.payload { + AuditEventPayload::ElectionsStakeSubmitted { + stake, + max_factor, + policy, + submission_time, + } => { + node.stake_submissions.push(ProjectedStakeSubmission { + ts: ev.ts, + node_id, + stake: stake.clone(), + max_factor: *max_factor, + policy: policy.clone(), + submission_time: *submission_time, + }); + } + AuditEventPayload::ElectionsStakeSkipped { reason, .. } => { + node.stake_skips.push(ProjectedStakeSkip { ts: ev.ts, node_id, reason: *reason }); + } + AuditEventPayload::ElectionsStakeFailed { reason } => { + node.stake_failures.push(ProjectedStakeFailure { + ts: ev.ts, + node_id, + reason: reason.clone(), + }); + } + AuditEventPayload::ElectionsWithdrawProcessed { msg_hash } => { + node.withdraws.push(ProjectedWithdraw { + ts: ev.ts, + node_id, + outcome: AuditOutcome::Success, + msg_hash: Some(msg_hash.clone()), + error: None, + }); + } + AuditEventPayload::ElectionsWithdrawFailed { reason } => { + node.withdraws.push(ProjectedWithdraw { + ts: ev.ts, + node_id, + outcome: AuditOutcome::Failure, + msg_hash: None, + error: Some(reason.clone()), + }); + } + _ => {} + } + } + + projection +} + +/// Merges projected audit data into live snapshot participants for `election_id`. +pub fn merge_projection_into_participants( + participants: &mut [OurElectionParticipant], + projection: &ElectionsProjection, + election_id: u64, +) { + let Some(by_node) = projection.nodes.get(&election_id) else { return }; + + for participant in participants.iter_mut() { + let Some(node_proj) = by_node.get(&participant.node_id) else { continue }; + + merge_stake_submissions(&mut participant.stake_submissions, &node_proj.stake_submissions); + + if let Some(err) = latest_error_message(node_proj) { + participant.last_error = Some(err); + } + } +} + +fn merge_stake_submissions( + existing: &mut Vec, + projected: &[ProjectedStakeSubmission], +) { + for sub in projected { + let converted = projected_to_stake_submission(sub); + let duplicate = existing + .iter() + .any(|s| s.submission_time == converted.submission_time && s.stake == converted.stake); + if !duplicate { + existing.push(converted); + } + } + existing.sort_by_key(|s| s.submission_time); +} + +fn projected_to_stake_submission(sub: &ProjectedStakeSubmission) -> StakeSubmission { + StakeSubmission { + stake: sub.stake.clone(), + max_factor: max_stake_factor_raw_to_multiplier(sub.max_factor), + submission_time: sub.submission_time, + submission_time_utc: time_format::format_ts(sub.submission_time), + } +} + +fn latest_error_message(node_proj: &NodeElectionProjection) -> Option { + let mut candidates: Vec<(DateTime, String)> = Vec::new(); + + for skip in &node_proj.stake_skips { + candidates.push((skip.ts, format!("stake skipped: {}", format_skip_reason(skip.reason)))); + } + for failure in &node_proj.stake_failures { + candidates.push((failure.ts, format!("stake failed: {}", failure.reason))); + } + for withdraw in &node_proj.withdraws { + if withdraw.outcome == AuditOutcome::Failure + && let Some(error) = &withdraw.error + { + candidates.push((withdraw.ts, format!("withdraw failed: {error}"))); + } + } + + candidates.sort_by_key(|(ts, _)| *ts); + candidates.last().map(|(_, msg)| msg.clone()) +} + +fn format_skip_reason(reason: StakeSkipReason) -> &'static str { + match reason { + StakeSkipReason::LowWalletBalance => "low_wallet_balance", + StakeSkipReason::WithdrawRequestsPending => "withdraw_requests_pending", + StakeSkipReason::PoolNotReady => "pool_not_ready", + StakeSkipReason::AdaptiveSleepingPeriod => "adaptive_sleeping_period", + StakeSkipReason::AdaptiveWaitingPeriod => "adaptive_waiting_period", + StakeSkipReason::ElectionsDisabled => "elections_disabled", + StakeSkipReason::RecoverPending => "recover_pending", + StakeSkipReason::InsufficientStakeFunds => "insufficient_stake_funds", + } +} + +pub(crate) fn election_id_from_event(ev: &AuditEvent) -> Option { + match &ev.target { + AuditTarget::Node { election_id: Some(id), .. } => Some(*id), + AuditTarget::Elections { election_id } => Some(*election_id), + _ => None, + } +} + +pub(crate) fn node_id_from_event(ev: &AuditEvent) -> Option { + match &ev.target { + AuditTarget::Node { id, .. } => Some(id.clone()), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::audit::{AuditActor, AuditEvent}; + + const ELECTION_ID: u64 = 1_779_265_552; + const NODE_ID: &str = "node-1"; + + fn elections_actor() -> AuditActor { + AuditActor::service("elections-task") + } + + #[test] + fn projection_groups_events_by_election_id() { + const OTHER_ELECTION: u64 = ELECTION_ID + 100; + const NODE_B: &str = "node-2"; + + let events = vec![ + AuditEvent::elections_stake_submitted( + elections_actor(), + NODE_ID, + ELECTION_ID, + crate::audit::ElectionsStakeSubmittedParams { + stake: "100000000000".into(), + max_factor: 196_608, + policy: "split50".into(), + submission_time: 1_700_000_000, + }, + ), + AuditEvent::elections_stake_skipped( + elections_actor(), + NODE_B, + OTHER_ELECTION, + StakeSkipReason::ElectionsDisabled, + None, + None, + ), + ]; + + let projection = project_elections(&events, &[ELECTION_ID, OTHER_ELECTION]); + + let current = projection.nodes.get(&ELECTION_ID).unwrap().get(NODE_ID).unwrap(); + assert_eq!(current.stake_submissions.len(), 1); + assert!(current.stake_skips.is_empty()); + + let other = projection.nodes.get(&OTHER_ELECTION).unwrap().get(NODE_B).unwrap(); + assert!(other.stake_submissions.is_empty()); + assert_eq!(other.stake_skips.len(), 1); + assert_eq!(other.stake_skips[0].reason, StakeSkipReason::ElectionsDisabled); + } + + #[test] + fn projection_ignores_non_elections_source() { + let events = vec![ + AuditEvent::rest_api_auth_login_success( + AuditActor::user("alice", Some("admin".into()), None), + "alice", + ), + AuditEvent::elections_stake_failed(elections_actor(), NODE_ID, ELECTION_ID, "boom"), + ]; + + let projection = project_elections(&events, &[ELECTION_ID]); + + assert_eq!(projection.nodes.len(), 1); + let node = projection.nodes.get(&ELECTION_ID).unwrap().get(NODE_ID).unwrap(); + assert_eq!(node.stake_failures.len(), 1); + assert_eq!(node.stake_failures[0].reason, "boom"); + } + + #[test] + fn projection_only_includes_recent_election_ids() { + let events = vec![ + AuditEvent::elections_stake_failed(elections_actor(), NODE_ID, ELECTION_ID, "current"), + AuditEvent::elections_stake_failed( + elections_actor(), + NODE_ID, + ELECTION_ID + 1, + "excluded", + ), + ]; + + let projection = project_elections(&events, &[ELECTION_ID]); + + assert_eq!(projection.nodes.len(), 1); + let node = projection.nodes.get(&ELECTION_ID).unwrap().get(NODE_ID).unwrap(); + assert_eq!(node.stake_failures[0].reason, "current"); + } + + #[test] + fn merge_projection_enriches_participants_without_duplicates() { + let events = vec![ + AuditEvent::elections_stake_submitted( + elections_actor(), + NODE_ID, + ELECTION_ID, + crate::audit::ElectionsStakeSubmittedParams { + stake: "200000000000".into(), + max_factor: 196_608, + policy: "all".into(), + submission_time: 1_700_000_100, + }, + ), + AuditEvent::elections_stake_skipped( + elections_actor(), + NODE_ID, + ELECTION_ID, + StakeSkipReason::InsufficientStakeFunds, + Some("100".into()), + Some("50".into()), + ), + ]; + let projection = project_elections(&events, &[ELECTION_ID]); + + let mut participants = vec![OurElectionParticipant { + node_id: NODE_ID.to_string(), + stake_submissions: vec![StakeSubmission { + stake: "200000000000".into(), + max_factor: 3.0, + submission_time: 1_700_000_100, + submission_time_utc: time_format::format_ts(1_700_000_100), + }], + ..Default::default() + }]; + + merge_projection_into_participants(&mut participants, &projection, ELECTION_ID); + + assert_eq!(participants[0].stake_submissions.len(), 1); + assert_eq!( + participants[0].last_error.as_deref(), + Some("stake skipped: insufficient_stake_funds") + ); + } + + #[test] + fn merge_projection_is_noop_when_ring_projection_empty() { + let mut participants = vec![OurElectionParticipant { + node_id: NODE_ID.to_string(), + last_error: Some("existing".into()), + ..Default::default() + }]; + + merge_projection_into_participants( + &mut participants, + &ElectionsProjection::default(), + ELECTION_ID, + ); + + assert_eq!(participants[0].last_error.as_deref(), Some("existing")); + assert!(participants[0].stake_submissions.is_empty()); + } +} diff --git a/src/node-control/service/src/http/http_server_task.rs b/src/node-control/service/src/http/http_server_task.rs index 3dde014f..bf39408b 100644 --- a/src/node-control/service/src/http/http_server_task.rs +++ b/src/node-control/service/src/http/http_server_task.rs @@ -351,11 +351,6 @@ pub struct ElectionsResponse { pub result: Option, pub next_elections: Option, pub our_participants: Vec, - /// Most recent elections audit events, newest first. Populated from the - /// in-memory ring buffer; empty when audit is disabled. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - #[schema(value_type = Vec)] - pub recent_events: Vec, } #[derive(Clone, Default, serde::Deserialize)] @@ -496,26 +491,32 @@ pub async fn v1_elections_handler( axum::extract::State(state): axum::extract::State, axum::extract::Query(query): axum::extract::Query, ) -> axum::Json { - use crate::audit::AuditSource; + use crate::audit::{ + AuditSource, collect_recent_election_ids, merge_projection_into_participants, + project_elections, + }; let include_participants = query.include_participants.unwrap_or(false); let view = state.store.get_elections_view(include_participants); + let current_election_id = view.elections.as_ref().map(|e| e.election_id); - let recent_events: Vec = state - .audit_ring - .filter_collect(|e| e.payload.source() == AuditSource::Elections) - .into_iter() - .rev() - .filter_map(|e| serde_json::to_value(e).ok()) - .collect(); + // Single snapshot of the ring; `project_elections` filters by `recent_ids` internally. + let elections_events = + state.audit_ring.filter_collect(|e| e.payload.source() == AuditSource::Elections); + let recent_ids = collect_recent_election_ids(current_election_id, &elections_events, 3); + let projection = project_elections(&elections_events, &recent_ids); + + let mut our_participants = view.our_participants; + if let Some(election_id) = current_election_id { + merge_projection_into_participants(&mut our_participants, &projection, election_id); + } axum::Json(ElectionsResponse { ok: true, result: view.elections, status: view.status, next_elections: view.next_elections, - our_participants: view.our_participants, - recent_events, + our_participants, }) } @@ -1147,6 +1148,38 @@ mod tests { store: Arc, runtime_cfg: Arc, elections_task: Arc, + ) -> AppState { + test_state_with_ring( + store, + runtime_cfg, + elections_task, + crate::audit::AuditEventBuffer::new(0), + ) + .await + } + + async fn test_state_with_ring( + store: Arc, + runtime_cfg: Arc, + elections_task: Arc, + audit_ring: Arc, + ) -> AppState { + test_state_with_audit( + store, + runtime_cfg, + elections_task, + Arc::new(NoopAuditLog), + audit_ring, + ) + .await + } + + async fn test_state_with_audit( + store: Arc, + runtime_cfg: Arc, + elections_task: Arc, + audit: Arc, + audit_ring: Arc, ) -> AppState { let user_store = Arc::new(UserStore::new(runtime_cfg.clone() as Arc)); AppState { @@ -1157,12 +1190,34 @@ mod tests { user_store, login_rate_limiter: Arc::new(tokio::sync::Mutex::new(LoginRateLimiter::default())), config_changed: Arc::new(tokio::sync::Notify::new()), - audit: Arc::new(NoopAuditLog), + audit, actor_builder: Arc::new(AuditActorBuilder::new(runtime_cfg)), - audit_ring: crate::audit::AuditEventBuffer::new(0), + audit_ring, } } + const PROJECTION_ELECTION_ID: u64 = 1_779_265_552; + + fn elections_store_with_participant(participant: OurElectionParticipant) -> Arc { + let store = Arc::new(SnapshotStore::new()); + store.update_with(|s| { + s.elections_status = ElectionsStatus::Active; + s.elections = Some(ElectionsSnapshot { + election_id: PROJECTION_ELECTION_ID, + ..Default::default() + }); + s.our_participants.push(participant); + }); + store + } + + async fn call_elections_handler(state: AppState) -> serde_json::Value { + let app = routes(false, state); + let resp = app.oneshot(get_request("/v1/elections")).await.unwrap(); + assert_eq!(resp.status(), 200); + body_json(resp).await + } + fn test_app_config(policy: StakePolicy) -> Arc { test_app_config_with_bindings(policy, HashMap::new()) } @@ -1760,6 +1815,198 @@ mod tests { assert_eq!(participants[0]["position"], 5); } + #[tokio::test] + async fn handler_returns_unchanged_response_when_ring_empty() { + let store = elections_store_with_participant(OurElectionParticipant { + node_id: "node-1".to_string(), + stake_accepted: true, + stake_submissions: vec![StakeSubmission { + stake: "100".to_string(), + max_factor: 3.0, + submission_time: 12345, + submission_time_utc: "2024-01-01T00:00:00Z".to_string(), + }], + accepted_stake: Some("100".to_string()), + last_error: Some("snapshot error".to_string()), + ..Default::default() + }); + let runtime_cfg = + Arc::new(RuntimeConfigStore::from_app_config(test_app_config(StakePolicy::Minimum))); + let state = test_state_with_ring( + store, + runtime_cfg, + test_elections_task(), + crate::audit::AuditEventBuffer::new(10), + ) + .await; + + let v = call_elections_handler(state).await; + let participant = &v["our_participants"][0]; + assert_eq!(participant["node_id"], "node-1"); + assert_eq!(participant["stake_accepted"], true); + assert_eq!(participant["accepted_stake"], "100"); + assert_eq!(participant["last_error"], "snapshot error"); + assert_eq!(participant["stake_submissions"].as_array().unwrap().len(), 1); + assert!(v.get("recent_events").is_none()); + } + + #[tokio::test] + async fn handler_includes_stake_submissions_from_audit() { + use crate::audit::{AuditEvent, AuditEventBuffer}; + + let store = elections_store_with_participant(OurElectionParticipant { + node_id: "node-1".to_string(), + ..Default::default() + }); + let audit_ring = AuditEventBuffer::new(50); + audit_ring.push(AuditEvent::elections_stake_submitted( + crate::audit::AuditActor::service("elections-task"), + "node-1", + PROJECTION_ELECTION_ID, + crate::audit::ElectionsStakeSubmittedParams { + stake: "300000000000".into(), + max_factor: 196_608, + policy: "split50".into(), + submission_time: 1_700_000_000, + }, + )); + + let runtime_cfg = + Arc::new(RuntimeConfigStore::from_app_config(test_app_config(StakePolicy::Minimum))); + let state = + test_state_with_ring(store, runtime_cfg, test_elections_task(), audit_ring).await; + + let v = call_elections_handler(state).await; + let submissions = v["our_participants"][0]["stake_submissions"].as_array().unwrap(); + assert_eq!(submissions.len(), 1); + assert_eq!(submissions[0]["stake"], "300000000000"); + assert_eq!(submissions[0]["max_factor"], 3.0); + assert_eq!(submissions[0]["submission_time"], 1_700_000_000); + assert!(v["our_participants"][0].get("last_error").is_none()); + } + + #[tokio::test] + async fn handler_marks_last_error_from_stake_skipped() { + use crate::audit::{AuditEvent, AuditEventBuffer, StakeSkipReason}; + + let store = elections_store_with_participant(OurElectionParticipant { + node_id: "node-1".to_string(), + ..Default::default() + }); + let audit_ring = AuditEventBuffer::new(50); + audit_ring.push(AuditEvent::elections_stake_skipped( + crate::audit::AuditActor::service("elections-task"), + "node-1", + PROJECTION_ELECTION_ID, + StakeSkipReason::AdaptiveSleepingPeriod, + None, + None, + )); + + let runtime_cfg = + Arc::new(RuntimeConfigStore::from_app_config(test_app_config(StakePolicy::Minimum))); + let state = + test_state_with_ring(store, runtime_cfg, test_elections_task(), audit_ring).await; + + let v = call_elections_handler(state).await; + assert_eq!( + v["our_participants"][0]["last_error"].as_str(), + Some("stake skipped: adaptive_sleeping_period") + ); + } + + #[tokio::test] + async fn handler_merges_withdraw_outcomes() { + use crate::audit::{AuditEvent, AuditEventBuffer}; + + let store = elections_store_with_participant(OurElectionParticipant { + node_id: "node-1".to_string(), + ..Default::default() + }); + let audit_ring = AuditEventBuffer::new(50); + audit_ring.push(AuditEvent::elections_withdraw_processed( + crate::audit::AuditActor::service("elections-task"), + "node-1", + PROJECTION_ELECTION_ID, + "abc123", + )); + audit_ring.push(AuditEvent::elections_withdraw_failed( + crate::audit::AuditActor::service("elections-task"), + "node-1", + PROJECTION_ELECTION_ID, + "send failed", + )); + + let runtime_cfg = + Arc::new(RuntimeConfigStore::from_app_config(test_app_config(StakePolicy::Minimum))); + let state = + test_state_with_ring(store, runtime_cfg, test_elections_task(), audit_ring).await; + + let v = call_elections_handler(state).await; + assert_eq!( + v["our_participants"][0]["last_error"].as_str(), + Some("withdraw failed: send failed") + ); + } + + #[tokio::test] + async fn handler_projects_events_recorded_through_jsonl_audit_log() { + use crate::audit::{ + AuditEvent, AuditLogConfig, StakeSkipReason, jsonl_log::JsonlAuditLog, log::AuditLog, + }; + use tempfile::tempdir; + + let store = elections_store_with_participant(OurElectionParticipant { + node_id: "node-1".to_string(), + ..Default::default() + }); + + let dir = tempdir().unwrap(); + let cfg = AuditLogConfig { + path: dir.path().join("audit.jsonl"), + ring_buffer_capacity: 50, + batch_interval_ms: 60_000, + ..AuditLogConfig::default() + }; + let log = JsonlAuditLog::start(cfg).await.unwrap(); + let audit_ring = log.ring(); + + log.record(AuditEvent::elections_stake_submitted( + crate::audit::AuditActor::service("elections-task"), + "node-1", + PROJECTION_ELECTION_ID, + crate::audit::ElectionsStakeSubmittedParams { + stake: "500000000000".into(), + max_factor: 196_608, + policy: "all".into(), + submission_time: 1_700_000_500, + }, + )) + .await; + log.record(AuditEvent::elections_stake_skipped( + crate::audit::AuditActor::service("elections-task"), + "node-1", + PROJECTION_ELECTION_ID, + StakeSkipReason::PoolNotReady, + None, + None, + )) + .await; + + let runtime_cfg = + Arc::new(RuntimeConfigStore::from_app_config(test_app_config(StakePolicy::Minimum))); + let state = + test_state_with_audit(store, runtime_cfg, test_elections_task(), log, audit_ring).await; + + let v = call_elections_handler(state).await; + let participant = &v["our_participants"][0]; + let submissions = participant["stake_submissions"].as_array().unwrap(); + assert_eq!(submissions.len(), 1); + assert_eq!(submissions[0]["stake"], "500000000000"); + assert_eq!(participant["last_error"].as_str(), Some("stake skipped: pool_not_ready")); + assert!(v.get("recent_events").is_none()); + } + #[tokio::test] async fn validators_returns_empty_snapshot() { let store = Arc::new(SnapshotStore::new()); From 4805e28d554e39f9236b819c50bbddf32c650e47 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Thu, 11 Jun 2026 19:27:01 +0300 Subject: [PATCH 28/30] fix(audit): address Copilot review comments on ring buffer and projection - Atomic dedup: replace two-step contains+push with push_unless_dedup_duplicate that holds a single write lock, eliminating the TOCTOU race on concurrent record() calls for the same elections.stake_skipped key. - Zero-copy dedup: replace dedup_key() -> Option with dedup_identity() -> Option>, removing per-event String allocations on the hot path. - Contract fix in collect_recent_election_ids: early-return when max == 0 and after inserting current_election_id when ids.len() >= max. Co-authored-by: Cursor --- src/node-control/service/src/audit/event.rs | 28 ++++-- .../service/src/audit/jsonl_log.rs | 13 +-- .../service/src/audit/projection.rs | 29 +++++++ .../service/src/audit/ring_buffer.rs | 85 ++++++++++++++++--- 4 files changed, 127 insertions(+), 28 deletions(-) diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index 6de401af..6d42494a 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -73,21 +73,31 @@ pub struct ElectionsStakeSubmittedParams { pub submission_time: u64, } +/// Zero-copy dedup identity for [`AuditEvent::dedup_identity`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct AuditDedupIdentity<'a> { + pub node_id: &'a str, + pub election_id: u64, + pub reason: StakeSkipReason, +} + impl AuditEvent { - /// Returns a stable deduplication key for events that should appear at most once - /// per election in the audit log (e.g. `elections.stake_skipped` with a persistent - /// reason like `ElectionsDisabled`). + /// Non-allocating dedup identity for events that should appear at most once per + /// election in the ring buffer (e.g. `elections.stake_skipped`). /// /// Returns `None` for events that are always recorded without deduplication. - /// The key encodes `(event_category, node_id, election_id, reason)` so that - /// the same skip reason for the same node in the same election is written only once. - pub fn dedup_key(&self) -> Option { + pub(crate) fn dedup_identity(&self) -> Option> { if let AuditEventPayload::ElectionsStakeSkipped { reason, .. } = &self.payload - && let AuditTarget::Node { id: node_id, election_id: Some(election_id) } = &self.target + && let AuditTarget::Node { id, election_id: Some(election_id) } = &self.target { - return Some(format!("stake_skipped:{node_id}:{election_id}:{reason:?}")); + Some(AuditDedupIdentity { + node_id: id.as_str(), + election_id: *election_id, + reason: *reason, + }) + } else { + None } - None } /// Internal constructor that stamps `id`/`ts`. Crate-private so call sites diff --git a/src/node-control/service/src/audit/jsonl_log.rs b/src/node-control/service/src/audit/jsonl_log.rs index 71ab9d26..0865a874 100644 --- a/src/node-control/service/src/audit/jsonl_log.rs +++ b/src/node-control/service/src/audit/jsonl_log.rs @@ -158,17 +158,12 @@ impl AuditLog for JsonlAuditLog { } async fn record(&self, event: AuditEvent) { - // Deduplication: drop events whose key already exists in the ring. - // Prevents e.g. repeated elections.stake_skipped for the same (node, election, reason). - if let Some(key) = event.dedup_key() - && self.ring.contains_dedup_key(&key) - { - return; - } - // Push into ring first: readers see the event immediately and even // queue-dropped events remain accessible on the REST read-path. - self.ring.push(event.clone()); + // Dedup check runs atomically inside the ring write lock. + if !self.ring.push_unless_dedup_duplicate(event.clone()) { + return; + } let event_id = event.id; let source = event.payload.source(); diff --git a/src/node-control/service/src/audit/projection.rs b/src/node-control/service/src/audit/projection.rs index e3599884..7d7d48f2 100644 --- a/src/node-control/service/src/audit/projection.rs +++ b/src/node-control/service/src/audit/projection.rs @@ -77,12 +77,19 @@ pub fn collect_recent_election_ids( events: &[AuditEvent], max: usize, ) -> Vec { + if max == 0 { + return Vec::new(); + } + let mut ids = Vec::new(); let mut seen = std::collections::HashSet::new(); if let Some(id) = current_election_id { seen.insert(id); ids.push(id); + if ids.len() >= max { + return ids; + } } for ev in events.iter().rev() { @@ -273,6 +280,28 @@ mod tests { AuditActor::service("elections-task") } + #[test] + fn collect_recent_election_ids_returns_empty_when_max_is_zero() { + let events = vec![AuditEvent::elections_stake_failed( + elections_actor(), + NODE_ID, + ELECTION_ID, + "err", + )]; + assert!(collect_recent_election_ids(Some(ELECTION_ID), &events, 0).is_empty()); + assert!(collect_recent_election_ids(None, &events, 0).is_empty()); + } + + #[test] + fn collect_recent_election_ids_respects_max_after_current_id() { + let events = vec![ + AuditEvent::elections_stake_failed(elections_actor(), NODE_ID, ELECTION_ID + 1, "a"), + AuditEvent::elections_stake_failed(elections_actor(), NODE_ID, ELECTION_ID + 2, "b"), + ]; + let ids = collect_recent_election_ids(Some(ELECTION_ID), &events, 1); + assert_eq!(ids, vec![ELECTION_ID]); + } + #[test] fn projection_groups_events_by_election_id() { const OTHER_ELECTION: u64 = ELECTION_ID + 100; diff --git a/src/node-control/service/src/audit/ring_buffer.rs b/src/node-control/service/src/audit/ring_buffer.rs index 68e87be4..b2fd79eb 100644 --- a/src/node-control/service/src/audit/ring_buffer.rs +++ b/src/node-control/service/src/audit/ring_buffer.rs @@ -37,7 +37,28 @@ impl AuditEventBuffer { /// Appends `event`, evicting the oldest entry when the buffer is at capacity. pub fn push(&self, event: AuditEvent) { let mut buf = self.inner.write(); - if buf.len() == self.capacity { + Self::push_locked(&mut buf, self.capacity, event); + } + + /// Appends `event` unless the buffer already holds an entry with the same + /// [`AuditEvent::dedup_identity`]. The contains-check and push run under one write + /// lock so concurrent `record()` calls cannot duplicate dedup-keyed events. + /// + /// Returns `true` if the event was appended, `false` if it was suppressed. + /// Events without a dedup identity are always appended. + pub fn push_unless_dedup_duplicate(&self, event: AuditEvent) -> bool { + let mut buf = self.inner.write(); + if let Some(key) = event.dedup_identity() + && buf.iter().any(|e| e.dedup_identity() == Some(key)) + { + return false; + } + Self::push_locked(&mut buf, self.capacity, event); + true + } + + fn push_locked(buf: &mut VecDeque, capacity: usize, event: AuditEvent) { + if buf.len() == capacity { buf.pop_front(); } buf.push_back(event); @@ -67,20 +88,13 @@ impl AuditEventBuffer { pub fn capacity(&self) -> usize { self.capacity } - - /// Returns `true` if the buffer already contains an event with the given - /// deduplication key. Used by [`crate::audit::jsonl_log::JsonlAuditLog`] to - /// suppress repeated identical `elections.stake_skipped` events within one election. - pub fn contains_dedup_key(&self, key: &str) -> bool { - self.inner.read().iter().any(|e| e.dedup_key().as_deref() == Some(key)) - } } #[cfg(test)] mod tests { use super::*; - use crate::audit::{AuditEvent, AuditSource}; - use std::sync::Arc; + use crate::audit::{AuditEvent, AuditSource, StakeSkipReason}; + use std::sync::{Arc, Barrier}; fn ev(tag: &str) -> AuditEvent { AuditEvent::system_service_started(tag) @@ -194,6 +208,57 @@ mod tests { assert_eq!(buf.len(), 50); } + fn stake_skipped(node_id: &str) -> AuditEvent { + AuditEvent::elections_stake_skipped( + crate::audit::AuditActor::service("elections-task"), + node_id, + 1_779_265_552, + StakeSkipReason::ElectionsDisabled, + None, + None, + ) + } + + #[test] + fn push_unless_dedup_duplicate_allows_first_then_suppresses() { + let buf = AuditEventBuffer::new(10); + let first = stake_skipped("node-1"); + let second = stake_skipped("node-1"); + + assert!(buf.push_unless_dedup_duplicate(first)); + assert!(!buf.push_unless_dedup_duplicate(second)); + assert_eq!(buf.len(), 1); + } + + #[test] + fn push_unless_dedup_duplicate_always_appends_without_dedup_key() { + let buf = AuditEventBuffer::new(10); + assert!(buf.push_unless_dedup_duplicate(ev("a"))); + assert!(buf.push_unless_dedup_duplicate(ev("b"))); + assert_eq!(buf.len(), 2); + } + + #[test] + fn push_unless_dedup_duplicate_is_atomic_under_concurrency() { + let buf = AuditEventBuffer::new(10); + let barrier = Arc::new(Barrier::new(8)); + let mut handles = vec![]; + + for _ in 0..8 { + let b = buf.clone(); + let gate = barrier.clone(); + handles.push(std::thread::spawn(move || { + gate.wait(); + b.push_unless_dedup_duplicate(stake_skipped("node-1")); + })); + } + + for h in handles { + h.join().expect("thread panicked"); + } + assert_eq!(buf.len(), 1, "only one concurrent stake_skipped must be retained"); + } + #[test] fn zero_capacity_buffer_silently_drops() { // capacity=0 is normalised to 1 internally; no crash, snapshot is non-empty after push From 7cd7d3236ad3c71b6dfcc5da21055b46fa4e6143 Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Thu, 11 Jun 2026 20:32:16 +0300 Subject: [PATCH 29/30] docs(audit): add operator-facing audit-log.md and README section --- src/node-control/README.md | 10 ++ src/node-control/docs/audit-log.md | 175 +++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 src/node-control/docs/audit-log.md diff --git a/src/node-control/README.md b/src/node-control/README.md index 3983af96..99784152 100644 --- a/src/node-control/README.md +++ b/src/node-control/README.md @@ -23,6 +23,7 @@ - [Voting Commands](#voting-commands) - [REST API Endpoints](#rest-api-endpoints) - [Configuration](#configuration) +- [Audit log](#audit-log) - [Config Structure](#config-structure) - [Section Descriptions](#section-descriptions) - [Default Config Example](#default-config-example) @@ -2521,9 +2522,18 @@ curl -X POST http://127.0.0.1:8080/v1/task/elections \ --- +## Audit log + +nodectl writes a structured audit log of domain events (elections, config +mutations, auth) to `./logs/audit.jsonl`. See [docs/audit-log.md](docs/audit-log.md) +for configuration, retention, PII handling, and log analysis. + +--- + ## Related Setup Guides - [Hashicorp Vault Dedicated Setup](./docs/hcp-vault-setup.md) - [Node Control Service Setup](./docs/nodectl-setup.md) - [Contracts automation (auto-deploy / auto-topup)](./docs/contracts-automation.md) — `automation` config, REST and CLI - [Security Guide](./docs/nodectl-security.md) — roles, token lifecycle, rate limiting, monitoring +- [Audit Log](./docs/audit-log.md) — configuration, durability, PII, log analysis diff --git a/src/node-control/docs/audit-log.md b/src/node-control/docs/audit-log.md new file mode 100644 index 00000000..675fd1a0 --- /dev/null +++ b/src/node-control/docs/audit-log.md @@ -0,0 +1,175 @@ +# Audit Log + +## What it is + +nodectl writes a structured, append-only log of domain events — elections, config +mutations, authentication, vault operations — to a newline-delimited JSON file +(`audit.jsonl`). + +The audit log is **separate from the `tracing` service log** (stderr / journald). +Use the table below to decide where to look. + +| Use case | Where | +|---|---| +| Debugging service internals, stack traces | tracing logs (`RUST_LOG`) | +| HTTP access / request logs | *(not implemented; would be tracing spans)* | +| Metrics / counters | Prometheus *(future)* | +| Domain events: who did what and when | **audit log** | + +## Out of scope + +- Per-RPC / per-request logging +- Metrics / dashboards +- Debug noise (heartbeats, cache refreshes, routine polls) +- High-frequency sources (> ~10 events/sec) +- Tamper-evidence (hash chain, signed events) — see RFC 9162 for future work + +## Event types + +Events are grouped by source: + +| Prefix | Events | +|---|---| +| `elections.*` | Key generated, stake submitted/accepted/skipped/failed/recovered, withdraw processed/failed | +| `rest_api.*` | Config updated, auth login succeeded/rejected, token rejected | +| `vault.*` | Key created / removed *(producers not wired yet)* | +| `rewards.*` | Distribution started/completed/failed, recipient skipped *(producers not wired yet)* | +| `system.*` | Service started/stopped, audit events dropped | + +Each event contains: + +- `id` — UUID v7 (sortable by creation time) +- `ts` — RFC3339 timestamp with millisecond precision (`2026-05-22T12:10:30.123Z`) +- `outcome` — `success`, `failure`, or `skipped` +- `event_type` — dotted string (e.g. `elections.stake_submitted`) +- `data` — event-specific payload (omitted when `include_payload = false`) +- `actor` — who triggered the action (`service` task or `user` identity) +- `target` — what the action was applied to (node, config, vault key, …) + +## File layout + +``` +./logs/audit.jsonl ← current file (line 0 is a file header, not an event) +./logs/audit.jsonl.1 ← most-recent rotation +./logs/audit.jsonl.2 +… +./logs/audit.jsonl.9 +``` + +The first line of every file is a **header** (no `event_type` field): + +```json +{"schema_version":1,"service":"nodectl","service_version":"0.7.0","host":"validator-1","started_at":"2026-05-22T12:00:00.000Z"} +``` + +Defaults: 100 MiB per file, 10 files → ~1 GiB total history. + +## Configuration + +All fields live under the `audit_log` key in the nodectl config file. +None of them require a service restart — the values are read at startup. + +| Field | Default | Description | +|---|---|---| +| `enabled` | `true` | Set to `false` to disable the audit log entirely | +| `path` | `./logs/audit.jsonl` | Path to the active log file; rotated files get `.1`…`.N` suffixes | +| `max_size_bytes` | `104857600` (100 MiB) | Rotate when the live file exceeds this size | +| `max_files` | `10` | Number of rotated files to keep (oldest is deleted on overflow) | +| `batch_interval_ms` | `1000` | How often (ms) the writer flushes a batch to disk | +| `batch_max_events` | `100` | Flush early when a batch reaches this many events | +| `queue_capacity` | `10000` | In-memory channel capacity between `record()` callers and the writer task | +| `queue_full_timeout_ms` | `250` | How long (ms) `record()` waits before dropping an event when the queue is full | +| `fsync_on_batch` | `false` | Call `fsync` after every batch — see [Durability](#durability) | +| `include_payload` | `true` | Write `data` fields; set to `false` to log event metadata only | +| `record_client_ip` | `false` | Include client IP in `rest_api.*` events — see [PII](#pii-and-retention) | +| `ip_anonymize` | `false` | Mask last IPv4 octet / last two IPv6 groups when recording IP | +| `ring_buffer_capacity` | `100` | In-memory ring for the REST read-path (see [Where it's consumed](#where-its-consumed)) | + +Example minimal override (all other fields keep their defaults): + +```json +{ + "audit_log": { + "path": "/var/log/nodectl/audit.jsonl", + "max_files": 30, + "fsync_on_batch": true + } +} +``` + +## Durability + +With `fsync_on_batch = false` (default), the kernel page cache is flushed on +the OS's own schedule. On a hard kill (`SIGKILL`) or power loss, up to +`batch_interval_ms` (~1 s) of events may be lost. + +Set `fsync_on_batch = true` for strict durability at higher disk cost (one +`fsync` per second by default; one per `batch_max_events` events at high +throughput). + +Events dropped because the writer queue is full are counted in the +`system.audit_events_dropped` event emitted on the next flush. + +## PII and retention + +Audit events may contain operator usernames, optionally client IP addresses, +and config change details. In GDPR-style regimes, IP addresses and usernames +are personal data. + +- `record_client_ip = false` (default): no IP is ever written. +- `record_client_ip = true`, `ip_anonymize = false`: full IP written. +- `record_client_ip = true`, `ip_anonymize = true`: last IPv4 octet zeroed, + last two IPv6 groups masked (`::0:0`). + +Retention is bounded by `max_size_bytes × max_files`. Tune for your policy. +Log files are **not** automatically deleted after a time-based retention period — +external tooling (logrotate, cron) is needed if you require time-based purges. + +## File permissions + +On Unix, the live file and all rotated files are created with mode `0600` +(owner read/write only). The directory is not created with any special mode — +ensure the directory itself has appropriate permissions. + +Tamper-evidence (hash chains, signed events) is **out of scope** for the +current release. Treat the audit log as protected by host trust and filesystem +ACLs, not by cryptography. + +## Where it's consumed + +`GET /v1/elections` reads from the **in-memory ring buffer** (last +`ring_buffer_capacity` events, default 100) and enriches `our_participants` +with: + +- `stake_submissions` — stake submission history from audit +- `last_error` — latest error-class event (stake skipped, stake failed, withdraw failed) + +The JSONL file on disk is **not** parsed on the hot path. + +## Analyzing the log + +```sh +# Count events by type +jq -r .event_type logs/audit.jsonl | sort | uniq -c | sort -rn + +# All events for one election round +jq 'select(.target.election_id == 1779265552)' logs/audit.jsonl + +# Failed or skipped stakes in the last file +jq 'select(.outcome == "failure" or .outcome == "skipped") + | select(.event_type | startswith("elections.stake"))' logs/audit.jsonl + +# Config mutations by a specific user +jq 'select(.event_type == "rest_api.config_updated" and .actor.id == "alice")' \ + logs/audit.jsonl + +# Tail-follow live events +tail -f logs/audit.jsonl | jq . + +# All events in a time range +jq 'select(.ts >= "2026-05-22T10:00:00Z" and .ts < "2026-05-22T11:00:00Z")' \ + logs/audit.jsonl + +# Events across rotated files (newest first) +cat logs/audit.jsonl.1 logs/audit.jsonl | jq . +``` From 3b7335545ae9f350a8024e6560c46e77bd0b73fe Mon Sep 17 00:00:00 2001 From: mrnkslv Date: Mon, 15 Jun 2026 17:12:02 +0300 Subject: [PATCH 30/30] fix: bug found after testing --- src/node-control/docs/audit-log.md | 8 +- src/node-control/service/src/audit/enums.rs | 2 +- src/node-control/service/src/audit/event.rs | 77 +- src/node-control/service/src/audit/factory.rs | 2 +- .../service/src/audit/in_memory.rs | 2 +- .../service/src/audit/jsonl_log.rs | 2 +- .../service/src/audit/jsonl_writer.rs | 52 +- src/node-control/service/src/audit/log.rs | 2 +- src/node-control/service/src/audit/mod.rs | 2 +- .../service/src/audit/ring_buffer.rs | 4 +- .../test_run_net_py/test_audit_integration.py | 705 ++++++++++++++++++ 11 files changed, 777 insertions(+), 81 deletions(-) create mode 100644 src/node/tests/test_run_net_py/test_audit_integration.py diff --git a/src/node-control/docs/audit-log.md b/src/node-control/docs/audit-log.md index 675fd1a0..3cff4a43 100644 --- a/src/node-control/docs/audit-log.md +++ b/src/node-control/docs/audit-log.md @@ -49,17 +49,19 @@ Each event contains: ## File layout ``` -./logs/audit.jsonl ← current file (line 0 is a file header, not an event) +./logs/audit.jsonl ← current file (line 0 is a system.service_started event) ./logs/audit.jsonl.1 ← most-recent rotation ./logs/audit.jsonl.2 … ./logs/audit.jsonl.9 ``` -The first line of every file is a **header** (no `event_type` field): +The first line of every (rotated) file is a regular `system.service_started` +event whose `data` carries the service `version` and `host`. There is no +special header format — every line is a uniform JSONL event: ```json -{"schema_version":1,"service":"nodectl","service_version":"0.7.0","host":"validator-1","started_at":"2026-05-22T12:00:00.000Z"} +{"id":"019ecb64-...","ts":"2026-05-22T12:00:00.000Z","outcome":"success","event_type":"system.service_started","data":{"version":"0.7.0","host":"validator-1"},"actor":{"kind":"system"},"target":{"kind":"system"}} ``` Defaults: 100 MiB per file, 10 files → ~1 GiB total history. diff --git a/src/node-control/service/src/audit/enums.rs b/src/node-control/service/src/audit/enums.rs index 727b95ea..9dec6734 100644 --- a/src/node-control/service/src/audit/enums.rs +++ b/src/node-control/service/src/audit/enums.rs @@ -87,7 +87,7 @@ pub enum AuditEventPayload { // ── system ─────────────────────────────────────────────────────────────── #[serde(rename = "system.service_started")] - SystemServiceStarted { version: String }, + SystemServiceStarted { version: String, host: String }, #[serde(rename = "system.service_stopped")] SystemServiceStopped {}, diff --git a/src/node-control/service/src/audit/event.rs b/src/node-control/service/src/audit/event.rs index d43d4299..379f578e 100644 --- a/src/node-control/service/src/audit/event.rs +++ b/src/node-control/service/src/audit/event.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use uuid::Uuid; /// Renders timestamps as RFC3339 with millisecond precision and a trailing `Z` -/// (e.g. `2026-05-22T12:10:30.123Z`), used for `ts` and `started_at`. +/// (e.g. `2026-05-22T12:10:30.123Z`), used for the `ts` field. mod ts_millis_rfc3339 { use super::*; @@ -31,26 +31,11 @@ mod ts_millis_rfc3339 { } } -/// First JSONL line of every (rotated) audit file. Readers distinguish it from -/// events by the absence of an `event_type` field. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AuditFileHeader { - pub schema_version: u16, - /// Logical service name, e.g. `"nodectl"`. - pub service: String, - /// Service semver. - pub service_version: String, - pub host: String, - #[serde(with = "ts_millis_rfc3339")] - pub started_at: DateTime, -} - /// A single audit record. /// /// Wire shape: `id`, `ts`, `outcome`, the flattened payload /// (`event_type` + `data`), `actor`, `target`. `severity`/`source` are derived -/// from the payload at the display layer and `schema_version` lives in -/// [`AuditFileHeader`], so none of them are stored per event. +/// from the payload at the display layer, so they are not stored per event. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct AuditEvent { /// UUID v7 — sortable by creation time. @@ -287,12 +272,12 @@ impl AuditEvent { ) } - pub fn system_service_started(version: impl Into) -> Self { + pub fn system_service_started(version: impl Into, host: impl Into) -> Self { Self::new( AuditActor::System, AuditTarget::System, AuditOutcome::Success, - AuditEventPayload::SystemServiceStarted { version: version.into() }, + AuditEventPayload::SystemServiceStarted { version: version.into(), host: host.into() }, ) } @@ -341,6 +326,31 @@ mod tests { AuditEvent { id: fixture_id(), ts: fixture_ts(), outcome, payload, actor, target } } + #[test] + fn serializes_service_started_to_expected_json() { + let event = fixed( + AuditOutcome::Success, + AuditActor::System, + AuditTarget::System, + AuditEventPayload::SystemServiceStarted { + version: "0.5.1".into(), + host: "node-host".into(), + }, + ); + assert_json_eq( + &event, + json!({ + "id": FIXTURE_ID, + "ts": FIXTURE_TS, + "outcome": "success", + "event_type": "system.service_started", + "data": { "version": "0.5.1", "host": "node-host" }, + "actor": { "kind": "system" }, + "target": { "kind": "system" } + }), + ); + } + #[test] fn serializes_stake_submitted_to_expected_json() { let event = fixed( @@ -405,30 +415,6 @@ mod tests { ); } - #[test] - fn file_header_serializes_with_millis_ts() { - let header = AuditFileHeader { - schema_version: 1, - service: "nodectl".into(), - service_version: "0.5.1".into(), - host: "node-host".into(), - started_at: fixture_ts(), - }; - let value = serde_json::to_value(&header).expect("serialize header"); - assert_eq!( - value, - json!({ - "schema_version": 1, - "service": "nodectl", - "service_version": "0.5.1", - "host": "node-host", - "started_at": FIXTURE_TS - }) - ); - // Header has no event_type — that is how readers tell it apart from events. - assert!(value.get("event_type").is_none()); - } - fn sample_event(payload: AuditEventPayload) -> AuditEvent { fixed( AuditOutcome::Success, @@ -481,7 +467,10 @@ mod tests { AuditEventPayload::RestApiTokenRejected { reason: "expired".into() }, AuditEventPayload::VaultKeyCreated {}, AuditEventPayload::VaultKeyRemoved {}, - AuditEventPayload::SystemServiceStarted { version: "0.5.0".into() }, + AuditEventPayload::SystemServiceStarted { + version: "0.5.0".into(), + host: "test-host".into(), + }, AuditEventPayload::SystemServiceStopped {}, AuditEventPayload::SystemAuditEventsDropped { dropped_events: 3, diff --git a/src/node-control/service/src/audit/factory.rs b/src/node-control/service/src/audit/factory.rs index 135c6edd..ef37d3be 100644 --- a/src/node-control/service/src/audit/factory.rs +++ b/src/node-control/service/src/audit/factory.rs @@ -45,7 +45,7 @@ mod tests { use tempfile::tempdir; fn sample_event() -> AuditEvent { - AuditEvent::system_service_started("test") + AuditEvent::system_service_started("test", "") } #[tokio::test] diff --git a/src/node-control/service/src/audit/in_memory.rs b/src/node-control/service/src/audit/in_memory.rs index fc999ffb..f2665bc1 100644 --- a/src/node-control/service/src/audit/in_memory.rs +++ b/src/node-control/service/src/audit/in_memory.rs @@ -46,7 +46,7 @@ mod tests { #[tokio::test] async fn records_and_drains_events() { let log = InMemoryAuditLog::new(); - let event = AuditEvent::system_service_started("test"); + let event = AuditEvent::system_service_started("test", ""); log.record(event.clone()).await; let drained = log.drain(); assert_eq!(drained.len(), 1); diff --git a/src/node-control/service/src/audit/jsonl_log.rs b/src/node-control/service/src/audit/jsonl_log.rs index f42c5343..d6a6a447 100644 --- a/src/node-control/service/src/audit/jsonl_log.rs +++ b/src/node-control/service/src/audit/jsonl_log.rs @@ -205,7 +205,7 @@ mod tests { use tempfile::tempdir; fn sample_event(tag: &str) -> AuditEvent { - AuditEvent::system_service_started(tag) + AuditEvent::system_service_started(tag, "") } fn test_config(path: PathBuf) -> AuditLogConfig { diff --git a/src/node-control/service/src/audit/jsonl_writer.rs b/src/node-control/service/src/audit/jsonl_writer.rs index d3cd5b20..5859ddbd 100644 --- a/src/node-control/service/src/audit/jsonl_writer.rs +++ b/src/node-control/service/src/audit/jsonl_writer.rs @@ -6,8 +6,7 @@ * * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ -use crate::audit::{AuditEvent, AuditFileHeader, AuditLogConfig, jsonl_log::AuditInitError}; -use chrono::Utc; +use crate::audit::{AuditEvent, AuditLogConfig, jsonl_log::AuditInitError}; use std::{ sync::{ Arc, Once, @@ -16,9 +15,6 @@ use std::{ time::Duration, }; -/// Schema version stamped into the per-file [`AuditFileHeader`]. -const AUDIT_SCHEMA_VERSION: u16 = 1; - static HOSTNAME_FALLBACK_WARNED: Once = Once::new(); #[derive(Debug)] @@ -39,7 +35,7 @@ pub(crate) enum AuditCommand { } pub(crate) struct AuditWriter { - /// Host identity stamped into each new file's [`AuditFileHeader`]. + /// Host identity written as `data.host` into the first [`AuditEvent`] of each file segment. host: String, config: Arc, /// Live append handle. `None` only transiently during rotation (the old @@ -128,26 +124,22 @@ impl AuditWriter { last_dropped_seen: 0, write_delay, }; - writer.write_header_if_empty().await.map_err(AuditInitError::FileOpen)?; + writer.write_startup_event_if_new().await.map_err(AuditInitError::FileOpen)?; Ok(writer) } - fn file_header(&self) -> AuditFileHeader { - AuditFileHeader { - schema_version: AUDIT_SCHEMA_VERSION, - service: "nodectl".into(), - service_version: env!("CARGO_PKG_VERSION").into(), - host: self.host.clone(), - started_at: Utc::now(), - } - } - - async fn write_header_if_empty(&mut self) -> std::io::Result<()> { + /// Writes a `system.service_started` event as the very first line of a new file segment. + /// + /// Called on fresh open (empty file) and after rotation. Every segment starts with this + /// event so readers always know which service version and host produced the following lines, + /// with no special-case header format to distinguish. + async fn write_startup_event_if_new(&mut self) -> std::io::Result<()> { if self.current_size != 0 { return Ok(()); } use tokio::io::AsyncWriteExt; - let mut line = serde_json::to_vec(&self.file_header()) + let ev = AuditEvent::system_service_started(env!("CARGO_PKG_VERSION"), self.host.as_str()); + let mut line = serde_json::to_vec(&ev) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; line.push(b'\n'); let file = self @@ -422,7 +414,7 @@ impl AuditWriter { } self.file = Some(file); self.current_size = 0; - self.write_header_if_empty().await?; + self.write_startup_event_if_new().await?; Ok(()) } @@ -474,11 +466,11 @@ mod tests { } fn sample_event(tag: &str) -> AuditEvent { - AuditEvent::system_service_started(tag) + AuditEvent::system_service_started(tag, "") } fn large_event(payload_kb: usize) -> AuditEvent { - AuditEvent::system_service_started("x".repeat(payload_kb * 1024)) + AuditEvent::system_service_started("x".repeat(payload_kb * 1024), "") } fn test_config(dir: &Path, mut cfg: AuditLogConfig) -> AuditLogConfig { @@ -531,16 +523,24 @@ mod tests { tx.send(AuditCommand::Shutdown).await.unwrap(); } - /// Reads event lines, skipping the per-file [`AuditFileHeader`] (no `event_type`). + /// Reads all event lines, skipping only the very first `system.service_started` line + /// that the writer automatically prepends to every new file segment. Test events of + /// the same type (if any) are preserved so counting assertions stay correct. fn read_json_lines(path: &Path) -> Vec { assert!(path.exists(), "audit file missing at {}", path.display()); let content = std::fs::read_to_string(path).unwrap(); - content + let mut lines: Vec = content .lines() .filter(|line| !line.is_empty()) .map(|line| serde_json::from_str::(line).expect("valid json line")) - .filter(|value| value.get("event_type").is_some()) - .collect() + .collect(); + // Drop only the first line when it is the writer-injected startup event. + if lines.first().and_then(|v| v.get("event_type")).and_then(|t| t.as_str()) + == Some("system.service_started") + { + lines.remove(0); + } + lines } fn count_rotated_files(dir: &Path) -> usize { diff --git a/src/node-control/service/src/audit/log.rs b/src/node-control/service/src/audit/log.rs index 3ef705c0..8ba5d66d 100644 --- a/src/node-control/service/src/audit/log.rs +++ b/src/node-control/service/src/audit/log.rs @@ -28,6 +28,6 @@ mod tests { #[tokio::test] async fn noop_audit_log_record_completes() { let log = NoopAuditLog; - log.record(AuditEvent::system_service_started("test")).await; + log.record(AuditEvent::system_service_started("test", "")).await; } } diff --git a/src/node-control/service/src/audit/mod.rs b/src/node-control/service/src/audit/mod.rs index e154a9ff..7c0f71be 100644 --- a/src/node-control/service/src/audit/mod.rs +++ b/src/node-control/service/src/audit/mod.rs @@ -24,7 +24,7 @@ pub use common::app_config::AuditLogConfig; pub use enums::{ AuditEventPayload, AuditOutcome, AuditSeverity, AuditSource, ConfigFieldChange, StakeSkipReason, }; -pub use event::{AuditEvent, AuditFileHeader, ElectionsStakeSubmittedParams}; +pub use event::{AuditEvent, ElectionsStakeSubmittedParams}; pub use factory::AuditLogFactory; #[cfg(test)] pub use in_memory::InMemoryAuditLog; diff --git a/src/node-control/service/src/audit/ring_buffer.rs b/src/node-control/service/src/audit/ring_buffer.rs index 3cd4a1a9..4352f0fe 100644 --- a/src/node-control/service/src/audit/ring_buffer.rs +++ b/src/node-control/service/src/audit/ring_buffer.rs @@ -96,7 +96,7 @@ mod tests { use std::sync::{Arc, Barrier}; fn ev(tag: &str) -> AuditEvent { - AuditEvent::system_service_started(tag) + AuditEvent::system_service_started(tag, "") } // ── original tests ──────────────────────────────────────────────────────── @@ -185,7 +185,7 @@ mod tests { let b = buf.clone(); handles.push(std::thread::spawn(move || { for i in 0..100u32 { - b.push(AuditEvent::system_service_started(&format!("t{t}-{i}"))); + b.push(AuditEvent::system_service_started(&format!("t{t}-{i}"), "")); } })); } diff --git a/src/node/tests/test_run_net_py/test_audit_integration.py b/src/node/tests/test_run_net_py/test_audit_integration.py new file mode 100644 index 00000000..13a85049 --- /dev/null +++ b/src/node/tests/test_run_net_py/test_audit_integration.py @@ -0,0 +1,705 @@ +#!/usr/bin/env python3 +""" +Audit-log integration test suite for nodectl. + +Can be run stand-alone after run_singlehost_nodectl.py has brought up the +service, or invoked as a phase from inside that bootstrap script. + +Exit code: 0 — all required checks pass; 1 — one or more failures. + +Required env vars (or CLI flags): + CONFIG_PATH — path to the running nodectl-config.json + NODECTL_API_TOKEN — JWT token for REST calls (produced by phase 8 auth setup) + +Optional env vars: + AUDIT_LOG_PATH — override audit file path (default: read from config, + fall back to /logs/audit.jsonl) + NODECTL_REST_URL — override REST base URL (default: read from config http.bind) + +Usage: + python3 test_audit_integration.py [--config /path/to/nodectl-config.json] + [--rest-url http://127.0.0.1:8080] + [--audit-log /path/to/audit.jsonl] + [--verbose] +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, Optional + +# ── RFC 3339 millis with Z: 2026-05-22T12:10:30.123Z ───────────────────────── +_TS_PATTERN = re.compile( + r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$" +) + +REQUIRED_EVENT_FIELDS = ("id", "ts", "outcome", "event_type", "actor", "target") +VALID_OUTCOMES = {"success", "failure", "skipped"} + +# Canonical set of event_type values produced by the current service version. +# Events outside this set indicate schema drift (removed/renamed variant). +KNOWN_EVENT_TYPES = { + "elections.key_generated", + "elections.stake_submitted", + "elections.stake_accepted", + "elections.stake_skipped", + "elections.stake_failed", + "elections.stake_recovered", + "elections.stake_recover_failed", + "elections.withdraw_processed", + "elections.withdraw_failed", + "rewards.distribution_started", + "rewards.distribution_completed", + "rewards.distribution_failed", + "rewards.recipient_skipped", + "rest_api.config_updated", + "rest_api.auth_login_succeeded", + "rest_api.auth_login_rejected", + "rest_api.token_rejected", + "vault.key_created", + "vault.key_removed", + "system.service_started", + "system.service_stopped", + "system.audit_events_dropped", +} + + +# ══════════════════════════════════════════════════════════════════════════════ +# Tiny colour logger +# ══════════════════════════════════════════════════════════════════════════════ + +class Logger: + def __init__(self, verbose: bool = False) -> None: + self.verbose = verbose + + def _emit(self, colour: str, label: str, msg: str) -> None: + print(f"\033[{colour}m[{label}]\033[0m {msg}", flush=True) + + def pass_(self, msg: str) -> None: self._emit("32", "PASS", msg) + def fail(self, msg: str) -> None: self._emit("31", "FAIL", msg) + def skip(self, msg: str) -> None: self._emit("33", "SKIP", msg) + def info(self, msg: str) -> None: self._emit("36", "INFO", msg) + def debug(self, msg: str) -> None: + if self.verbose: + self._emit("37", "DBG ", msg) + + +# ══════════════════════════════════════════════════════════════════════════════ +# Result accumulator +# ══════════════════════════════════════════════════════════════════════════════ + +class Results: + def __init__(self) -> None: + self.passed = 0 + self.failed = 0 + self.skipped = 0 + + def record_pass(self, log: Logger, msg: str) -> None: + self.passed += 1 + log.pass_(msg) + + def record_fail(self, log: Logger, msg: str) -> None: + self.failed += 1 + log.fail(msg) + + def record_skip(self, log: Logger, msg: str) -> None: + self.skipped += 1 + log.skip(msg) + + @property + def ok(self) -> bool: + return self.failed == 0 + + def summary(self, log: Logger) -> None: + colour = "32" if self.ok else "31" + log._emit(colour, "SUM", + f"passed={self.passed} failed={self.failed} skipped={self.skipped}") + + +# ══════════════════════════════════════════════════════════════════════════════ +# Config / path resolution +# ══════════════════════════════════════════════════════════════════════════════ + +def resolve_audit_log_path(config_path: Path) -> Path: + """ + Find the audit.jsonl path using the following priority: + + 1. $AUDIT_LOG_PATH env var + 2. `audit.path` field in the config file + - absolute path used as-is + - relative path resolved against: config-dir, then CWD, first that exists + 3. Default candidates (first that exists wins): + - ./logs/audit.jsonl (service CWD — matches the nodectl default) + - /logs/audit.jsonl + """ + env_override = os.environ.get("AUDIT_LOG_PATH", "").strip() + if env_override: + return Path(env_override) + + try: + cfg = json.loads(config_path.read_text()) + audit_path_str = cfg.get("audit", {}).get("path", "") + if audit_path_str: + p = Path(audit_path_str) + if p.is_absolute(): + return p + # Relative: try config-dir first, then CWD + candidate_cfg = config_path.parent / p + if candidate_cfg.exists(): + return candidate_cfg + candidate_cwd = Path.cwd() / p + if candidate_cwd.exists(): + return candidate_cwd + return candidate_cfg # return best-guess even if missing + except Exception: + pass + + # Default: service writes to ./logs/audit.jsonl relative to its CWD. + # The bootstrap script starts nodectl from test_run_net_py/, so that + # matches /logs/audit.jsonl. Fall back to config-dir if + # the CWD candidate does not exist. + cwd_candidate = Path.cwd() / "logs" / "audit.jsonl" + if cwd_candidate.exists(): + return cwd_candidate + + cfg_candidate = config_path.parent / "logs" / "audit.jsonl" + if cfg_candidate.exists(): + return cfg_candidate + + # Neither exists — return CWD candidate so the error message is meaningful + return cwd_candidate + + +def resolve_rest_base_url(config_path: Path) -> str: + """Derive REST base URL from config http.bind or env.""" + env_override = os.environ.get("NODECTL_REST_URL", "").strip() + if env_override: + return env_override.rstrip("/") + + try: + cfg = json.loads(config_path.read_text()) + bind = str(cfg.get("http", {}).get("bind", "127.0.0.1:8080")) + except Exception: + bind = "127.0.0.1:8080" + + if bind.startswith("["): + bracket_end = bind.find("]") + host = bind[1:bracket_end] if bracket_end > 0 else "127.0.0.1" + rest = bind[bracket_end + 1:].lstrip() if bracket_end > 0 else "" + port = rest[1:] if rest.startswith(":") else "8080" + elif bind.count(":") == 1: + host, port = bind.split(":", 1) + else: + host, port = "127.0.0.1", "8080" + + if host in ("0.0.0.0", "::"): + host = "127.0.0.1" + + return (f"http://[{host}]:{port}" if ":" in host else f"http://{host}:{port}").rstrip("/") + + +def rest_get_json(base_url: str, path: str, token: str, timeout: int = 15) -> dict: + url = base_url + path + req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) + body = "" + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read().decode(errors="replace") + return json.loads(body) + + +# ══════════════════════════════════════════════════════════════════════════════ +# JSONL parsing helpers +# ══════════════════════════════════════════════════════════════════════════════ + +def read_jsonl(path: Path) -> tuple[list[dict], list[str]]: + """ + Returns (records, errors). Each record is a parsed JSON object. + errors contains descriptions of lines that failed to parse. + """ + records: list[dict] = [] + errors: list[str] = [] + for i, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + raw = raw.strip() + if not raw: + continue + try: + obj = json.loads(raw) + if not isinstance(obj, dict): + errors.append(f"line {i}: not a JSON object: {raw[:120]}") + else: + records.append(obj) + except json.JSONDecodeError as e: + errors.append(f"line {i}: {e.msg}: {raw[:120]}") + return records, errors + + + + +# ══════════════════════════════════════════════════════════════════════════════ +# Test suites +# ══════════════════════════════════════════════════════════════════════════════ + +class AuditFileTests: + """Tests that inspect the audit.jsonl file on disk.""" + + def __init__(self, audit_path: Path, log: Logger, results: Results) -> None: + self.path = audit_path + self.log = log + self.r = results + self._records: list[dict] = [] + self._events: list[dict] = [] + + def run_all(self) -> None: + self.log.info(f"=== Audit file checks: {self.path} ===") + + if not self._check_file_exists(): + return # nothing to test without the file + + records, errors = read_jsonl(self.path) + + if errors: + for e in errors: + self.r.record_fail(self.log, f"JSONL parse error — {e}") + else: + self.r.record_pass(self.log, "All lines parse as valid JSON objects") + + self._records = records + # Unified JSONL format: every line is an event carrying `event_type`. + # Guard against any stray line without it (e.g. a legacy header artifact). + self._events = [r for r in records if "event_type" in r] + + self._check_file_header(records) + self._check_event_fields() + self._check_ts_format() + self._check_outcome_values() + self._check_service_started_present() + self._check_no_dedup_duplicates() + self._check_actor_shape() + self._check_no_unknown_event_types() + self._check_target_shape() + + # ── individual checks ───────────────────────────────────────────────────── + + def _check_file_exists(self) -> bool: + if self.path.exists(): + size = self.path.stat().st_size + self.r.record_pass(self.log, f"audit.jsonl exists ({size} bytes)") + return True + self.r.record_fail(self.log, f"audit.jsonl not found: {self.path}") + return False + + def _check_file_header(self, records: list[dict]) -> None: + """First line must be a system.service_started event with version and host fields.""" + if not records: + self.r.record_fail(self.log, "audit.jsonl is empty — no events at all") + return + + first = records[0] + et = first.get("event_type") + if et != "system.service_started": + self.r.record_fail(self.log, + f"First line must be system.service_started, got event_type={et!r}") + return + + data = first.get("data", {}) + missing = [f for f in ("version", "host") if f not in data] + if missing: + self.r.record_fail(self.log, + f"system.service_started missing data fields: {missing}") + else: + self.r.record_pass(self.log, + f"First event is system.service_started " + f"(version={data.get('version')!r}, host={data.get('host')!r})") + + def _check_event_fields(self) -> None: + if not self._events: + self.r.record_skip(self.log, "No events in file — skipping field checks") + return + bad: list[str] = [] + for ev in self._events: + missing = [f for f in REQUIRED_EVENT_FIELDS if f not in ev] + if missing: + bad.append(f"event_type={ev.get('event_type')!r} id={ev.get('id')!r}: " + f"missing {missing}") + if bad: + for b in bad[:5]: + self.r.record_fail(self.log, f"Event missing required fields — {b}") + if len(bad) > 5: + self.r.record_fail(self.log, f" … and {len(bad) - 5} more") + else: + self.r.record_pass(self.log, + f"All {len(self._events)} events have required fields " + f"({', '.join(REQUIRED_EVENT_FIELDS)})") + + def _check_ts_format(self) -> None: + if not self._events: + return + bad: list[str] = [] + for ev in self._events: + ts = ev.get("ts", "") + if not _TS_PATTERN.match(ts): + bad.append(f"event_type={ev.get('event_type')!r}: ts={ts!r}") + if bad: + for b in bad[:5]: + self.r.record_fail(self.log, f"Bad ts format — {b}") + else: + self.r.record_pass(self.log, + f"All {len(self._events)} events have RFC3339-millis-Z timestamps") + + def _check_outcome_values(self) -> None: + if not self._events: + return + bad: list[str] = [] + for ev in self._events: + o = ev.get("outcome") + if o not in VALID_OUTCOMES: + bad.append(f"event_type={ev.get('event_type')!r}: outcome={o!r}") + if bad: + for b in bad[:5]: + self.r.record_fail(self.log, f"Invalid outcome — {b}") + else: + self.r.record_pass(self.log, + f"All {len(self._events)} events have valid outcome values") + + def _check_service_started_present(self) -> None: + found = any(ev.get("event_type") == "system.service_started" + for ev in self._events) + if found: + self.r.record_pass(self.log, "system.service_started event present") + else: + self.r.record_fail(self.log, + "system.service_started event not found in audit.jsonl " + "(expected on service startup)") + + def _check_no_unknown_event_types(self) -> None: + unknown = { + ev.get("event_type") for ev in self._events + if ev.get("event_type") not in KNOWN_EVENT_TYPES + } + if unknown: + for et in sorted(unknown): + self.r.record_fail(self.log, + f"Unknown event_type {et!r} — removed from schema or schema drift " + f"(check AuditEventPayload enum)") + else: + self.r.record_pass(self.log, + f"All {len(self._events)} events use known event_type values") + + def _check_no_dedup_duplicates(self) -> None: + """No two stake_skipped events with the same (node_id, election_id, reason).""" + skipped_events = [ + ev for ev in self._events + if ev.get("event_type") == "elections.stake_skipped" + ] + if not skipped_events: + self.r.record_skip(self.log, + "No elections.stake_skipped events — dedup check skipped") + return + + seen: set[tuple] = set() + dups: list[str] = [] + for ev in skipped_events: + target = ev.get("target", {}) + node_id = target.get("id", "") + election_id = target.get("election_id") + data = ev.get("data", {}) + reason = data.get("reason", "") + key = (node_id, election_id, reason) + if key in seen: + dups.append( + f"node_id={node_id!r} election_id={election_id} reason={reason!r}" + ) + else: + seen.add(key) + + if dups: + for d in dups[:5]: + self.r.record_fail(self.log, + f"Duplicate stake_skipped in file (dedup failed) — {d}") + else: + self.r.record_pass(self.log, + f"No duplicate stake_skipped events among {len(skipped_events)} " + f"(unique keys: {len(seen)})") + + def _check_actor_shape(self) -> None: + if not self._events: + return + bad: list[str] = [] + for ev in self._events: + actor = ev.get("actor") + if not isinstance(actor, dict): + bad.append(f"event_type={ev.get('event_type')!r}: actor is not an object") + continue + if "kind" not in actor: + bad.append(f"event_type={ev.get('event_type')!r}: actor missing 'kind'") + if bad: + for b in bad[:5]: + self.r.record_fail(self.log, f"Bad actor shape — {b}") + else: + self.r.record_pass(self.log, f"All {len(self._events)} actors have 'kind' field") + + def _check_target_shape(self) -> None: + if not self._events: + return + bad: list[str] = [] + for ev in self._events: + target = ev.get("target") + if not isinstance(target, dict): + bad.append(f"event_type={ev.get('event_type')!r}: target is not an object") + continue + if "kind" not in target: + bad.append(f"event_type={ev.get('event_type')!r}: target missing 'kind'") + if bad: + for b in bad[:5]: + self.r.record_fail(self.log, f"Bad target shape — {b}") + else: + self.r.record_pass(self.log, f"All {len(self._events)} targets have 'kind' field") + + # ── summary helpers ──────────────────────────────────────────────────────── + + def print_event_type_counts(self) -> None: + if not self._events: + return + counts: dict[str, int] = {} + for ev in self._events: + et = ev.get("event_type", "") + counts[et] = counts.get(et, 0) + 1 + self.log.info("Event type distribution:") + for et, n in sorted(counts.items()): + self.log.info(f" {n:4d} {et}") + + +class AuditRestTests: + """Tests that call GET /v1/elections and validate audit-enriched fields.""" + + def __init__( + self, + rest_url: str, + token: str, + log: Logger, + results: Results, + ) -> None: + self.rest_url = rest_url + self.token = token + self.log = log + self.r = results + + def run_all(self) -> None: + self.log.info(f"=== REST API checks: {self.rest_url}/v1/elections ===") + + data = self._fetch_elections() + if data is None: + return + + self._check_response_shape(data) + self._check_recent_events_empty(data) + self._check_participants_structure(data) + self._check_stake_submissions_no_duplicates(data) + + # ── individual checks ───────────────────────────────────────────────────── + + def _fetch_elections(self) -> Optional[dict]: + try: + data = rest_get_json(self.rest_url, "/v1/elections", self.token) + self.r.record_pass(self.log, "GET /v1/elections → HTTP 200") + return data + except urllib.error.HTTPError as e: + body = e.read().decode(errors="replace")[:400] + self.r.record_fail(self.log, + f"GET /v1/elections → HTTP {e.code}: {body}") + except urllib.error.URLError as e: + self.r.record_fail(self.log, + f"GET /v1/elections connection failed: {e.reason}") + except json.JSONDecodeError as e: + self.r.record_fail(self.log, + f"GET /v1/elections returned invalid JSON: {e.msg}") + return None + + def _check_response_shape(self, data: dict) -> None: + # Accept both {ok, result} and flat shapes (depending on version) + result = data.get("result", data) + if not isinstance(result, dict): + self.r.record_fail(self.log, + f"elections response result is not an object: {type(result)}") + return + + has_election_id = "election_id" in result or "election_id" in data + has_participants = ( + "our_participants" in result or "our_participants" in data + ) + if has_participants: + self.r.record_pass(self.log, "elections response contains 'our_participants'") + else: + self.r.record_fail(self.log, + "elections response has no 'our_participants' field") + + def _check_recent_events_empty(self, data: dict) -> None: + """recent_events must be absent or an empty list (sma-106 skips serialization).""" + result = data.get("result", data) + recent = result.get("recent_events") + if recent is None: + self.r.record_pass(self.log, + "recent_events absent from response (skip_serializing_if = empty)") + elif isinstance(recent, list) and len(recent) == 0: + self.r.record_pass(self.log, "recent_events is present but empty []") + else: + self.r.record_fail(self.log, + f"recent_events is unexpected: {str(recent)[:200]}") + + def _check_participants_structure(self, data: dict) -> None: + result = data.get("result", data) + participants = result.get("our_participants", []) + if not isinstance(participants, list): + self.r.record_fail(self.log, + f"our_participants is not a list: {type(participants)}") + return + if not participants: + self.r.record_skip(self.log, + "our_participants is empty — skipping per-participant checks") + return + + bad_last_error: list[str] = [] + for p in participants: + node_id = p.get("node_id", "?") + last_error = p.get("last_error") + if last_error is not None and not isinstance(last_error, str): + bad_last_error.append( + f"node_id={node_id!r}: last_error is not a string: {last_error!r}" + ) + + if bad_last_error: + for b in bad_last_error: + self.r.record_fail(self.log, f"Bad last_error type — {b}") + else: + with_errors = [p for p in participants if p.get("last_error")] + self.r.record_pass(self.log, + f"our_participants: {len(participants)} participant(s), " + f"{len(with_errors)} with last_error populated") + if with_errors: + for p in with_errors: + self.log.info( + f" node_id={p.get('node_id')!r} " + f"last_error={p.get('last_error')!r}" + ) + + def _check_stake_submissions_no_duplicates(self, data: dict) -> None: + result = data.get("result", data) + participants = result.get("our_participants", []) + if not isinstance(participants, list) or not participants: + return + + dups: list[str] = [] + for p in participants: + node_id = p.get("node_id", "?") + subs = p.get("stake_submissions") or [] + seen: set[tuple] = set() + for s in subs: + key = (s.get("stake"), s.get("submission_time")) + if key in seen: + dups.append( + f"node_id={node_id!r} stake={key[0]!r} time={key[1]!r}" + ) + else: + seen.add(key) + + if dups: + for d in dups[:5]: + self.r.record_fail(self.log, + f"Duplicate stake_submission in REST response — {d}") + else: + total_subs = sum(len(p.get("stake_submissions") or []) + for p in participants) + self.r.record_pass(self.log, + f"No duplicate stake_submissions across {len(participants)} " + f"participant(s) ({total_subs} total)") + + +# ══════════════════════════════════════════════════════════════════════════════ +# Entry point +# ══════════════════════════════════════════════════════════════════════════════ + +def parse_args() -> argparse.Namespace: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--config", metavar="PATH", + default=os.environ.get("CONFIG_PATH", ""), + help="Path to nodectl-config.json " + "(default: $CONFIG_PATH)") + ap.add_argument("--rest-url", metavar="URL", + default="", + help="Base REST URL, e.g. http://127.0.0.1:8080 " + "(default: derive from config http.bind or $NODECTL_REST_URL)") + ap.add_argument("--audit-log", metavar="PATH", + default="", + help="Override audit.jsonl path " + "(default: derive from config or $AUDIT_LOG_PATH)") + ap.add_argument("--verbose", "-v", action="store_true", + help="Print debug lines") + return ap.parse_args() + + +def main() -> None: + args = parse_args() + log = Logger(verbose=args.verbose) + results = Results() + + # ── Resolve config path ──────────────────────────────────────────────────── + config_path_str = args.config or os.environ.get("CONFIG_PATH", "") + if not config_path_str: + log._emit("31", "FATAL", + "CONFIG_PATH is not set. " + "Pass --config or set the CONFIG_PATH environment variable.") + sys.exit(1) + + config_path = Path(config_path_str) + if not config_path.exists(): + log._emit("31", "FATAL", f"Config file not found: {config_path}") + sys.exit(1) + + # ── Resolve audit log path ───────────────────────────────────────────────── + if args.audit_log: + audit_path = Path(args.audit_log) + elif os.environ.get("AUDIT_LOG_PATH"): + audit_path = Path(os.environ["AUDIT_LOG_PATH"]) + else: + audit_path = resolve_audit_log_path(config_path) + + # ── Resolve REST URL ─────────────────────────────────────────────────────── + rest_url = (args.rest_url or "").rstrip("/") or resolve_rest_base_url(config_path) + + ts = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + log.info(f"=== {ts} test_audit_integration.py ===") + log.info(f"config: {config_path}") + log.info(f"audit log: {audit_path}") + log.info(f"rest url: {rest_url}") + + # ── Suite 1: audit.jsonl file ───────────────────────────────────────────── + file_tests = AuditFileTests(audit_path, log, results) + file_tests.run_all() + file_tests.print_event_type_counts() + + # ── Suite 2: REST API ───────────────────────────────────────────────────── + token = os.environ.get("NODECTL_API_TOKEN", "").strip() + if not token: + results.record_skip(log, + "NODECTL_API_TOKEN not set — skipping all REST API checks") + else: + rest_tests = AuditRestTests(rest_url, token, log, results) + rest_tests.run_all() + + # ── Final summary ───────────────────────────────────────────────────────── + print() + results.summary(log) + sys.exit(0 if results.ok else 1) + + +if __name__ == "__main__": + main()