diff --git a/crates/hi-agent/src/agent/turn.rs b/crates/hi-agent/src/agent/turn.rs index b882f709..305acd6e 100644 --- a/crates/hi-agent/src/agent/turn.rs +++ b/crates/hi-agent/src/agent/turn.rs @@ -23,10 +23,10 @@ use super::mutation_recovery_turn::MutationRecoveryControl; use crate::command; use crate::compaction; use crate::heuristics::{ - RECOVERY_SAMPLING, StallMode, emit_tool_output, humanize_count, looks_like_continue, - looks_like_unfinished_step, looks_mutating, mode_blocks_tool, parse_text_tool_calls, - plan_has_pending_steps, recovery_sampling, recovery_telemetry, respects_deps, - textcall_id_offset, tool_deps, tool_mode_label, + RECOVERY_SAMPLING, StallMode, VERIFY_REPEAT_STOP, emit_tool_output, humanize_count, + looks_like_continue, looks_like_unfinished_step, looks_mutating, mode_blocks_tool, + parse_text_tool_calls, plan_has_pending_steps, recovery_sampling, recovery_telemetry, + respects_deps, textcall_id_offset, tool_deps, tool_mode_label, }; use crate::snapshot::changed_files_between; use crate::steering::{ @@ -91,6 +91,7 @@ fn build_turn_telemetry( effective_max_steps, verify_rounds, repeated_verify_failures, + verify_repeat_stopped: false, recovery_retries, repeat_nudges, continue_nudges, @@ -1239,6 +1240,8 @@ impl crate::Agent { let mut stalled_repeating = false; // Whether the turn ended without enough evidence for a read-only review. let mut stalled_unfinished = false; + // Whether a third identical verify failure ended the turn early. + let mut verify_repeat_stopped = false; // Whether the turn was cut short by the per-turn step cap, so the // finalization recap is skipped (the work may be incomplete). let mut ended_at_cap = false; @@ -4113,6 +4116,27 @@ If the task is already complete, stop and give your final recap." // a misdiagnosed cause is the costliest failure behavior — // when the failure signature didn't change, demand a // validated diagnosis instead of another blind patch. + // Three identical failures = one normal repair AND one + // diagnose-first repair both changed nothing. Weak-model + // A/B data: recovery from here is 0% with or without + // richer nudges — further rounds only burn tokens, so end + // the turn honestly instead (kill switch: + // HI_VERIFY_REPEAT_STOP=0). No new nudge: a trailing + // nudge is stripped at turn end anyway, and the previous + // round's escalated nudge already carries this identical + // output. + if repeated + && verifier.consecutive_repeated_failures() >= 2 + && *VERIFY_REPEAT_STOP + { + stalled_unfinished = true; + verify_repeat_stopped = true; + last_verify_attributions = hi_tools::parse_attributions(&output, 3); + ui.status( + "verification failed identically 3 times; stopping the turn — repair attempts are not changing the outcome. /retry, or send 'continue'.", + ); + break 'turn; + } let followup = if repeated { "This is the SAME failure as the previous verify round — your last \ change did not alter the outcome. Do not edit again yet. First state \ @@ -4239,6 +4263,7 @@ If the task is already complete, stop and give your final recap." turn_checkpoint_allowed.map(|_| turn_checkpoint_created); self.last_turn_telemetry.advertised_tools = advertised_tool_names.into_iter().collect(); self.last_turn_telemetry.tool_schema_tokens = tool_schema_tokens; + self.last_turn_telemetry.verify_repeat_stopped = verify_repeat_stopped; // Verifier-gated skill auto-curation: after a turn that PASSED verification // and actually changed files, optionally distill a reusable technique into a diff --git a/crates/hi-agent/src/heuristics.rs b/crates/hi-agent/src/heuristics.rs index 05e79826..7fea8efe 100644 --- a/crates/hi-agent/src/heuristics.rs +++ b/crates/hi-agent/src/heuristics.rs @@ -617,6 +617,18 @@ pub(crate) static RECOVERY_SAMPLING: std::sync::LazyLock = std::sync::Lazy ) }); +/// Whether a third identical verify failure ends the turn instead of running +/// another repair round. A/B data (weak-model runs): once a failure signature +/// survives one normal repair AND one diagnose-first repair, further rounds +/// recover 0% and only burn tokens. Off via `HI_VERIFY_REPEAT_STOP=0/off/false/no` +/// for eval A/Bs. Read once. +pub(crate) static VERIFY_REPEAT_STOP: std::sync::LazyLock = std::sync::LazyLock::new(|| { + !matches!( + std::env::var("HI_VERIFY_REPEAT_STOP").ok().as_deref(), + Some("0" | "off" | "false" | "no") + ) +}); + /// Sampling for a model round, escalating with the count of consecutive /// content-less rounds (`retries`; 0 = the normal first attempt). Returns /// `(temperature, top_p, frequency_penalty)`. On a normal round — or when recovery diff --git a/crates/hi-agent/src/lib.rs b/crates/hi-agent/src/lib.rs index 011ea297..893ba531 100644 --- a/crates/hi-agent/src/lib.rs +++ b/crates/hi-agent/src/lib.rs @@ -132,6 +132,9 @@ pub struct TurnTelemetry { /// How many verify rounds re-failed with the same failure signature as /// their predecessor — repair attempts that did not change the outcome. pub repeated_verify_failures: u32, + /// Whether a third identical verify failure ended the turn early + /// instead of running another doomed repair round. + pub verify_repeat_stopped: bool, /// Times a content-less / malformed response was silently re-sampled /// (recovery sampling). 0 on a clean turn. pub recovery_retries: u32, @@ -223,6 +226,7 @@ impl Default for TurnTelemetry { effective_max_steps: 0, verify_rounds: 0, repeated_verify_failures: 0, + verify_repeat_stopped: false, recovery_retries: 0, repeat_nudges: 0, continue_nudges: 0, diff --git a/crates/hi-agent/src/tests/verify.rs b/crates/hi-agent/src/tests/verify.rs index c8b2002f..695b238d 100644 --- a/crates/hi-agent/src/tests/verify.rs +++ b/crates/hi-agent/src/tests/verify.rs @@ -117,7 +117,10 @@ async fn repeated_verify_failure_escalates_nudge() { "test", "printf 'error: same failure\\n' >&2; exit 1", )]); - cfg.max_verify_repairs = 2; + // One repair round: the second (identical) failure carries the + // diagnose-first escalation; the stop-early path (third identical) is + // covered by `third_identical_verify_failure_stops_the_turn`. + cfg.max_verify_repairs = 1; let tmp = workspace.path("changed.rs"); let p = tmp.to_string_lossy().to_string(); let responses = vec![ @@ -125,13 +128,13 @@ async fn repeated_verify_failure_escalates_nudge() { completion(vec![Content::Text("attempt 1".into())], 1, 1), completion(vec![Content::Text("attempt 2".into())], 1, 1), completion(vec![Content::Text("attempt 3".into())], 1, 1), - completion(vec![Content::Text("attempt 4".into())], 1, 1), ]; let mut agent = agent(responses, cfg); agent.run_turn("x", &mut NullUi).await.unwrap(); assert_eq!(agent.last_verify(), Some(false)); - assert_eq!(agent.last_turn_telemetry().verify_rounds, 3); - assert_eq!(agent.last_turn_telemetry().repeated_verify_failures, 2); + assert_eq!(agent.last_turn_telemetry().verify_rounds, 2); + assert_eq!(agent.last_turn_telemetry().repeated_verify_failures, 1); + assert!(!agent.last_turn_telemetry().verify_repeat_stopped); let nudge = agent .messages() .iter() @@ -150,6 +153,63 @@ async fn repeated_verify_failure_escalates_nudge() { ); } +#[tokio::test] +async fn third_identical_verify_failure_stops_the_turn() { + let workspace = IsolatedWorkspace::new("verify-repeat-stop"); + // Budget allows 6 rounds, but after three identical failures (one normal + // repair + one diagnose-first repair, both changing nothing) the turn must + // stop. The Canned provider holds exactly the completions to REACH round + // three — a fourth repair round would panic on an empty provider. + let mut cfg = workspace.config(); + cfg.verification = crate::VerificationMode::Explicit(vec![VerifyStage::new( + "test", + "printf 'error: same failure\\n' >&2; exit 1", + )]); + cfg.max_verify_repairs = 5; + let tmp = workspace.path("changed.rs"); + let p = tmp.to_string_lossy().to_string(); + let responses = vec![ + write_completion(&p), + completion(vec![Content::Text("attempt 1".into())], 1, 1), + completion(vec![Content::Text("attempt 2".into())], 1, 1), + completion(vec![Content::Text("attempt 3".into())], 1, 1), + ]; + let mut agent = agent(responses, cfg); + let mut ui = RecUi::default(); + agent.run_turn("x", &mut ui).await.unwrap(); + let telemetry = agent.last_turn_telemetry(); + assert_eq!(telemetry.verify_rounds, 3, "stops at the third round"); + assert!(telemetry.verify_repeat_stopped); + assert!(telemetry.stalled_unfinished); + assert_eq!(agent.last_verify(), Some(false)); + assert!( + ui.statuses + .iter() + .any(|s| s.contains("failed identically 3 times")), + "explicit stop status: {:?}", + ui.statuses + ); + // The previous round's escalated nudge survives as context (its output is + // identical by definition — the signatures matched), and the transcript + // ends on the model's last attempt, clean for the next turn. + let nudge = agent + .messages() + .iter() + .rev() + .find(|m| m.role == Role::User && m.text().contains("Verification stage")) + .expect("previous verify nudge retained"); + assert!( + nudge.text().contains("SAME failure"), + "escalated nudge retained: {}", + nudge.text() + ); + assert_eq!( + agent.messages().last().map(|m| m.role), + Some(Role::Assistant), + "transcript ends on the model's final attempt" + ); +} + #[tokio::test] async fn different_failures_do_not_escalate() { let workspace = IsolatedWorkspace::new("verify-vary"); @@ -167,7 +227,9 @@ async fn different_failures_do_not_escalate() { c = counter.to_string_lossy() ), )]); - cfg.max_verify_repairs = 1; + // Three rounds of DIFFERENT failures: neither the escalation nor the + // stop-early path may fire when the signature keeps changing. + cfg.max_verify_repairs = 2; let tmp = workspace.path("changed.rs"); let p = tmp.to_string_lossy().to_string(); let responses = vec![ @@ -175,13 +237,15 @@ async fn different_failures_do_not_escalate() { completion(vec![Content::Text("attempt 1".into())], 1, 1), completion(vec![Content::Text("attempt 2".into())], 1, 1), completion(vec![Content::Text("attempt 3".into())], 1, 1), + completion(vec![Content::Text("attempt 4".into())], 1, 1), ]; let mut agent = agent(responses, cfg); agent.run_turn("x", &mut NullUi).await.unwrap(); let _ = std::fs::remove_file(&counter); assert_eq!(agent.last_verify(), Some(false)); - assert_eq!(agent.last_turn_telemetry().verify_rounds, 2); + assert_eq!(agent.last_turn_telemetry().verify_rounds, 3); assert_eq!(agent.last_turn_telemetry().repeated_verify_failures, 0); + assert!(!agent.last_turn_telemetry().verify_repeat_stopped); let nudge = agent .messages() .iter() diff --git a/crates/hi-agent/src/verify.rs b/crates/hi-agent/src/verify.rs index bd5586ff..a6dab853 100644 --- a/crates/hi-agent/src/verify.rs +++ b/crates/hi-agent/src/verify.rs @@ -169,6 +169,10 @@ pub(crate) struct Verifier { stage_mutation_counts: std::collections::BTreeMap, last_failure_signature: Option, repeated_failure_count: u32, + /// Consecutive rounds whose failure signature matched their predecessor. + /// 1 = the current failure repeats once (round N == N−1); 2 = three + /// identical failures in a row. Resets when the signature changes. + consecutive_repeats: u32, max_rounds: u32, round: u32, } @@ -185,6 +189,7 @@ impl Verifier { stage_mutation_counts: std::collections::BTreeMap::new(), last_failure_signature: None, repeated_failure_count: 0, + consecutive_repeats: 0, max_rounds, round: 0, } @@ -244,6 +249,12 @@ impl Verifier { self.repeated_failure_count } + /// Length of the current run of identical failures beyond the first + /// (1 = the newest failure matches its predecessor; 2 = three in a row). + pub(crate) fn consecutive_repeated_failures(&self) -> u32 { + self.consecutive_repeats + } + /// Record this round's failure signature and report whether it matches the /// previous round's — i.e. the repair attempt did not change the failure. /// The signature must be computed from output WITHOUT round-dependent @@ -260,6 +271,9 @@ impl Verifier { self.last_failure_signature = Some(signature); if repeated { self.repeated_failure_count = self.repeated_failure_count.saturating_add(1); + self.consecutive_repeats = self.consecutive_repeats.saturating_add(1); + } else { + self.consecutive_repeats = 0; } repeated } diff --git a/crates/hi-cli/src/main.rs b/crates/hi-cli/src/main.rs index ec93806b..17f034fa 100644 --- a/crates/hi-cli/src/main.rs +++ b/crates/hi-cli/src/main.rs @@ -1762,6 +1762,7 @@ fn write_report( "effective_max_steps": tel.effective_max_steps, "verify_rounds": tel.verify_rounds, "repeated_verify_failures": tel.repeated_verify_failures, + "verify_repeat_stopped": tel.verify_repeat_stopped, "recovery_retries": tel.recovery_retries, "repeat_nudges": tel.repeat_nudges, "continue_nudges": tel.continue_nudges, diff --git a/crates/hi-tui/src/tests.rs b/crates/hi-tui/src/tests.rs index ab488ac4..0454e439 100644 --- a/crates/hi-tui/src/tests.rs +++ b/crates/hi-tui/src/tests.rs @@ -1127,6 +1127,7 @@ fn ctrl_question_toggles_the_observability_panel() { effective_max_steps: 120, verify_rounds: 2, repeated_verify_failures: 0, + verify_repeat_stopped: false, recovery_retries: 1, repeat_nudges: 0, continue_nudges: 1,