run_all_cases.sh

#!/bin/bash

# RTLMeter Debug Agent - Batch Run All Cases
# This script runs the debug agent on all cases from case_info_all.json
# with 15 iterations and 60 second timeout

set -e  # Exit on any error

# Configuration
MAX_ITERATIONS=15
TIMEOUT_SECONDS=60
PROJECT_ROOT="."
RESULTS_DIR="debug_results"
LOG_DIR="debug_logs"

# Create output directories
mkdir -p "$RESULTS_DIR"
mkdir -p "$LOG_DIR"

# Extract all case keys from case_info_all.json
echo "Extracting cases from benchmark/case_info_all.json..."
CASES=$(python3 -c "
import json
with open('benchmark/case_info_all.json', 'r') as f:
    data = json.load(f)
for case in data.keys():
    print(case)
")

# Count total cases
TOTAL_CASES=$(echo "$CASES" | wc -l)
echo "Found $TOTAL_CASES cases to process"

# Initialize counters
SUCCESS_COUNT=0
FAILURE_COUNT=0
CURRENT_CASE=0

# Function to run a single case
run_case() {
    local case="$1"
    local case_num="$2"
    local total="$3"
    
    echo ""
    echo "=========================================="
    echo "[$case_num/$total] Processing case: $case"
    echo "=========================================="
    
    # Create safe filename for logs
    local safe_case=$(echo "$case" | sed 's/[^a-zA-Z0-9]/_/g')
    local log_file="$LOG_DIR/${safe_case}.log"
    local result_file="$RESULTS_DIR/${safe_case}.txt"
    
    # Record start time
    local start_time=$(date '+%Y-%m-%d %H:%M:%S')
    echo "Start time: $start_time"
    
    # Run the debug agent
    echo "Running: python agent/debug_agent.py --case \"$case\" --max-iterations $MAX_ITERATIONS --timeout $TIMEOUT_SECONDS"
    
    if timeout 1800 python agent/debug_agent.py \
        --case "$case" \
        --max-iterations "$MAX_ITERATIONS" \
        --timeout "$TIMEOUT_SECONDS" \
        --project-root "$PROJECT_ROOT" \
        > "$log_file" 2>&1; then
        
        local end_time=$(date '+%Y-%m-%d %H:%M:%S')
        echo "✅ SUCCESS: Case $case completed successfully"
        echo "End time: $end_time"
        
        # Create success summary
        cat > "$result_file" << EOF
CASE: $case
STATUS: SUCCESS
START_TIME: $start_time
END_TIME: $end_time
MAX_ITERATIONS: $MAX_ITERATIONS
TIMEOUT: $TIMEOUT_SECONDS
LOG_FILE: $log_file
EOF
        
        SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
        
    else
        local end_time=$(date '+%Y-%m-%d %H:%M:%S')
        echo "❌ FAILED: Case $case failed or timed out"
        echo "End time: $end_time"
        
        # Create failure summary
        cat > "$result_file" << EOF
CASE: $case
STATUS: FAILED
START_TIME: $start_time
END_TIME: $end_time
MAX_ITERATIONS: $MAX_ITERATIONS
TIMEOUT: $TIMEOUT_SECONDS
LOG_FILE: $log_file
ERROR: Check log file for details
EOF
        
        FAILURE_COUNT=$((FAILURE_COUNT + 1))
    fi
    
    echo "Progress: $SUCCESS_COUNT successes, $FAILURE_COUNT failures out of $case_num cases processed"
}

# Main execution loop
echo ""
echo "Starting batch processing with configuration:"
echo "  Max iterations per case: $MAX_ITERATIONS"
echo "  Timeout per case: $TIMEOUT_SECONDS seconds"
echo "  Total timeout per case: 30 minutes (1800 seconds)"
echo "  Results directory: $RESULTS_DIR"
echo "  Logs directory: $LOG_DIR"
echo ""

# Process each case
while IFS= read -r case; do
    CURRENT_CASE=$((CURRENT_CASE + 1))
    run_case "$case" "$CURRENT_CASE" "$TOTAL_CASES"
done <<< "$CASES"

# Final summary
echo ""
echo "=========================================="
echo "BATCH PROCESSING COMPLETE"
echo "=========================================="
echo "Total cases processed: $TOTAL_CASES"
echo "Successful cases: $SUCCESS_COUNT"
echo "Failed cases: $FAILURE_COUNT"
echo "Success rate: $(( SUCCESS_COUNT * 100 / TOTAL_CASES ))%"
echo ""
echo "Results saved in: $RESULTS_DIR/"
echo "Logs saved in: $LOG_DIR/"

# Generate summary report
SUMMARY_FILE="$RESULTS_DIR/batch_summary.txt"
cat > "$SUMMARY_FILE" << EOF
RTLMeter Debug Agent - Batch Run Summary
Generated: $(date)

Configuration:
- Max iterations per case: $MAX_ITERATIONS
- Timeout per case: $TIMEOUT_SECONDS seconds
- Total cases: $TOTAL_CASES

Results:
- Successful cases: $SUCCESS_COUNT
- Failed cases: $FAILURE_COUNT
- Success rate: $(( SUCCESS_COUNT * 100 / TOTAL_CASES ))%

Successful Cases:
EOF

# List successful cases
for result_file in "$RESULTS_DIR"/*.txt; do
    if [ -f "$result_file" ] && grep -q "STATUS: SUCCESS" "$result_file"; then
        case_name=$(grep "CASE:" "$result_file" | cut -d' ' -f2-)
        echo "  ✅ $case_name" >> "$SUMMARY_FILE"
    fi
done

echo "" >> "$SUMMARY_FILE"
echo "Failed Cases:" >> "$SUMMARY_FILE"

# List failed cases
for result_file in "$RESULTS_DIR"/*.txt; do
    if [ -f "$result_file" ] && grep -q "STATUS: FAILED" "$result_file"; then
        case_name=$(grep "CASE:" "$result_file" | cut -d' ' -f2-)
        echo "  ❌ $case_name" >> "$SUMMARY_FILE"
    fi
done

echo ""
echo "Summary report saved to: $SUMMARY_FILE"
echo ""
echo "To view individual case results:"
echo "  ls $RESULTS_DIR/"
echo "To view individual case logs:"
echo "  ls $LOG_DIR/"