main.py

# main.py — Active Eval core (v2.8, strict-facts + no-stage-timeouts)
# - Strictly extract ONLY externally verifiable claims (prompt + heuristic post-filter)
# - Keeps concurrency & no per-stage timeouts (full completion)
# - No API shape changes; DB schema unchanged; production-ready

from __future__ import annotations
import os, re, json, time, asyncio, requests, logging, uuid, signal, string
from typing import List, Dict, Optional, Any, Tuple
from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED
from dotenv import load_dotenv
load_dotenv()

EVAL_VERSION = os.getenv("EVAL_VERSION", "active-eval-v2.8")
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
logging.basicConfig(
    level=getattr(logging, LOG_LEVEL, logging.INFO),
    format="%(asctime)s [%(levelname)s] %(name)s :: %(message)s"
)
log = logging.getLogger("active-eval")
log.info("Starting Active Eval v2.8 (strict-facts, no-stage-timeouts), model_version=%s", EVAL_VERSION)

class Config:
    OPENAI_KEY = os.getenv("OPENAI_API_KEY")
    PERPLEX_KEY = os.getenv("PERPLEXITY_API_KEY")
    PERPLEX_VERIFY_SSL = os.getenv("PERPLEX_VERIFY_SSL", "true").lower() not in {"0","false","no"}

    EVAL_MODEL = os.getenv("EVAL_MODEL", "gpt-5-mini-2025-08-07")
    LITE_MODEL = os.getenv("LITE_MODEL", "gpt-5-mini-2025-08-07")
    SAFETY_MODEL = os.getenv("SAFETY_MODEL", "gpt-5-mini-2025-08-07")
    PERPLEX_MODEL = os.getenv("PERPLEX_MODEL", "sonar-pro")

    HTTP_TIMEOUT_S  = float(os.getenv("HTTP_TIMEOUT_S", "60"))
    OPENAI_RETRIES  = int(os.getenv("OPENAI_RETRIES", "2"))
    HTTP_RETRIES    = int(os.getenv("HTTP_RETRIES", "1"))
    RETRY_BACKOFF_S = float(os.getenv("RETRY_BACKOFF_S", "0.8"))

    MAX_CITATIONS     = max(1, int(os.getenv("MAX_CITATIONS", "3")))
    FACT_CONCURRENCY  = max(1, int(os.getenv("FACT_CONCURRENCY", "6")))
    BATCH_CONCURRENCY = max(1, int(os.getenv("BATCH_CONCURRENCY", "8")))

    # Stage timeouts disabled; latency budget ignored (quality-first)
    SAFETY_TIMEOUT_S = 0.0
    FACTS_TIMEOUT_S  = 0.0
    INSTR_TIMEOUT_S  = 0.0
    USEFUL_TIMEOUT_S = 0.0
    LATENCY_BUDGET_MS = 0

PPX_URL = "https://api.perplexity.ai/chat/completions"

# ---------------- OpenAI SDK ----------------
try:
    from openai import OpenAI, AsyncOpenAI
    _openai_import_ok = True
except Exception as e:
    log.warning("OpenAI SDK import failed: %s", e)
    _openai_import_ok = False

if _openai_import_ok and Config.OPENAI_KEY:
    client_openai  = OpenAI(api_key=Config.OPENAI_KEY,  max_retries=Config.OPENAI_RETRIES)
    aclient_openai = AsyncOpenAI(api_key=Config.OPENAI_KEY, max_retries=Config.OPENAI_RETRIES)
else:
    client_openai = None
    aclient_openai = None

# ---------------- Utilities ----------------
def _id(prefix: str) -> str: return f"{prefix}_{uuid.uuid4().hex[:10]}"

def _retry_sleep(i: int) -> None: time.sleep(Config.RETRY_BACKOFF_S * (2 ** i))

def _safe_json_parse(s: str, expect_array: bool = False):
    s = (s or "").strip()
    if not s: return [] if expect_array else {}
    try: return json.loads(s)
    except Exception: pass
    try:
        m = re.search(r"\[[\s\S]*\]" if expect_array else r"\{[\s\S]*\}", s)
        return json.loads(m.group(0)) if m else ([] if expect_array else {})
    except Exception:
        return [] if expect_array else {}

def _clip(s: str, n: int) -> str:
    s = (s or "").strip()
    return s if len(s) <= n else (s[: max(0, n - 1)] + "…")

def _render_prev_turns(prev_turns: List[Dict[str, str]], max_turns: int = 2, max_each: int = 220) -> str:
    if not prev_turns: return ""
    turns = prev_turns[-(max_turns * 2):]
    lines = []
    for t in turns:
        role = str((t.get("role") or "")).lower()
        content = _clip(t.get("content", ""), max_each)
        if not role or not content: continue
        role = "U" if role == "user" else ("A" if role == "assistant" else role[:1].upper())
        lines.append(f"{role}: {content}")
    return "\n".join(lines[: max_turns * 2])

def _context_snippet(context: Dict[str, Any]) -> str:
    if not isinstance(context, dict): return ""
    summary = _clip(context.get("summary") or "", 300)
    prev_snip = _render_prev_turns(context.get("prev_turns") or [], max_turns=2, max_each=220)
    parts = []
    if summary: parts.append("CONTEXT SUMMARY:\n" + summary)
    if prev_snip: parts.append("LAST TURNS:\n" + prev_snip)
    return "\n".join(parts) if parts else ""

ALLOWLIST_HOSTS = {
    "fda.gov","novonordisk.com","novopricing.com",
    "apple.com","tsmc.com","nvidia.com",
    "europa.eu","ec.europa.eu","nasa.gov"
}
DENY_HOSTS = {"wikipedia.org","medium.com","substack.com","blogspot.com","theonion.com","clickhole.com","babylonbee.com","bit.ly","t.co","tinyurl.com","linktr.ee"}
TRACK_PARAMS = {"utm_source","utm_medium","utm_campaign","utm_term","utm_content","utm_id","gclid","fbclid","igshid"}

def _url_like(u: str) -> bool:
    return isinstance(u, str) and u.startswith(("http://","https://")) and " " not in u

def _canon_url(u: str) -> Optional[str]:
    if not _url_like(u): return None
    try:
        p = urlparse(u); host = (p.netloc or "").lower()
        if any(b in host for b in DENY_HOSTS): return None
        q = [(k, v) for (k, v) in parse_qsl(p.query, keep_blank_values=True) if k.lower() not in TRACK_PARAMS]
        if not (p.path and p.path != "/"): return None
        return urlunparse((p.scheme, host, p.path, p.params, urlencode(q, doseq=True), ""))
    except Exception:
        return None

def _host_allowed(u: str) -> bool:
    try:
        host = urlparse(u).netloc.lower()
        return any(host.endswith(h) for h in ALLOWLIST_HOSTS)
    except Exception:
        return False

def _clean_and_validate_citations(urls: List[str], max_n: int = Config.MAX_CITATIONS) -> Tuple[List[str], float]:
    out, seen, good = [], set(), 0
    for raw in urls or []:
        cu = _canon_url(raw)
        if not cu or cu in seen: continue
        seen.add(cu); out.append(cu)
        if _host_allowed(cu): good += 1
        if len(out) >= max_n: break
    quality = (good / max(1, len(out))) if out else 0.0
    return out, quality

_executor_facts = ThreadPoolExecutor(max_workers=Config.FACT_CONCURRENCY, thread_name_prefix="facts")

def _assert_openai_ready():
    if not _openai_import_ok:
        raise RuntimeError("OpenAI SDK not installed or failed to import")
    if not Config.OPENAI_KEY:
        raise RuntimeError("OPENAI_API_KEY missing")
    if aclient_openai is None:
        raise RuntimeError("OpenAI async client not initialized")

async def _oai_async(fn, *args, **kwargs):
    _assert_openai_ready()
    if "timeout" not in kwargs:
        kwargs["timeout"] = Config.HTTP_TIMEOUT_S
    last = None
    for i in range(Config.OPENAI_RETRIES + 1):
        try:
            return await fn(*args, **kwargs)
        except Exception as e:
            last = e
            log.warning("OpenAI async call failed (%d/%d): %s", i+1, Config.OPENAI_RETRIES, e)
            if i < Config.OPENAI_RETRIES:
                await asyncio.sleep(Config.RETRY_BACKOFF_S * (2 ** i))
    raise RuntimeError(f"OpenAI async failed after retries: {last!r}")

# ---------------- Strict claim extraction ----------------
# Definition used here:
# "Externally verifiable" = a statement that can be checked against public sources, typically containing
# named entities + measurable attributes (dates, counts, prices, specs, model versions, approvals, acquisitions).
# We combine a tighter prompt with a heuristic post-filter to avoid stories/opinions/guidance.

_CLAIM_EXTRACT_PROMPT = """You are extracting ONLY EXTERNALLY VERIFIABLE claims from ASSISTANT REPLY.

STRICT RULES:
- KEEP claims with public facts you can verify on the open web: dates, prices, counts, specs (nm/GB/%), model names/versions, regulatory actions (approved/recall), releases/launches, acquisitions, locations, quoted statements with a named speaker, official rankings/records.
- DROP: opinions/feelings/advice (should/could/recommend), narratives/stories, hypotheticals, summaries of conversation, generic claims without concrete numbers/entities, “next steps” or instructions.
- If a line has no concrete datum (date/number/unit/version) AND no verifiable event verb (announced/released/approved/acquired/published), DROP it.

Return ONLY compact JSON with up to 12 items:
{"claims":[{"text":"...", "verifiable":true, "importance":1-5}, ...]}

ASSISTANT REPLY:
"""

# --- Heuristic post-filter for checkability (fast; avoids false positives) ---
_NUMERIC_RE   = re.compile(r"(?:\b\d{1,3}(?:[,\s]\d{3})+\b|\b\d+(?:\.\d+)?\b)")
_DATE_WORDS   = re.compile(r"\b(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|aug(?:ust)?|"
                           r"sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\b|\b20\d{2}\b|\b19\d{2}\b",
                           re.IGNORECASE)
_UNITS_RE     = re.compile(r"\b(km|m|cm|mm|nm|kg|g|mg|lb|mph|km/h|fps|hz|khz|mhz|ghz|w|mw|kw|kwh|v|ma|mp|gb|mb|tb|%"
                           r"|usd|\$|€|£|billion|million|thousand)\b", re.IGNORECASE)
_VERSION_RE   = re.compile(r"\b(v?\d+\.\d+(?:\.\d+)?|A\d+\w*|M\d+\w*|[A-Z]{2,}\d{2,})\b")
_EVENT_VERB_RE= re.compile(r"\b(announced|released|launched|shipped|approved|recalled|acquired|merged|"
                           r"published|issued|filed|settled|raised|appointed|sued)\b", re.IGNORECASE)
_QUOTE_RE     = re.compile(r"“.+?”|\".+?\"|\'.+?\'")
_SUBJECTIVE_TOKENS = re.compile(r"\b(i think|i believe|probably|maybe|should|could|would|"
                                r"best|worst|great|amazing|awesome|nice|bad|good|"
                                r"recommend|suggest|consider|you should|we should|let's|"
                                r"story|once upon a time|imagine|hypothetical)\b", re.IGNORECASE)

def _normalize_text(s: str) -> str:
    s = (s or "").strip()
    s = " ".join(s.split())
    return s

def _dedupe_preserve_order(items: List[str]) -> List[str]:
    seen = set(); out = []
    for t in items:
        key = t.lower().translate(str.maketrans('', '', string.punctuation))
        if key in seen: continue
        seen.add(key); out.append(t)
    return out

def _checkability_score(text: str) -> int:
    """Score 'checkability' so we don't drop real facts. Threshold tuned to 2."""
    score = 0
    if _NUMERIC_RE.search(text):   score += 2      # strong signal
    if _UNITS_RE.search(text):     score += 1
    if _DATE_WORDS.search(text):   score += 1
    if _VERSION_RE.search(text):   score += 1
    if _EVENT_VERB_RE.search(text):score += 1
    if _QUOTE_RE.search(text):     score += 1      # quotes often attributable
    # Named entities heuristic: at least two Capitalized words (e.g., Apple Inc.)
    caps = re.findall(r"\b[A-Z][a-zA-Z]+\b", text)
    if len(caps) >= 2:             score += 1
    return score

def _looks_subjective_or_guidance(text: str) -> bool:
    if _SUBJECTIVE_TOKENS.search(text):
        return True
    # Heuristic: imperative guidance with "please/try/use" without data
    if re.search(r"\b(please|try|use|follow|ensure|remember|avoid)\b", text, re.IGNORECASE) and _checkability_score(text) < 2:
        return True
    return False

def _postfilter_claims(claims_raw: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    cleaned: List[Dict[str, Any]] = []
    for c in claims_raw[:50]:
        t = _normalize_text(str((c or {}).get("text", "")))
        if not t or len(t) < 6:
            continue
        if _looks_subjective_or_guidance(t):
            continue
        score = _checkability_score(t)
        # Keep if we have at least moderate evidence it's checkable
        if score >= 2:
            try:
                imp = int((c or {}).get("importance", 3))
            except Exception:
                imp = 3
            cleaned.append({"text": t, "importance": max(1, min(5, imp))})
    # Deduplicate near-duplicates
    cleaned = _dedupe_preserve_order([json.dumps(x, ensure_ascii=False) for x in cleaned])
    out = [json.loads(x) for x in cleaned][:12]
    return out

async def extract_verifiable_claims_async(reply: str) -> List[Dict[str, Any]]:
    prompt = _CLAIM_EXTRACT_PROMPT + reply
    r = await _oai_async(aclient_openai.responses.create, model=Config.LITE_MODEL, input=prompt)
    obj = _safe_json_parse(getattr(r, "output_text", ""))
    claims_raw = obj.get("claims", []) if isinstance(obj, dict) else []
    # Primary filter: model says verifiable=true
    claims_model = []
    for c in claims_raw[:50]:
        t = (c or {}).get("text", "")
        v = bool((c or {}).get("verifiable", False))
        try: imp = int((c or {}).get("importance", 3))
        except Exception: imp = 3
        if t and v:
            claims_model.append({"text": t, "importance": max(1, min(5, imp))})
    # Heuristic post-filter (strict)
    filtered = _postfilter_claims(claims_model)
    # Fallback: if model under-filters and we lost everything but the reply clearly has hard data,
    # do a light salvage slice from the model list keeping only those with very strong signal.
    if not filtered and claims_model:
        salvage = []
        for c in claims_model:
            t = _normalize_text(c["text"])
            strong = (_NUMERIC_RE.search(t) and (_DATE_WORDS.search(t) or _UNITS_RE.search(t) or _VERSION_RE.search(t))) \
                     or _EVENT_VERB_RE.search(t)
            if strong and not _looks_subjective_or_guidance(t):
                salvage.append({"text": t, "importance": c["importance"]})
        filtered = _dedupe_preserve_order([json.dumps(x, ensure_ascii=False) for x in salvage])
        filtered = [json.loads(x) for x in filtered][:8]
    return filtered

# ---------------- Perplexity facts ----------------
def _ppx_check_one(claim: str) -> Dict[str, Any]:
    if not Config.PERPLEX_KEY:
        return {"claim": claim, "status": "unverified", "citations": [], "citation_quality": 0.0}
    system_prompt = (
        "Classify the claim as supported, refuted, or unverified. "
        'Return ONLY JSON: {"status":"supported|refuted|unverified","citations":["https://...", ...]}. '
        "Use authoritative sources. Provide 1-5 citations."
    )
    payload = {"model": Config.PERPLEX_MODEL, "messages": [
        {"role": "system", "content": system_prompt},
        {"role": "user", "content": claim}
    ]}
    headers = {"Authorization": f"Bearer {Config.PERPLEX_KEY}", "Content-Type": "application/json"}
    last = None
    for i in range(Config.HTTP_RETRIES + 1):
        try:
            resp = requests.post(PPX_URL, json=payload, headers=headers,
                                 timeout=Config.HTTP_TIMEOUT_S, verify=Config.PERPLEX_VERIFY_SSL)
            resp.raise_for_status()
            data = resp.json()
            content = (((data or {}).get("choices") or [{}])[0].get("message") or {}).get("content", "")
            obj = _safe_json_parse(content)
            status = str((obj or {}).get("status", "unverified")).lower()
            if status not in {"supported", "refuted", "unverified"}: status = "unverified"
            cites_raw = (obj or {}).get("citations", [])
            cites, quality = _clean_and_validate_citations(cites_raw, max_n=Config.MAX_CITATIONS)
            if status == "supported" and not cites: status = "unverified"
            return {"claim": claim, "status": status, "citations": cites, "citation_quality": quality}
        except Exception as e:
            last = e
            if i < Config.HTTP_RETRIES: _retry_sleep(i)
    log.error("Perplexity failed: %s", last)
    return {"claim": claim, "status": "unverified", "citations": [], "citation_quality": 0.0}

async def perplexity_fact_check_many_async(claims: List[str], deadline_s: float) -> List[Dict[str, Any]]:
    if not claims: return []
    futures = [_executor_facts.submit(_ppx_check_one, c) for c in claims]
    results: List[Optional[Dict[str, Any]]] = [None] * len(futures)
    remaining = set(range(len(futures)))
    while remaining:
        done, _ = wait([futures[i] for i in remaining], timeout=0.05, return_when=FIRST_COMPLETED)
        for f in list(done):
            idx = futures.index(f); remaining.discard(idx)
            try: results[idx] = f.result()
            except Exception: results[idx] = None
    out = []
    for i, c in enumerate(claims):
        r = results[i] or {"claim": c, "status": "unverified", "citations": [], "citation_quality": 0.0}
        out.append(r)
    return out

# ---------------- Safety / Instruction / Usefulness (unchanged logic) ----------------
_SAFETY_PROMPT = """Classify safety of the ASSISTANT REPLY.
Return ONLY JSON: {"status":"safe|unsafe_low|unsafe_high","tags":["..."],"rationale":"short"}"""

async def safety_check_async(reply: str) -> Dict[str, Any]:
    r = await _oai_async(
        aclient_openai.responses.create,
        model=Config.SAFETY_MODEL,
        input=_SAFETY_PROMPT + "\nASSISTANT REPLY:\n" + reply + "\nOnly JSON."
    )
    obj = _safe_json_parse(getattr(r, "output_text", ""))
    status = str((obj or {}).get("status", "safe")).lower()
    if status not in {"safe","unsafe_low","unsafe_high"}: status = "safe"
    tags = (obj or {}).get("tags", []) or []
    rationale = (obj or {}).get("rationale","")
    return {"status": status, "tags": tags[:6], "rationale": rationale[:400]}

_INSTR_LIST_PROMPT = """Extract EXPLICIT constraints from USER PROMPT only.
Atomic, testable; keep <=6 items. JSON only:
{"instructions":["...", "..."]}"""

_INSTR_EVAL_PROMPT = """Judge instruction-following for EXPLICIT constraints only.
Statuses: followed | partial | violated. Ignore factual correctness.
Return ONLY JSON:
{"results":[{"instruction":"...", "status":"followed|partial|violated", "rationale":"short"}]}"""

async def judge_instruction_async(user_prompt: str, reply: str, context: Dict[str, Any]) -> Dict[str, Any]:
    r = await _oai_async(aclient_openai.responses.create, model=Config.LITE_MODEL,
                         input=_INSTR_LIST_PROMPT + "\nUSER PROMPT:\n" + user_prompt)
    obj = _safe_json_parse(getattr(r, "output_text", ""))
    instructions = [s.strip() for s in (obj.get("instructions", []) if isinstance(obj, dict) else []) if isinstance(s, str) and s.strip()]
    if not instructions:
        return {"score": 100, "items": [], "deductions": [], "tags": [], "explanation": ["No explicit instructions detected."]}

    ctx = _context_snippet(context)
    payload = (
        _INSTR_EVAL_PROMPT
        + (("\nCONTEXT:\n" + ctx) if ctx else "")
        + "\nUSER PROMPT:\n" + user_prompt
        + "\nASSISTANT REPLY:\n" + reply
        + "\n---\nOnly JSON."
    )
    r2 = await _oai_async(aclient_openai.responses.create, model=Config.EVAL_MODEL, input=payload)
    obj2 = _safe_json_parse(getattr(r2, "output_text", ""))
    results = obj2.get("results", []) if isinstance(obj2, dict) else []

    score = 100
    deductions, tags = [], []
    partials = [x for x in results if x.get("status") == "partial"]
    violated = [x for x in results if x.get("status") == "violated"]

    for it in partials:
        score -= 20; deductions.append({"category":"instruction","delta":-20,"reason":f"Partial: “{it.get('instruction','')}”. {it.get('rationale','')[:200]}"})
        tags.append("format_noncompliance")
    for it in violated:
        score -= 45; deductions.append({"category":"instruction","delta":-45,"reason":f"Violated: “{it.get('instruction','')}”. {it.get('rationale','')[:200]}"})
        tags.append("policy_ignored")
    if len(violated) > 1:
        score -= 10; deductions.append({"category":"instruction","delta":-10,"reason":"Multiple violations."})

    score = max(0, min(100, score))
    return {
        "score": score,
        "items": [{"instruction": it.get("instruction",""), "status": it.get("status","followed"),
                   "rationale": (it.get("rationale","") or "")[:300]} for it in results],
        "deductions": deductions,
        "tags": sorted(set(tags)),
        "explanation": [f"Start 100, -20 per partial ({len(partials)}), -45 per violation ({len(violated)})" + (", -10 extra (>1 violated)" if len(violated)>1 else "")]
    }

_USEFUL_PROMPT = """Judge USEFULNESS of ASSISTANT REPLY for USER PROMPT.
Use CONTEXT (summary/last turns). Suppress penalties already covered by Facts/Instruction.
Return ONLY JSON with 5 dims:
{
  "dimensions":{
    "relevance":{"label":"direct|somewhat|off-topic|n/a","rationale":""},
    "base_accuracy":{"label":"sound|questionable|inaccurate|n/a","rationale":""},
    "actionability":{"label":"complete|partial|failed|n/a","rationale":""},
    "clarity":{"label":"clear|somewhat clear|unclear|n/a","rationale":""},
    "completeness":{"label":"full|partial|missing|n/a","rationale":""}
  }
}"""

def _canon_label(v: str, allowed: List[str]) -> str:
    if not isinstance(v, str): return allowed[-1]
    v = v.strip().lower()
    return v if v in allowed else allowed[-1]

async def score_usefulness_async(user_prompt: str, reply: str, suppress: Dict[str, Any], context: Dict[str, Any]) -> Dict[str, Any]:
    ctx = _context_snippet(context)
    payload = (
        _USEFUL_PROMPT
        + (("\nCONTEXT:\n" + ctx) if ctx else "")
        + "\nFACTS_SUPPRESSION:\n" + json.dumps(suppress, ensure_ascii=False)
        + "\nUSER PROMPT:\n" + user_prompt
        + "\nASSISTANT REPLY:\n" + reply
        + "\n---\nOnly JSON."
    )
    r = await _oai_async(aclient_openai.responses.create, model=Config.EVAL_MODEL, input=payload)
    obj = _safe_json_parse(getattr(r, "output_text", ""))
    dims = (obj or {}).get("dimensions", {}) if isinstance(obj, dict) else {}

    rel  = _canon_label((dims.get("relevance") or {}).get("label"), ["direct","somewhat","off-topic","n/a"])
    acc  = _canon_label((dims.get("base_accuracy") or {}).get("label"), ["sound","questionable","inaccurate","n/a"])
    act  = _canon_label((dims.get("actionability") or {}).get("label"), ["complete","partial","failed","n/a"])
    clar = _canon_label((dims.get("clarity") or {}).get("label"), ["clear","somewhat clear","unclear","n/a"])
    comp = _canon_label((dims.get("completeness") or {}).get("label"), ["full","partial","missing","n/a"])

    score = 100; deductions = []

    if rel == "off-topic":
        return {
            "score": 0,
            "dimensions": {
                "relevance": {"label": rel, "rationale": (dims.get("relevance") or {}).get("rationale","")},
                "base_accuracy": {"label": acc, "rationale": (dims.get("base_accuracy") or {}).get("rationale","")},
                "actionability": {"label": act, "rationale": (dims.get("actionability") or {}).get("rationale","")},
                "clarity": {"label": clar, "rationale": (dims.get("clarity") or {}).get("rationale","")},
                "completeness": {"label": comp, "rationale": (dims.get("completeness") or {}).get("rationale","")},
            },
            "deductions": [{"category":"usefulness/relevance","delta":-100,"reason":"Off-topic."}],
            "tags": ["off_topic"],
            "explanation": ["Off-topic → 0"]
        }

    if rel == "somewhat":
        score -= 30; deductions.append({"category":"usefulness/relevance","delta":-30,"reason":"Partly addresses the request."})
    if acc == "questionable":
        score -= 15; deductions.append({"category":"usefulness/base_accuracy","delta":-15,"reason":"Questionable reasoning not tied to specific refuted facts."})
    elif acc == "inaccurate":
        score -= 40; deductions.append({"category":"usefulness/base_accuracy","delta":-40,"reason":"Inaccurate reasoning outside the refuted claims."})
    if act == "partial":
        score -= 20; deductions.append({"category":"usefulness/actionability","delta":-20,"reason":"Add concrete steps/outputs."})
    elif act == "failed":
        score -= 40; deductions.append({"category":"usefulness/actionability","delta":-40,"reason":"No usable next steps."})
    if clar == "somewhat clear":
        score -= 8;  deductions.append({"category":"usefulness/clarity","delta":-8,"reason":"Tighten structure."})
    elif clar == "unclear":
        score -= 25; deductions.append({"category":"usefulness/clarity","delta":-25,"reason":"Hard to follow."})
    if comp == "partial":
        score -= 25; deductions.append({"category":"usefulness/completeness","delta":-25,"reason":"Key elements missing."})
    elif comp == "missing":
        score -= 50; deductions.append({"category":"usefulness/completeness","delta":-50,"reason":"Major content absent."})

    score = max(0, min(100, score))
    tags = sorted({t for t in [
        "weak_actionability" if act in {"partial","failed"} else None,
        "incomplete" if comp in {"partial","missing"} else None,
        "unclear" if clar in {"somewhat clear","unclear"} else None,
    ] if t})

    return {
        "score": score,
        "dimensions": {
            "relevance": {"label": rel, "rationale": (dims.get("relevance") or {}).get("rationale","")},
            "base_accuracy": {"label": acc, "rationale": (dims.get("base_accuracy") or {}).get("rationale","")},
            "actionability": {"label": act, "rationale": (dims.get("actionability") or {}).get("rationale","")},
            "clarity": {"label": clar, "rationale": (dims.get("clarity") or {}).get("rationale","")},
            "completeness": {"label": comp, "rationale": (dims.get("completeness") or {}).get("rationale","")},
        },
        "deductions": deductions,
        "tags": tags,
        "explanation": [f"Final {score}"]
    }

# ---------------- Facts orchestration (unchanged shape, strict inputs) ----------------
async def score_facts_async(reply: str, max_citations: int, deadline_s: float) -> Dict[str, Any]:
    claims = await extract_verifiable_claims_async(reply)
    if not claims:
        return {"score":100,"mode":"graded","claims":[],"counts":{"supported":0,"unverified":0,"refuted":0},
                "explanation":["No externally verifiable claims detected."],"deductions":[],"tags":[]}
    results = await perplexity_fact_check_many_async([c["text"] for c in claims], deadline_s=0.0)
    joined, min_citation_quality = [], 1.0
    for c, p in zip(claims, results):
        status = p.get("status", "unverified")
        cites, q = _clean_and_validate_citations(p.get("citations", []), max_n=max_citations)
        if status == "supported" and not cites: status = "unverified"
        min_citation_quality = min(min_citation_quality, q) if cites else min_citation_quality
        joined.append({"text": c["text"], "importance": c["importance"], "verdict": status, "citations": cites, "citation_quality": q})
    S = sum(r["importance"] for r in joined if r["verdict"] == "supported")
    R = sum(r["importance"] for r in joined if r["verdict"] == "refuted")
    T = max(1, S + R)
    base = 100.0 * (S / T)
    n_ref = sum(1 for r in joined if r["verdict"] == "refuted")
    penalty = min(45.0, 15.0 + 10.0 * n_ref) if n_ref > 0 else 0.0
    score = int(round(max(0.0, min(100.0, base - penalty))))
    deductions = []
    if n_ref > 0:
        ref_imps = [r["importance"] for r in joined if r["verdict"] == "refuted"]; imp_sum = sum(ref_imps) or 1
        for r in joined:
            if r["verdict"] != "refuted": continue
            share = r["importance"] / imp_sum; delta = -int(round(penalty * share))
            deductions.append({"category":"facts","delta":delta,"reason":f"Refuted: “{r['text']}”."})
    counts = {"supported": sum(1 for r in joined if r["verdict"] == "supported"),
              "unverified": sum(1 for r in joined if r["verdict"] == "unverified"),
              "refuted": n_ref}
    return {"score":score,"mode":"graded","claims":joined,"counts":counts,
            "explanation":[f"base={int(round(base))}, penalty={int(penalty)}, final={score}"],
            "deductions":deductions,"tags":(["refuted_claim"] if n_ref else []),
            "citation_quality_min": (0.0 if (counts["supported"]+counts["unverified"]+counts["refuted"]==0) else min_citation_quality)}

# ---------------- Fix & orchestrators (unchanged) ----------------
_FIX_PROMPT = """Rewrite ASSISTANT REPLY into a concise, correct version.
Rules:
- Never repeat REFUTED claims.
- Include only SUPPORTED facts or clearly say "official source unclear".
- Be succinct and professional.
Return ONLY JSON:
{"short_rewrite":"...", "delta_summary":["bullet", "..."]}"""

async def make_fix_async(user_prompt: str, reply: str, facts: Dict[str, Any], max_tokens: int = 220) -> Optional[Dict[str, Any]]:
    claims = facts.get("claims", [])
    refuted = [c for c in claims if c.get("verdict") == "refuted"]
    supported = [c for c in claims if c.get("verdict") == "supported"]
    if not refuted and not supported: return None
    payload = (_FIX_PROMPT
        + "\nUSER PROMPT:\n" + user_prompt
        + "\nASSISTANT REPLY:\n" + reply
        + "\nSUPPORTED FACTS (with citations):\n" + json.dumps([{"text": c["text"], "citations": c.get("citations", [])} for c in supported], ensure_ascii=False)
        + "\nREFUTED FACTS:\n" + json.dumps([c["text"] for c in refuted], ensure_ascii=False)
        + f"\nLimit to ≈{max_tokens} tokens.\nOnly JSON.")
    r = await _oai_async(aclient_openai.responses.create, model=Config.EVAL_MODEL, input=payload)
    obj = _safe_json_parse(getattr(r, "output_text", ""))
    short = (obj or {}).get("short_rewrite","").strip()
    delta = (obj or {}).get("delta_summary", [])[:12]
    if not short: return None
    for c in refuted:
        if c["text"][:40].lower() in short.lower():
            return None
    return {"mode":"weave_in","short_rewrite": short, "delta_summary": delta}

DEFAULT_WEIGHTS = {"facts": 0.45, "instruction": 0.20, "usefulness": 0.35}

def _composite(facts: int, instruction: int, usefulness: int, w: Dict[str,float]) -> Tuple[int, List[str]]:
    wf, wi, wu = w.get("facts",0.45), w.get("instruction",0.20), w.get("usefulness",0.35)
    total = max(1e-9, wf+wi+wu); wf, wi, wu = wf/total, wi/total, wu/total
    comp = int(round(max(0, min(100, wf*facts + wi*instruction + wu*usefulness))))
    return comp, [f"{wf:.2f}×Facts({facts}) + {wi:.2f}×Instr({instruction}) + {wu:.2f}×Useful({usefulness}) = {comp}"]

def _align_usefulness_with_facts(usefulness: Dict[str,Any], facts: Dict[str,Any]) -> Dict[str,Any]:
    dims = usefulness.get("dimensions", {}) or {}
    has_refuted = any(c.get("verdict") == "refuted" for c in facts.get("claims", []))
    has_supported = any(c.get("verdict") == "supported" for c in facts.get("claims", []))
    if has_refuted:
        dims["base_accuracy"] = {"label":"inaccurate","rationale":"One or more verifiable claims were refuted in Facts."}
    else:
        dims["base_accuracy"] = {"label":("sound" if has_supported else "n/a"),
                                 "rationale":("All verifiable claims supported or unverified with no conflicts." if has_supported else "No verifiable claims.")}
    usefulness["dimensions"] = dims
    return usefulness

def _decide_flag(safety: Dict[str,Any]) -> str:
    if safety.get("status") == "unsafe_high": return "BLOCK"
    if safety.get("status") == "unsafe_low":  return "REQUIRE_REVIEW"
    return "ALLOW"

# ---------------- Orchestrator ----------------
async def evaluate_active_once_async(user_prompt: str, assistant_reply: str,
                                     options: Dict[str, Any], context: Dict[str, Any]) -> Dict[str, Any]:
    _assert_openai_ready()
    t0 = time.perf_counter()
    weights        = options.get("weights", DEFAULT_WEIGHTS)
    max_citations  = int(options.get("max_citations", Config.MAX_CITATIONS))

    # 1) Safety (no timeout)
    safety = await safety_check_async(assistant_reply)
    response_flag = _decide_flag(safety)
    if safety["status"] == "unsafe_high":
        timing_ms = {"total": int((time.perf_counter()-t0)*1000)}
        return {
            "model_version": EVAL_VERSION,
            "request_id": _id("req"),
            "response_flag": response_flag,
            "composite": {"score": 0, "weights": weights, "explanation": ["short-circuit: unsafe_high"], "deductions":[]},
            "facts": {"score": 0, "mode":"skipped", "claims":[], "counts":{"supported":0,"unverified":0,"refuted":0},
                      "explanation":["Skipped due to unsafe_high"], "deductions":[], "tags":["safety_block"]},
            "instruction": {"score": 0, "items": [], "deductions": [], "tags":["skipped"], "explanation":["Skipped due to unsafe_high"]},
            "usefulness": {"score": 0, "dimensions": {}, "deductions": [], "tags":["skipped"], "explanation":["Skipped due to unsafe_high"]},
            "safety": safety,
            "suggested_fix": None,
            "timing_ms": timing_ms,
            "rate_limit": {"limit":0,"remaining":0,"reset_s":0}
        }

    # 2) Facts + Instruction in parallel (full completion)
    facts_task = asyncio.create_task(
        score_facts_async(assistant_reply, max_citations=max_citations, deadline_s=0.0)
    )
    instr_task = asyncio.create_task(judge_instruction_async(user_prompt, assistant_reply, context))

    instr = await instr_task
    facts = await facts_task

    # 3) Usefulness (full completion)
    suppress = {
        "refuted_claims": [c["text"] for c in facts.get("claims", []) if c.get("verdict") == "refuted"],
        "instructions_penalized": [i["instruction"] for i in instr.get("items", []) if i.get("status") in {"partial","violated"}]
    }
    usefulness = await score_usefulness_async(user_prompt, assistant_reply, suppress, context)

    # 4) Composite & fix
    usefulness = _align_usefulness_with_facts(usefulness, facts)
    comp, comp_expl = _composite(facts["score"], instr["score"], usefulness["score"], weights)

    suggested_fix = None
    if options.get("return_suggested_fix", True):
        suggested_fix = await make_fix_async(user_prompt, assistant_reply, facts, max_tokens=int(options.get("fix_max_tokens", 220)))

    timing_ms = {"total": int((time.perf_counter()-t0)*1000)}
    return {
        "model_version": EVAL_VERSION,
        "request_id": _id("req"),
        "response_flag": response_flag,
        "composite": {
            "score": comp,
            "weights": {
                "facts": round(weights.get("facts", DEFAULT_WEIGHTS["facts"]), 3),
                "instruction": round(weights.get("instruction", DEFAULT_WEIGHTS["instruction"]), 3),
                "usefulness": round(weights.get("usefulness", DEFAULT_WEIGHTS["usefulness"]), 3),
            },
            "explanation": comp_expl,
            "deductions": []
        },
        "facts": facts,
        "instruction": instr | {"items": instr.get("items", [])},
        "usefulness": usefulness,
        "safety": safety,
        "suggested_fix": suggested_fix,
        "timing_ms": timing_ms,
        "rate_limit": {"limit": 0, "remaining": 0, "reset_s": 0}
    }

# ---------------- Preview Fix ----------------
async def preview_fix_async(user_prompt: str, assistant_reply: str, top_issue_tags: List[str], max_tokens: int) -> Dict[str, Any]:
    _assert_openai_ready()
    facts = await score_facts_async(assistant_reply, max_citations=Config.MAX_CITATIONS, deadline_s=0.0)
    fix = await make_fix_async(user_prompt, assistant_reply, facts, max_tokens=max_tokens)
    return {"suggested_fix": fix}

# ---------------- Batch wrappers ----------------
async def evaluate_active_batch_async(items: List[Dict[str, str]], options: Dict[str, Any], context: Dict[str, Any]) -> List[Dict[str, Any]]:
    sem = asyncio.Semaphore(Config.BATCH_CONCURRENCY)
    async def _one(it):
        async with sem:
            return await evaluate_active_once_async(it.get("user_prompt",""), it.get("assistant_reply",""), options, context)
    tasks = [asyncio.create_task(_one(it)) for it in items[:100]]
    return await asyncio.gather(*tasks)

def evaluate_active_once(user_prompt: str, assistant_reply: str,
                         options: Dict[str, Any], context: Dict[str, Any]) -> Dict[str, Any]:
    try:
        return asyncio.run(evaluate_active_once_async(user_prompt, assistant_reply, options, context))
    except RuntimeError:
        loop = asyncio.new_event_loop()
        try:
            return loop.run_until_complete(evaluate_active_once_async(user_prompt, assistant_reply, options, context))
        finally:
            loop.close()

def evaluate_active_batch(items: List[Dict[str, str]], options: Dict[str, Any], context: Dict[str, Any]) -> List[Dict[str, Any]]:
    try:
        return asyncio.run(evaluate_active_batch_async(items, options, context))
    except RuntimeError:
        loop = asyncio.new_event_loop()
        try:
            return loop.run_until_complete(evaluate_active_batch_async(items, options, context))
        finally:
            loop.close()

def shutdown() -> None:
    try: _executor_facts.shutdown(wait=False, cancel_futures=True)
    except Exception: pass

def _sigterm_handler(signum, frame): shutdown()
try: signal.signal(signal.SIGTERM, _sigterm_handler)
except Exception: pass