Chat-Summery.py

# add_chat_summary_score.py
# Computes chat-based distress score C from item text (items 1–8), adds to your dataset, and saves to Excel.

import re
from pathlib import Path
from typing import Dict, Optional, List
import pandas as pd
import numpy as np

# ---------- PATHS (EDIT THESE) ----------
SRC_PATH = Path(r"E:\Academic 7th Sem\Final Year Project\RAG\Data_Anotation\Output\PHQ9_with_updated_levels.xlsx")
OUT_DIR  = Path(r"E:\Academic 7th Sem\Final Year Project\RAG\Data_Anotation\Output")
OUT_DIR.mkdir(parents=True, exist_ok=True)
OUT_PATH = OUT_DIR / "PHQ9_with_chat_summary.xlsx"

# ---------- MAP YOUR TEXT COLUMNS (EDIT THESE) ----------
# Put the exact column names that hold the free-text answers for each item.
# If an item doesn't exist as free text, set it to None or delete that line.
TEXT_COLS: Dict[int, Optional[str]] = {
    1: "PHQ1_Text",  # Interest/pleasure (EDIT)
    2: "PHQ2_Text",  # Feeling down (EDIT)
    3: "PHQ3_Text",  # Sleep (EDIT)
    4: "PHQ4_Text",  # Energy (EDIT)
    5: "PHQ5_Text",  # Appetite/weight (EDIT)
    6: "PHQ6_Text",  # Self-worth/guilt (EDIT)
    7: "PHQ7_Text",  # Concentration (EDIT)
    8: "PHQ8_Text",  # Movement (EDIT)
    9: "PHQ9_Text",  # Self-harm (excluded from C)
}

# ---------- RUBRIC LEXICONS (Tweak if your phrasing differs) ----------
# Frequency phrases -> suggested score tier
FREQ_3 = [
    "nearly every day", "every day", "always", "constantly", "all the time"
]
FREQ_2 = [
    "more than half", "most days", "often", "frequently", "many days"
]
FREQ_1 = [
    "several days", "sometimes", "occasionally", "some days", "a few days", "rarely"
]

# Protective / positive phrases indicating none/minimal
PROTECTIVE = [
    "no problem", "no issues", "as usual", "feel fine", "okay", "normal appetite",
    "sleeping well", "focus well", "energetic", "enjoy", "enjoying", "as normal"
]

# Negation tokens (if they appear within a short window before a symptom word)
NEGATIONS = ["no", "not", "without", "never", "hardly", "rarely"]

# Symptom keywords by item (add variants that match your respondents' language)
SYMPTOMS: Dict[int, List[str]] = {
    1: ["interest", "enjoy", "pleasure", "activities", "bored"],                            # anhedonia
    2: ["down", "depressed", "sad", "hopeless", "empty", "worthless"],                      # mood
    3: ["sleep", "insomnia", "wake", "awake", "oversleep", "nightmare", "nightmares"],      # sleep
    4: ["tired", "fatigue", "exhausted", "energy"],                                         # energy
    5: ["appetite", "eat", "eating", "weight", "hungry", "hunger"],                         # appetite/weight
    6: ["failure", "guilty", "guilt", "worthless", "burden", "let others down"],            # self-worth/guilt
    7: ["concentrate", "concentration", "focus", "attention", "distracted"],                # concentration
    8: ["move", "movement", "restless", "fidgety", "slow", "slowness", "agitation"],        # psychomotor
    9: ["suicide", "die", "death", "self-harm", "kill myself", "harm myself"],              # item 9 (excluded from C)
}

# ---------- HELPERS ----------
_word_split = re.compile(r"[^\w']+")

def _contains_any(text: str, phrases: List[str]) -> bool:
    t = text.lower()
    return any(p in t for p in phrases)

def _tokenize(text: str) -> List[str]:
    return [w for w in _word_split.split(text.lower()) if w]

def _negation_near_symptom(text: str, symptom: str, window: int = 3) -> bool:
    """
    Returns True if any negation token occurs within 'window' tokens before 'symptom'.
    """
    tokens = _tokenize(text)
    for i, tok in enumerate(tokens):
        if tok == symptom or symptom in tok:
            start = max(0, i - window)
            if any(tokens[j] in NEGATIONS for j in range(start, i)):
                return True
    return False

def score_item_text(item_num: int, text: Optional[str]) -> int:
    """
    Rule-based 0..3 score for a single item's free-text.
    - Uses frequency phrases for 1/2/3
    - Uses symptom presence to avoid false zeros
    - Protective cues & negations around symptoms push toward 0 only if there isn't stronger evidence
    - Mixed evidence -> choose the higher score (safety)
    """
    if not isinstance(text, str) or not text.strip():
        return 0  # treat missing/empty as 0 for C

    t = text.lower()

    candidates = []

    # Frequency-based cues
    if _contains_any(t, FREQ_3):
        candidates.append(3)
    if _contains_any(t, FREQ_2):
        candidates.append(2)
    if _contains_any(t, FREQ_1):
        candidates.append(1)

    # Symptom presence as a mild fallback if there isn't a frequency cue
    syms = SYMPTOMS.get(item_num, [])
    symptom_hit = any(s in t for s in syms)
    if symptom_hit and not any(s >= 1 for s in candidates):
        candidates.append(1)

    # Protective cues / negations near symptom -> offer a 0 candidate
    protective_hit = _contains_any(t, PROTECTIVE)
    negated_symptom = any(_negation_near_symptom(t, s) for s in syms)
    if protective_hit or negated_symptom:
        candidates.append(0)

    # Default 0 if no signals at all
    if not candidates:
        candidates.append(0)

    return max(candidates)  # choose the higher score if mixed

def compute_C_for_row(row: pd.Series) -> float:
    """
    Sum items 1..8 (each 0..3) and normalize by 24.
    Missing/empty -> 0 by default (as per your spec).
    """
    total = 0
    for i in range(1, 9):  # exclude item 9
        col = TEXT_COLS.get(i)
        val = row[col] if col in row and col is not None else None
        total += score_item_text(i, val)
    return total / 24.0

# ---------- LOAD ----------
df = pd.read_excel(SRC_PATH, engine="openpyxl")

# Sanity-check missing columns early
missing_cols = [c for i, c in TEXT_COLS.items() if i <= 9 and c and c not in df.columns]
if missing_cols:
    print("WARNING: These text columns were not found and will be treated as empty (score=0):")
    for c in missing_cols:
        print(" -", c)

# ---------- COMPUTE C ----------
df["Chat_Summary_0_1"] = df.apply(compute_C_for_row, axis=1).round(4)
df["Chat_Summary_Percent"] = (df["Chat_Summary_0_1"] * 100).round(2)

# ---------- SAVE ----------
with pd.ExcelWriter(OUT_PATH, engine="openpyxl") as writer:
    df.to_excel(writer, index=False, sheet_name="Updated")

print(f"✅ Wrote: {OUT_PATH}")
print(df[["Chat_Summary_0_1", "Chat_Summary_Percent"]].head(10).to_string(index=False))