diff --git a/backend/services/llm/router.py b/backend/services/llm/router.py index a07018c..3e04583 100644 --- a/backend/services/llm/router.py +++ b/backend/services/llm/router.py @@ -63,14 +63,27 @@ "required": ["transcriptScores", "feedback", "modelAnswer"], } -_SYSTEM_PROMPT = """You are an expert interview coach reviewing a candidate's \ -spoken answer to an interview question. +_VOICE_INSTRUCTIONS = """\ +VOICE AND TONE (required for every feedback string you write — summary, strengths, \ +improvements, deliveryNotes, and all session-level fields): +Write as the interviewer speaking directly to the person who just answered, like live \ +coaching in the moment — not a written evaluation about them. +- Use second person: "you" and "your". Address them directly. +- Never refer to them as "the candidate", "the interviewee", or in third person \ +(e.g. "they", "their answer", "the candidate's response"). +- Avoid impersonal report phrasing such as "your performance", "the response", \ +"overall execution", or "the answer demonstrated". Prefer concrete, conversational \ +coaching: "You opened strong when you…", "I'd tighten the middle by…", \ +"You sounded confident when…".""" + +_SYSTEM_PROMPT = f"""You are an expert interview coach. You are debriefing the person \ +who just answered — speak to them directly. You receive: - The interview question. -- The full transcript of the candidate's spoken answer. +- The full transcript of their spoken answer. - The transcript broken into timestamped segments. Each segment carries three \ -delivery signals derived from the candidate's voice, each in the range 0..1: +delivery signals derived from their voice, each in the range 0..1: - arousal: vocal energy / activation (low = flat/calm, high = energetic) - dominance: assertiveness / confidence in tone - valence: positivity / warmth of tone @@ -83,22 +96,23 @@ - structure: logical organization (e.g. STAR for behavioral answers) - relevance: how well it answers the question asked - conciseness: brevity without rambling or filler -2. feedback: - - summary: 2-3 sentence cohesive overview of how the candidate did overall, \ -covering both content and delivery. +2. feedback (all fields must follow VOICE AND TONE below): + - summary: 2-3 sentences on how they did overall, covering content and delivery. - strengths: 2-4 specific things that went well. When relevant, reference \ -specific moments in the answer (quote a short phrase or cite the approximate \ -timestamp) and tie delivery signals to those moments. +specific moments (quote a short phrase or cite the approximate timestamp) and tie \ +delivery signals to those moments. - improvements: 2-4 specific, actionable suggestions. Reference specific \ segments where delivery (e.g. low arousal sounding flat, low valence sounding \ cold) or content could improve. - deliveryNotes: a short paragraph on vocal delivery overall, grounded in \ the per-segment arousal/dominance/valence trends. 3. modelAnswer: a concise, strong example answer to the same question (a few \ -sentences) the candidate could learn from. +sentences, first person) they could learn from. + +{_VOICE_INSTRUCTIONS} Be specific and grounded in the provided transcript and segments. Do not invent \ -facts the candidate did not say. Output only the JSON object.""" +facts they did not say. Output only the JSON object.""" _SESSION_RESPONSE_SCHEMA = { "type": "object", @@ -139,8 +153,8 @@ ], } -_SESSION_SYSTEM_PROMPT = """You are an expert interview coach reviewing a \ -candidate's full mock interview session. +_SESSION_SYSTEM_PROMPT = f"""You are an expert interview coach. You are debriefing \ +the person who just finished a mock interview — speak to them directly. You receive multiple question-and-answer pairs. Each answer includes the \ question text, full transcript, and timestamped segments with delivery signals \ @@ -148,8 +162,8 @@ only, not clinical labels. Produce a single JSON object with: -1. overallSummary: 3-5 sentence overview of how the candidate performed across \ -the whole session (content and delivery trends). +1. overallSummary: 3-5 sentences on how they did across the whole session \ +(content and delivery trends). 2. overallStrengths: 3-5 session-level strengths. 3. overallImprovements: 3-5 session-level, actionable improvements. 4. overallDeliveryNotes: one paragraph on vocal delivery patterns across answers. @@ -158,7 +172,11 @@ - question: echo back the exact id and text from the input - transcriptScores: clarity/structure/relevance/conciseness (0..1) for that answer - feedback: summary, strengths, improvements, deliveryNotes for that answer - - modelAnswer: a strong example answer for that question + - modelAnswer: a strong example answer for that question (first person) + +All feedback text fields must follow VOICE AND TONE below. + +{_VOICE_INSTRUCTIONS} Be specific and grounded in the transcripts. Do not invent facts. Output only JSON.""" diff --git a/frontend/app/InterviewClient.tsx b/frontend/app/InterviewClient.tsx index d62f9aa..986330b 100644 --- a/frontend/app/InterviewClient.tsx +++ b/frontend/app/InterviewClient.tsx @@ -12,11 +12,18 @@ import { MicWaveform, type MicWaveformHandle } from "@/app/components/MicWavefor import { ScorecardPanel } from "@/app/scorecard/ScorecardPanel"; import { SessionScorecardPanel } from "@/app/scorecard/SessionScorecardPanel"; import { buildReviewContext } from "@/lib/interview-coach/sessionAdapter"; +import { + buildQualitativeFeedbackScript, + buildSessionFeedbackScript, +} from "@/lib/interview-coach/feedbackSpeech"; import type { FeedbackMode, + FullSessionFeedbackResponse, ReviewContextPayload, + SessionFeedbackResponse, SessionReviewPayload, } from "@/lib/interview-coach/types"; +import { speakWithGroq } from "@/lib/speech/tts"; import styles from "./InterviewClient.module.css"; const API_BASE = process.env.NEXT_PUBLIC_API_URL ?? "http://localhost:8000"; @@ -48,30 +55,6 @@ function buildIntro(interviewer: Interviewer): string { return `Hi there, welcome. I'm ${interviewer.name}, ${interviewer.title}. Thank you so much for coming in today — we're really glad to have you. Let's go ahead and get started.`; } -async function speakWithGroq(text: string, voice: string, signal: AbortSignal): Promise { - const res = await fetch(`${API_BASE}/speech/tts`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ text, voice }), - signal, - }); - if (!res.ok) throw new Error(`TTS ${res.status}`); - const blob = await res.blob(); - const url = URL.createObjectURL(blob); - return new Promise((resolve, reject) => { - const audio = new Audio(url); - audio.onended = () => { - URL.revokeObjectURL(url); - resolve(); - }; - audio.onerror = () => { - URL.revokeObjectURL(url); - reject(new Error("Audio playback failed")); - }; - audio.play(); - }); -} - export default function InterviewClient() { const [interviewer, setInterviewer] = useState(DEFAULT_INTERVIEWER); const [feedbackMode, setFeedbackMode] = useState("perQuestion"); @@ -87,6 +70,7 @@ export default function InterviewClient() { const [statusText, setStatusText] = useState( "Select your interviewer, feedback timing, and press Start Interview." ); + const [feedbackSpeaking, setFeedbackSpeaking] = useState(false); const mediaRecorderRef = useRef(null); const accumulatedRef = useRef([]); @@ -101,6 +85,9 @@ export default function InterviewClient() { const questionRef = useRef(null); const savedAnswersRef = useRef([]); const waveformRef = useRef(null); + const autoFeedbackShownRef = useRef(null); + const spokenFeedbackKeyRef = useRef(null); + const sessionFeedbackSpokenRef = useRef(false); useEffect(() => { questionRef.current = question; @@ -110,6 +97,14 @@ export default function InterviewClient() { savedAnswersRef.current = savedAnswers; }, [savedAnswers]); + useEffect(() => { + if (stage !== "done" || feedbackMode !== "perQuestion") return; + if (!question || segments.length === 0) return; + if (autoFeedbackShownRef.current === String(question.id)) return; + autoFeedbackShownRef.current = String(question.id); + setReviewContext(buildReviewContext(question, segments)); + }, [stage, feedbackMode, question, segments]); + const newAbort = useCallback((): AbortSignal => { abortRef.current?.abort(); const ctrl = new AbortController(); @@ -117,6 +112,51 @@ export default function InterviewClient() { return ctrl.signal; }, []); + const speakFeedbackScript = useCallback( + async (script: string, session = false) => { + const signal = newAbort(); + setFeedbackSpeaking(true); + setStatusText( + session + ? `${interviewer.name} is sharing your session feedback…` + : `${interviewer.name} is sharing feedback on your answer…` + ); + try { + await speakWithGroq(script, interviewer.voice, signal); + setStatusText( + session + ? "Interview complete. Review your session feedback below." + : "Answer received. Review your response or move to the next question." + ); + } catch (err) { + if (err instanceof DOMException && err.name === "AbortError") return; + setStatusText("Could not play feedback audio. You can still read it below."); + } finally { + setFeedbackSpeaking(false); + } + }, + [interviewer, newAbort] + ); + + const handlePerQuestionFeedbackReady = useCallback( + (response: SessionFeedbackResponse) => { + if (!question) return; + if (spokenFeedbackKeyRef.current === String(question.id)) return; + spokenFeedbackKeyRef.current = String(question.id); + void speakFeedbackScript(buildQualitativeFeedbackScript(response.feedback)); + }, + [question, speakFeedbackScript] + ); + + const handleSessionFeedbackReady = useCallback( + (response: FullSessionFeedbackResponse) => { + if (sessionFeedbackSpokenRef.current) return; + sessionFeedbackSpokenRef.current = true; + void speakFeedbackScript(buildSessionFeedbackScript(response), true); + }, + [speakFeedbackScript] + ); + const clearTimers = useCallback(() => { if (silenceTimerRef.current) { clearTimeout(silenceTimerRef.current); @@ -137,6 +177,9 @@ export default function InterviewClient() { clearTimers(); pendingEndRef.current = false; usedIdsRef.current.clear(); + autoFeedbackShownRef.current = null; + spokenFeedbackKeyRef.current = null; + sessionFeedbackSpokenRef.current = false; setQuestion(null); setQuestionNumber(0); setSegments([]); @@ -375,6 +418,7 @@ export default function InterviewClient() { }, [question, segments]); const dismissFeedback = useCallback(() => { + abortRef.current?.abort(); setReviewContext(null); }, []); @@ -384,7 +428,7 @@ export default function InterviewClient() { const dotClass = [ styles.dot, - stage === "playing" ? styles.playing : "", + stage === "playing" || feedbackSpeaking ? styles.playing : "", stage === "recording" ? styles.recording : "", stage === "processing" ? styles.processing : "", stage === "done" || stage === "finished" ? styles.done : "", @@ -507,7 +551,11 @@ export default function InterviewClient() { {stage === "done" && ( <> {hasMoreQuestions && ( - )} @@ -560,14 +608,21 @@ export default function InterviewClient() { {reviewContext && ( <>
- + )} {sessionPayload && stage === "finished" && ( <>
- + )}
diff --git a/frontend/app/scorecard/ScorecardPanel.tsx b/frontend/app/scorecard/ScorecardPanel.tsx index 60a29eb..739702f 100644 --- a/frontend/app/scorecard/ScorecardPanel.tsx +++ b/frontend/app/scorecard/ScorecardPanel.tsx @@ -25,6 +25,8 @@ export type ScorecardPanelProps = { loadingTranscriptScores?: boolean; loadingFeedback?: boolean; showShellBadge?: boolean; + /** Called once when feedback has been fetched or resolved from `result`. */ + onFeedbackReady?: (response: SessionFeedbackResponse) => void; }; async function fetchFeedback( @@ -52,6 +54,7 @@ export function ScorecardPanel({ loadingTranscriptScores = false, loadingFeedback = false, showShellBadge = false, + onFeedbackReady, }: ScorecardPanelProps) { const [fetched, setFetched] = useState(null); const [fetching, setFetching] = useState(false); @@ -90,6 +93,12 @@ export function ScorecardPanel({ const isLoadingFeedback = loadingFeedback || (shouldFetch && fetching); + useEffect(() => { + if (isLoadingFeedback || !feedback || !onFeedbackReady) return; + if (!transcriptScores || !modelAnswer) return; + onFeedbackReady({ transcriptScores, feedback, modelAnswer }); + }, [feedback, isLoadingFeedback, modelAnswer, onFeedbackReady, transcriptScores]); + return ( <> {error ? ( diff --git a/frontend/app/scorecard/SessionScorecardPanel.tsx b/frontend/app/scorecard/SessionScorecardPanel.tsx index 4bdab3a..16bfeaa 100644 --- a/frontend/app/scorecard/SessionScorecardPanel.tsx +++ b/frontend/app/scorecard/SessionScorecardPanel.tsx @@ -14,6 +14,8 @@ const API_BASE = process.env.NEXT_PUBLIC_API_URL ?? "http://localhost:8000"; export type SessionScorecardPanelProps = { sessionInput: SessionReviewPayload; + /** Called once when session feedback has been fetched. */ + onFeedbackReady?: (response: FullSessionFeedbackResponse) => void; }; async function fetchSessionFeedback( @@ -30,7 +32,10 @@ async function fetchSessionFeedback( return (await res.json()) as FullSessionFeedbackResponse; } -export function SessionScorecardPanel({ sessionInput }: SessionScorecardPanelProps) { +export function SessionScorecardPanel({ + sessionInput, + onFeedbackReady, +}: SessionScorecardPanelProps) { const [result, setResult] = useState(null); const [fetching, setFetching] = useState(false); const [error, setError] = useState(null); @@ -57,6 +62,10 @@ export function SessionScorecardPanel({ sessionInput }: SessionScorecardPanelPro return () => controller.abort(); }, [sessionInput]); + useEffect(() => { + if (result && onFeedbackReady) onFeedbackReady(result); + }, [onFeedbackReady, result]); + if (error) { return (

diff --git a/frontend/lib/interview-coach/feedbackSpeech.ts b/frontend/lib/interview-coach/feedbackSpeech.ts new file mode 100644 index 0000000..beb319b --- /dev/null +++ b/frontend/lib/interview-coach/feedbackSpeech.ts @@ -0,0 +1,44 @@ +import type { FullSessionFeedbackResponse, QualitativeFeedback } from "@/lib/interview-coach/types"; + +function joinList(items: string[]): string { + if (items.length === 0) return ""; + if (items.length === 1) return items[0]; + return `${items.slice(0, -1).join(". ")}. ${items.at(-1)}`; +} + +/** Turn per-question LLM feedback into a spoken script for TTS. */ +export function buildQualitativeFeedbackScript(feedback: QualitativeFeedback): string { + const parts = ["Here's my feedback on your answer.", feedback.summary]; + + if (feedback.strengths.length > 0) { + parts.push(`Your strengths: ${joinList(feedback.strengths)}.`); + } + if (feedback.improvements.length > 0) { + parts.push(`Areas to improve: ${joinList(feedback.improvements)}.`); + } + if (feedback.deliveryNotes.trim()) { + parts.push(`On delivery: ${feedback.deliveryNotes}`); + } + + return parts.join(" "); +} + +/** Turn end-of-interview session feedback into a spoken script for TTS. */ +export function buildSessionFeedbackScript(result: FullSessionFeedbackResponse): string { + const parts = [ + "Great work completing the interview. Here's my overall feedback.", + result.overallSummary, + ]; + + if (result.overallStrengths.length > 0) { + parts.push(`Overall strengths: ${joinList(result.overallStrengths)}.`); + } + if (result.overallImprovements.length > 0) { + parts.push(`Overall improvements: ${joinList(result.overallImprovements)}.`); + } + if (result.overallDeliveryNotes.trim()) { + parts.push(`On delivery: ${result.overallDeliveryNotes}`); + } + + return parts.join(" "); +} diff --git a/frontend/lib/speech/tts.ts b/frontend/lib/speech/tts.ts new file mode 100644 index 0000000..81373b4 --- /dev/null +++ b/frontend/lib/speech/tts.ts @@ -0,0 +1,104 @@ +const API_BASE = process.env.NEXT_PUBLIC_API_URL ?? "http://localhost:8000"; + +/** Groq Orpheus TTS input limit (https://console.groq.com/docs/text-to-speech/orpheus). */ +export const GROQ_TTS_MAX_CHARS = 200; + +/** Split long text into chunks that fit Groq's TTS character limit. */ +export function splitTextForTts(text: string, maxChars = GROQ_TTS_MAX_CHARS): string[] { + const trimmed = text.trim(); + if (!trimmed) return []; + if (trimmed.length <= maxChars) return [trimmed]; + + const chunks: string[] = []; + let remaining = trimmed; + + while (remaining.length > 0) { + if (remaining.length <= maxChars) { + chunks.push(remaining); + break; + } + + let sliceEnd = maxChars; + const window = remaining.slice(0, maxChars); + const sentenceBreak = Math.max( + window.lastIndexOf(". "), + window.lastIndexOf("! "), + window.lastIndexOf("? ") + ); + if (sentenceBreak >= maxChars * 0.35) { + sliceEnd = sentenceBreak + 1; + } else { + const spaceBreak = window.lastIndexOf(" "); + if (spaceBreak > 0) sliceEnd = spaceBreak; + } + + const chunk = remaining.slice(0, sliceEnd).trim(); + if (!chunk) { + chunks.push(remaining.slice(0, maxChars)); + remaining = remaining.slice(maxChars).trimStart(); + continue; + } + + chunks.push(chunk); + remaining = remaining.slice(sliceEnd).trimStart(); + } + + return chunks; +} + +async function playWavBlob(blob: Blob, signal: AbortSignal): Promise { + const url = URL.createObjectURL(blob); + return new Promise((resolve, reject) => { + const audio = new Audio(url); + const cleanup = () => { + signal.removeEventListener("abort", onAbort); + URL.revokeObjectURL(url); + }; + const onAbort = () => { + audio.pause(); + cleanup(); + reject(new DOMException("Aborted", "AbortError")); + }; + signal.addEventListener("abort", onAbort); + audio.onended = () => { + cleanup(); + resolve(); + }; + audio.onerror = () => { + cleanup(); + reject(new Error("Audio playback failed")); + }; + void audio.play().catch((err) => { + cleanup(); + reject(err); + }); + }); +} + +async function fetchTtsAudio(text: string, voice: string, signal: AbortSignal): Promise { + const res = await fetch(`${API_BASE}/speech/tts`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text, voice }), + signal, + }); + if (!res.ok) throw new Error(`TTS ${res.status}`); + return res.blob(); +} + +/** + * Fetch WAV audio from POST /speech/tts and play it to completion (or until aborted). + * Long text is split into ≤200-character chunks and played sequentially. + */ +export async function speakWithGroq( + text: string, + voice: string, + signal: AbortSignal +): Promise { + const chunks = splitTextForTts(text); + for (const chunk of chunks) { + if (signal.aborted) throw new DOMException("Aborted", "AbortError"); + const blob = await fetchTtsAudio(chunk, voice, signal); + await playWavBlob(blob, signal); + } +}