From bc420718a18b9f3b01305b9d8c54ab1180100e25 Mon Sep 17 00:00:00 2001 From: haoc Date: Sat, 11 Jul 2026 00:08:49 +0800 Subject: [PATCH] feat: harden analysis trust and visual QA --- docs/VISUAL_QA_CAPTURE.md | 22 ++- package.json | 1 + scripts/capture_ui_gallery.mjs | 157 +++++++++++++++--- src/main/services/katago.ts | 5 +- src/renderer/src/App.tsx | 19 +-- src/renderer/src/features/board/GoBoardV2.tsx | 7 +- .../src/features/board/WinrateTimelineV2.tsx | 6 +- src/renderer/src/features/board/board-v2.css | 13 ++ .../src/features/board/boardGeometry.ts | 16 +- .../src/features/timeline/analysisTrust.ts | 84 ++++++++++ tests/analysis-trust-ui-contract.test.mjs | 100 +++++++++++ tests/katago-ranking-contract.test.mjs | 9 +- 12 files changed, 385 insertions(+), 54 deletions(-) create mode 100644 src/renderer/src/features/timeline/analysisTrust.ts create mode 100644 tests/analysis-trust-ui-contract.test.mjs diff --git a/docs/VISUAL_QA_CAPTURE.md b/docs/VISUAL_QA_CAPTURE.md index 7868f70..d31aecd 100644 --- a/docs/VISUAL_QA_CAPTURE.md +++ b/docs/VISUAL_QA_CAPTURE.md @@ -4,28 +4,32 @@ Sprint 7 adds an internal UI Gallery for repeatable visual review. It uses mock ## Open the Gallery -Start the app in development mode: +Build and open the gallery with the self-contained capture command: ```bash -pnpm dev +pnpm capture:ui-gallery ``` -Open either route: +The command builds the current renderer, serves it on an available localhost port, captures the required states with Playwright, and shuts the server down. It does not require KataGo, LLM credentials, or a separately running dev server. -```text -http://localhost:5173/#/ui-gallery -http://localhost:5173/?ui-gallery=1 +To inspect the gallery interactively instead, start the Vite renderer: + +```bash +pnpm dev:vite ``` +Then open `http://localhost:5173/#/ui-gallery`. + ## Capture Screenshots -If Playwright is available locally, run: +For a faster repeat capture after an unchanged build, run: ```bash -GOAGENT_UI_GALLERY_URL="http://localhost:5173/#/ui-gallery" \ -node scripts/capture_ui_gallery.mjs +pnpm capture:ui-gallery -- --skip-build ``` +Set `GOAGENT_BROWSER_PATH` when Chrome/Edge/Chromium is installed in a non-standard location. Set `GOAGENT_UI_GALLERY_URL` only when capturing an already-running renderer server. + The script writes screenshots to: ```text diff --git a/package.json b/package.json index a6bb121..d942aa4 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "website:dev": "pnpm --dir website dev", "website:check": "pnpm --dir website build", "check:website": "node scripts/check_website.mjs", + "capture:ui-gallery": "node scripts/capture_ui_gallery.mjs", "smoke:release-artifacts": "node scripts/release_artifact_smoke.mjs", "smoke:windows-packaged": "node scripts/smoke_windows_packaged_app.mjs", "check:teacher-quality": "pnpm build && pnpm eval:teacher && pnpm eval:claims && pnpm eval:quality-gate && pnpm check:knowledge-sources && pnpm eval:knowledge-coverage && pnpm eval:shape-recognition && pnpm eval:move-range && pnpm eval:vision-evidence && pnpm eval:katago-trace && pnpm eval:engine-silver && pnpm eval:teacher-style && pnpm eval:teacher-session && pnpm eval:tts-provider-policy && pnpm check:teacher-artifact", diff --git a/scripts/capture_ui_gallery.mjs b/scripts/capture_ui_gallery.mjs index 7f21361..20d4f72 100644 --- a/scripts/capture_ui_gallery.mjs +++ b/scripts/capture_ui_gallery.mjs @@ -1,10 +1,14 @@ #!/usr/bin/env node -import { mkdir } from 'node:fs/promises' -import { dirname, join, resolve } from 'node:path' -import { spawnSync } from 'node:child_process' +import { createServer } from 'node:http' +import { existsSync } from 'node:fs' +import { access, mkdir, readFile } from 'node:fs/promises' +import { dirname, extname, join, resolve, sep } from 'node:path' +import { spawn, spawnSync } from 'node:child_process' -const url = process.env.GOAGENT_UI_GALLERY_URL ?? 'http://localhost:5173/#/ui-gallery' +const configuredUrl = process.env.GOAGENT_UI_GALLERY_URL?.trim() +const rendererRoot = resolve('out/renderer') const outDir = resolve(process.env.GOAGENT_UI_GALLERY_OUT ?? 'release-evidence/ui-gallery') +const skipBuild = process.argv.includes('--skip-build') || process.env.GOAGENT_UI_GALLERY_SKIP_BUILD === '1' const captureTargets = [ ['board', '.ui-gallery__panel--board'], ['teacher-card', '.ui-gallery__panel--teacher'], @@ -14,6 +18,102 @@ const captureTargets = [ ['settings-readiness', '.beta-acceptance-panel'] ] +const mimeTypes = { + '.css': 'text/css; charset=utf-8', + '.html': 'text/html; charset=utf-8', + '.ico': 'image/x-icon', + '.jpeg': 'image/jpeg', + '.jpg': 'image/jpeg', + '.js': 'text/javascript; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.png': 'image/png', + '.svg': 'image/svg+xml', + '.webp': 'image/webp' +} + +function run(command, args, options = {}) { + return new Promise((resolvePromise, reject) => { + const child = spawn(command, args, { + stdio: 'inherit', + ...options + }) + child.once('error', reject) + child.once('close', (code) => { + if (code === 0) resolvePromise() + else reject(new Error(`${command} ${args.join(' ')} exited with ${code ?? 'unknown status'}`)) + }) + }) +} + +async function ensureRendererBuild() { + if (!skipBuild) { + await run('pnpm', ['build'], { cwd: process.cwd(), env: process.env }) + } + await access(join(rendererRoot, 'index.html')).catch(() => { + throw new Error('UI Gallery build is missing. Run `pnpm build` or omit `--skip-build`.') + }) +} + +function safeRendererPath(pathname) { + const relativePath = pathname === '/' ? 'index.html' : pathname.replace(/^\/+/, '') + const candidate = resolve(rendererRoot, relativePath) + return candidate === rendererRoot || candidate.startsWith(`${rendererRoot}${sep}`) ? candidate : null +} + +async function startRendererServer() { + await ensureRendererBuild() + const server = createServer(async (request, response) => { + try { + const pathname = decodeURIComponent(new URL(request.url ?? '/', 'http://127.0.0.1').pathname) + const requestedPath = safeRendererPath(pathname) + const fallbackPath = join(rendererRoot, 'index.html') + let filePath = requestedPath ?? fallbackPath + let body + try { + body = await readFile(filePath) + } catch { + filePath = fallbackPath + body = await readFile(filePath) + } + response.writeHead(200, { + 'Cache-Control': 'no-store', + 'Content-Type': mimeTypes[extname(filePath).toLowerCase()] ?? 'application/octet-stream' + }) + response.end(body) + } catch (error) { + response.writeHead(500, { 'Content-Type': 'text/plain; charset=utf-8' }) + response.end(String(error)) + } + }) + await new Promise((resolvePromise, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolvePromise) + }) + const address = server.address() + if (!address || typeof address === 'string') { + server.close() + throw new Error('Unable to start the local UI Gallery server.') + } + return { + server, + url: `http://127.0.0.1:${address.port}/#/ui-gallery` + } +} + +function systemBrowserPath() { + const candidates = [ + process.env.GOAGENT_BROWSER_PATH, + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge', + '/usr/bin/google-chrome', + '/usr/bin/chromium', + '/usr/bin/chromium-browser', + process.env.PROGRAMFILES ? join(process.env.PROGRAMFILES, 'Google', 'Chrome', 'Application', 'chrome.exe') : '', + process.env['PROGRAMFILES(X86)'] ? join(process.env['PROGRAMFILES(X86)'], 'Microsoft', 'Edge', 'Application', 'msedge.exe') : '' + ].filter(Boolean) + return candidates.find((candidate) => existsSync(candidate)) ?? '' +} + async function loadPlaywright() { try { return await import('playwright') @@ -22,7 +122,7 @@ async function loadPlaywright() { } } -async function captureWithCliFallback() { +async function captureWithCliFallback(url) { await mkdir(outDir, { recursive: true }) const whichResult = spawnSync('npx', ['--yes', '-p', 'playwright', 'which', 'playwright'], { encoding: 'utf8', @@ -30,7 +130,7 @@ async function captureWithCliFallback() { }) const playwrightBin = whichResult.stdout.trim() if (whichResult.status !== 0 || !playwrightBin) { - throw new Error('Playwright package is not installed and npx could not locate it. Run pnpm dev, then capture this route manually: ' + url) + throw new Error('Playwright package is unavailable and npx could not prepare it. Open this self-hosted route manually: ' + url) } const playwrightNodeModules = dirname(dirname(playwrightBin)) const fallbackScript = ` @@ -46,9 +146,11 @@ async function captureWithCliFallback() { const targets = ${JSON.stringify(captureTargets)} await mkdir(outDir, { recursive: true }) - const browser = await chromium.launch() + const browserPath = process.env.GOAGENT_BROWSER_PATH || undefined + const browser = await chromium.launch({ headless: true, executablePath: browserPath }) const page = await browser.newPage({ viewport: { width: 1440, height: 1100 }, deviceScaleFactor: 1 }) - await page.goto(url, { waitUntil: 'networkidle' }) + await page.goto(url, { waitUntil: 'domcontentloaded' }) + await page.locator('.ui-gallery').waitFor({ state: 'visible' }) await page.screenshot({ path: join(outDir, 'ui-gallery-overview.png'), fullPage: true }) for (const [name, selector] of targets) { const locator = page.locator(selector).first() @@ -56,13 +158,18 @@ async function captureWithCliFallback() { await locator.screenshot({ path: join(outDir, name + '.png') }) } } + const bindButton = page.getByRole('button', { name: '打开 SGF 绑定弹窗' }) + if (await bindButton.count()) { + await bindButton.click() + await page.locator('.student-dialog').screenshot({ path: join(outDir, 'student-bind-dialog.png') }) + } await browser.close() })().catch((error) => { console.error(error.message) process.exit(1) }) ` - const result = spawnSync('npx', [ + await run('npx', [ '--yes', '-p', 'playwright', @@ -74,27 +181,22 @@ async function captureWithCliFallback() { ...process.env, GOAGENT_CAPTURE_URL: url, GOAGENT_CAPTURE_OUT: outDir, + GOAGENT_BROWSER_PATH: systemBrowserPath(), PLAYWRIGHT_NODE_MODULES: playwrightNodeModules }, stdio: 'inherit' }) - if (result.status !== 0) { - throw new Error('Playwright package is not installed and npx Playwright capture failed. Run pnpm dev, then capture this route manually: ' + url) - } console.log(`Captured UI Gallery screenshots in ${outDir}`) } -async function capture() { - const playwright = await loadPlaywright() - if (!playwright) { - await captureWithCliFallback() - return - } +async function capturePage(playwright, url) { + const browserPath = systemBrowserPath() const { chromium } = playwright await mkdir(outDir, { recursive: true }) - const browser = await chromium.launch() + const browser = await chromium.launch({ headless: true, executablePath: browserPath || undefined }) const page = await browser.newPage({ viewport: { width: 1440, height: 1100 }, deviceScaleFactor: 1 }) - await page.goto(url, { waitUntil: 'networkidle' }) + await page.goto(url, { waitUntil: 'domcontentloaded' }) + await page.locator('.ui-gallery').waitFor({ state: 'visible' }) await page.screenshot({ path: join(outDir, 'ui-gallery-overview.png'), fullPage: true }) for (const [name, selector] of captureTargets) { @@ -114,6 +216,21 @@ async function capture() { console.log(`Captured UI Gallery screenshots in ${outDir}`) } +async function capture() { + const hosted = configuredUrl ? null : await startRendererServer() + const url = configuredUrl || hosted?.url + if (!url) throw new Error('Unable to resolve the UI Gallery URL.') + const playwright = await loadPlaywright() + try { + if (playwright) await capturePage(playwright, url) + else await captureWithCliFallback(url) + } finally { + if (hosted) { + await new Promise((resolvePromise) => hosted.server.close(resolvePromise)) + } + } +} + capture().catch((error) => { console.error(error.message) process.exit(1) diff --git a/src/main/services/katago.ts b/src/main/services/katago.ts index 1715ffd..e57334b 100644 --- a/src/main/services/katago.ts +++ b/src/main/services/katago.ts @@ -113,7 +113,7 @@ interface ActiveKataGoProcess { const QUICK_ANALYSIS_FAST_VISITS = 24 const QUICK_ANALYSIS_MAX_SWEEP_VISITS = 80 -const QUICK_ANALYSIS_REFINE_VISITS = 120 +const QUICK_ANALYSIS_REFINE_VISITS = 180 const QUICK_ANALYSIS_REFINE_TOP_N = 10 const QUICK_ANALYSIS_REFINE_MIN_LOSS = 4 const QUICK_ANALYSIS_WIDE_ROOT_NOISE = 0 @@ -1425,7 +1425,8 @@ export async function analyzeGameQuick( winrateLoss, scoreLoss }, - judgement: judgement(winrateLoss, scoreLoss) + judgement: judgement(winrateLoss, scoreLoss), + analysisQuality: buildAnalysisQuality(moveNumber, currentMove, beforeTopMoves, forcedActual) } analysis.moveClassification = classifyMoveAnalysis(analysis) analysis.pvConfidence = buildPvConfidenceReport(analysis) diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 6848a83..81818ba 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -64,6 +64,7 @@ import { KataGoAssetsPanel } from './features/settings/KataGoAssetsPanel' import { TeacherSpeechControls } from './features/tts/TeacherSpeechControls' import { TtsSettingsPanel } from './features/tts/TtsSettingsPanel' import { TeacherComposerPro } from './features/teacher/TeacherComposerPro' +import { analysisDisplaySeverity, isVerifiedTimelineIssue } from './features/timeline/analysisTrust' import { createUiTranslator, humanizeUiError, @@ -458,11 +459,11 @@ const LIVE_ANALYSIS_TIME_LIMIT_MS = 150_000 const LIVE_ANALYSIS_REPORT_INTERVAL_SECONDS = 0.2 const QUICK_GRAPH_FAST_VISITS = 24 const QUICK_GRAPH_FAST_VISITS_STRONG = 48 -const QUICK_GRAPH_REFINE_VISITS = 120 +const QUICK_GRAPH_REFINE_VISITS = 180 const QUICK_GRAPH_REFINE_TOP_N = 8 const LIBRARY_PAGE_SIZE = 10 const TIMELINE_ISSUE_MIN_LOSS = 1 -const ANALYSIS_CACHE_SCHEMA_VERSION = 'v5-komi-and-stable-quick-winrate' +const ANALYSIS_CACHE_SCHEMA_VERSION = 'v6-evidence-aware-issues' const ANALYSIS_CACHE_PREFIX = `goagent.analysisCache.${ANALYSIS_CACHE_SCHEMA_VERSION}.` const ANALYSIS_CACHE_INDEX_KEY = `goagent.analysisCache.${ANALYSIS_CACHE_SCHEMA_VERSION}.index` const ANALYSIS_CACHE_MAX_GAMES = 8 @@ -761,7 +762,7 @@ function timelineIssuesFromEvaluations( return [] } const loss = normalizeLossPercent(item.playedMove?.winrateLoss) - if (loss < TIMELINE_ISSUE_MIN_LOSS) { + if (loss < TIMELINE_ISSUE_MIN_LOSS || !isVerifiedTimelineIssue(item, TIMELINE_ISSUE_MIN_LOSS)) { return [] } const playedMove = item.playedMove?.move ?? item.currentMove?.gtp ?? record?.moves[item.moveNumber - 1]?.gtp ?? '' @@ -5659,17 +5660,7 @@ function formatSearchSpeed(visitsPerSecond: number): string { } function evaluationSeverity(item: KataGoMoveAnalysis): 'quiet' | 'inaccuracy' | 'mistake' | 'blunder' { - const winrateLoss = normalizeLossPercent(item.playedMove?.winrateLoss) - if (item.judgement === 'blunder' || winrateLoss >= 18) { - return 'blunder' - } - if (item.judgement === 'mistake' || winrateLoss >= 10) { - return 'mistake' - } - if (item.judgement === 'inaccuracy' || winrateLoss >= 4) { - return 'inaccuracy' - } - return 'quiet' + return analysisDisplaySeverity(item) } function formatIssueLoss(loss: number): string { diff --git a/src/renderer/src/features/board/GoBoardV2.tsx b/src/renderer/src/features/board/GoBoardV2.tsx index 5befc67..04c4a62 100644 --- a/src/renderer/src/features/board/GoBoardV2.tsx +++ b/src/renderer/src/features/board/GoBoardV2.tsx @@ -4,6 +4,7 @@ import type { GameRecord, KataGoMoveAnalysis } from '@main/lib/types' import type { UiTranslator } from '../../i18n' import { candidateVariationMoves, + candidatePerspectiveColor, getBoardSize, moveToColor, normalizeWinrate, @@ -169,14 +170,16 @@ function svgCursorPoint(event: BoardHoverEvent): { x: number; y: number } | null function CandidateMark({ candidate, boardSize, + perspectiveColor, active = false }: { candidate: RenderCandidate boardSize: number + perspectiveColor: 'B' | 'W' active?: boolean }): ReactElement { const p = xy(candidate, boardSize) - const className = `ks-candidate ks-candidate--${candidate.emphasis} ks-candidate--rank-${candidate.rank}${active ? ' ks-candidate--active' : ''}` + const className = `ks-candidate ks-candidate--${candidate.emphasis} ks-candidate--rank-${candidate.rank} ks-candidate-perspective--${perspectiveColor}${active ? ' ks-candidate--active' : ''}` const scale = candidate.emphasis === 'primary' ? 1.02 : candidate.emphasis === 'secondary' ? 0.96 : 0.9 const winrate = formatCandidateWinrate(candidate) const visits = formatCandidateVisits(candidate) @@ -460,6 +463,7 @@ export function GoBoardV2({ const boardSize = getBoardSize(record) const stones = useMemo(() => renderStones(record, moveNumber), [record, moveNumber]) const candidates = useMemo(() => renderCandidates(analysis, boardSize), [analysis, boardSize]) + const candidatePerspective = useMemo(() => candidatePerspectiveColor(analysis), [analysis]) const playedMove = useMemo(() => renderPlayedMove(analysis, boardSize), [analysis, boardSize]) const strandedStones = useMemo(() => territoryStrandedStones(stones, territoryJudgement), [stones, territoryJudgement]) const strandedStoneKeys = useMemo(() => new Set(strandedStones.map((stone) => `${stone.x},${stone.y}`)), [strandedStones]) @@ -708,6 +712,7 @@ export function GoBoardV2({ key={`${candidate.rank}-${candidate.label}`} candidate={candidate} boardSize={boardSize} + perspectiveColor={candidatePerspective} active={sameCandidate(activeCandidate?.renderCandidate, candidate)} /> ))} diff --git a/src/renderer/src/features/board/WinrateTimelineV2.tsx b/src/renderer/src/features/board/WinrateTimelineV2.tsx index 9312235..dca3478 100644 --- a/src/renderer/src/features/board/WinrateTimelineV2.tsx +++ b/src/renderer/src/features/board/WinrateTimelineV2.tsx @@ -2,8 +2,9 @@ import type { KeyboardEvent, PointerEvent as ReactPointerEvent, ReactElement, Re import { useMemo, useRef, useState } from 'react' import type { KataGoMoveAnalysis } from '@main/lib/types' import type { UiTranslator } from '../../i18n' -import { getAnalysisMoveNumber, getAnalysisWinrate, classifyMoveLoss, normalizeWinrate } from './boardGeometry' +import { getAnalysisMoveNumber, getAnalysisWinrate, normalizeWinrate } from './boardGeometry' import { moveFromTimelineSvgX } from './timelineInteraction' +import { analysisDisplaySeverity } from '../timeline/analysisTrust' import './board-v2.css' type Severity = 'blunder' | 'mistake' | 'inaccuracy' | 'turning-point' @@ -72,12 +73,13 @@ function buildPoints(evaluations: KataGoMoveAnalysis[], totalMoves: number): Tim return [] } const loss = extractLoss(item) + const severity = analysisDisplaySeverity(item) return [{ moveNumber, winrate, scoreLead: extractScoreLead(item), loss, - severity: loss === undefined ? undefined : classifyMoveLoss(loss) as Severity + severity: severity === 'quiet' ? undefined : severity as Severity }] }).filter((point) => point.moveNumber >= 0 && point.moveNumber <= totalMoves) .sort((a, b) => a.moveNumber - b.moveNumber) diff --git a/src/renderer/src/features/board/board-v2.css b/src/renderer/src/features/board/board-v2.css index d9c9dc5..5442348 100644 --- a/src/renderer/src/features/board/board-v2.css +++ b/src/renderer/src/features/board/board-v2.css @@ -395,6 +395,19 @@ font-weight: 620; } +.ks-candidate-perspective--W .ks-candidate-rank-badge { + fill: #f8f3e9; + stroke: rgba(41, 35, 28, 0.76); +} + +.ks-candidate-perspective--W .ks-candidate-rank { + fill: #27231e; +} + +.ks-candidate-perspective--B .ks-candidate-rank-badge { + fill: rgba(20, 22, 23, 0.94); +} + .ks-candidate-winrate { fill: var(--ks-candidate-ink, #071314); font-size: 12.8px; diff --git a/src/renderer/src/features/board/boardGeometry.ts b/src/renderer/src/features/board/boardGeometry.ts index 3e4845e..9a93ab9 100644 --- a/src/renderer/src/features/board/boardGeometry.ts +++ b/src/renderer/src/features/board/boardGeometry.ts @@ -184,6 +184,19 @@ export function oppositeColor(color: StoneColor): StoneColor { return color === 'B' ? 'W' : 'B' } +export function candidatePerspectiveColor(analysis: KataGoMoveAnalysis | null | undefined): StoneColor { + const trialNextColor = valueOf(valueOf(analysis, 'trialContext'), 'nextColor') + if (trialNextColor === 'B' || trialNextColor === 'W') { + return trialNextColor + } + const beforeMoves = Array.isArray(valueOf(valueOf(analysis, 'before'), 'topMoves')) + ? valueOf(valueOf(analysis, 'before'), 'topMoves') as unknown[] + : [] + const trialActive = valueOf(valueOf(analysis, 'trialContext'), 'active') === true + const currentColor = analysis?.currentMove ? moveToColor(analysis.currentMove) : 'B' + return beforeMoves.length > 0 && !trialActive ? currentColor : oppositeColor(currentColor) +} + export function candidateToPoint(candidate: KataGoCandidate | unknown, boardSize = 19): BoardPoint | null { return parseBoardPoint(candidate, boardSize) } @@ -341,8 +354,7 @@ export function renderCandidates(analysis: KataGoMoveAnalysis | null | undefined const trialActive = valueOf(valueOf(analysis, 'trialContext'), 'active') === true const isBeforePosition = beforeMoves.length > 0 && !trialActive const topMoves = isBeforePosition ? beforeMoves : afterMoves - const currentColor = analysis.currentMove ? moveToColor(analysis.currentMove) : 'B' - const displayColor = isBeforePosition ? currentColor : oppositeColor(currentColor) + const displayColor = candidatePerspectiveColor(analysis) return topMoves.slice(0, 6).flatMap((candidate, index) => { const point = candidateToPoint(candidate, boardSize) diff --git a/src/renderer/src/features/timeline/analysisTrust.ts b/src/renderer/src/features/timeline/analysisTrust.ts new file mode 100644 index 0000000..71338cb --- /dev/null +++ b/src/renderer/src/features/timeline/analysisTrust.ts @@ -0,0 +1,84 @@ +import type { AnalysisConfidence, KataGoMoveAnalysis, MoveClassificationSeverity } from '@main/lib/types' + +export type AnalysisDisplaySeverity = 'quiet' | 'inaccuracy' | 'mistake' | 'blunder' +export type AnalysisEvidenceState = 'verified' | 'provisional' | 'insufficient' + +export interface AnalysisTrustAssessment { + severity: AnalysisDisplaySeverity + confidence: AnalysisConfidence + evidenceState: AnalysisEvidenceState + shouldTeach: boolean + shouldDeepen: boolean + winrateLoss: number + scoreLoss: number +} + +function finiteMetric(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) ? Math.max(0, Math.abs(value)) : 0 +} + +function legacySeverity(analysis: KataGoMoveAnalysis): MoveClassificationSeverity { + if (analysis.judgement === 'good_move') return 'good' + if (analysis.judgement === 'unknown') return 'unclear' + return analysis.judgement +} + +function displaySeverity(severity: MoveClassificationSeverity): AnalysisDisplaySeverity { + if (severity === 'blunder' || severity === 'mistake' || severity === 'inaccuracy') return severity + return 'quiet' +} + +export function assessAnalysisTrust(analysis: KataGoMoveAnalysis): AnalysisTrustAssessment { + const classification = analysis.moveClassification + const severity = classification?.severity ?? legacySeverity(analysis) + const confidence = classification?.confidence ?? analysis.analysisQuality?.confidence ?? 'low' + const winrateLoss = finiteMetric(classification?.winrateLoss ?? analysis.playedMove?.winrateLoss) + const scoreLoss = finiteMetric(classification?.scoreLoss ?? analysis.playedMove?.scoreLoss) + const shouldTeach = classification?.shouldTeach ?? (severity !== 'good' && severity !== 'unclear') + const shouldDeepen = Boolean( + classification?.shouldDeepen || + analysis.analysisQuality?.deepenRecommended || + analysis.pvConfidence?.shouldDeepen + ) + const bestVisits = Math.max( + 0, + Number(analysis.analysisQuality?.bestVisits ?? analysis.before.topMoves[0]?.visits ?? 0) || 0 + ) + const actualVisits = Math.max( + 0, + Number(analysis.playedMove?.visits ?? analysis.analysisQuality?.actualVisits ?? 0) || 0 + ) + const actualSource = analysis.playedMove?.source + const hasDirectActualEvidence = actualSource === 'candidate' || actualSource === 'forced' || actualVisits >= 80 + const verified = Boolean( + analysis.currentMove && + analysis.playedMove && + hasDirectActualEvidence && + bestVisits >= 80 && + confidence !== 'low' + ) + const evidenceState: AnalysisEvidenceState = verified + ? 'verified' + : analysis.currentMove && analysis.playedMove + ? 'provisional' + : 'insufficient' + + return { + severity: verified && shouldTeach ? displaySeverity(severity) : 'quiet', + confidence, + evidenceState, + shouldTeach, + shouldDeepen, + winrateLoss, + scoreLoss + } +} + +export function analysisDisplaySeverity(analysis: KataGoMoveAnalysis): AnalysisDisplaySeverity { + return assessAnalysisTrust(analysis).severity +} + +export function isVerifiedTimelineIssue(analysis: KataGoMoveAnalysis, minimumLoss = 1): boolean { + const assessment = assessAnalysisTrust(analysis) + return assessment.evidenceState === 'verified' && assessment.shouldTeach && assessment.severity !== 'quiet' && assessment.winrateLoss >= minimumLoss +} diff --git a/tests/analysis-trust-ui-contract.test.mjs b/tests/analysis-trust-ui-contract.test.mjs new file mode 100644 index 0000000..970a817 --- /dev/null +++ b/tests/analysis-trust-ui-contract.test.mjs @@ -0,0 +1,100 @@ +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import test from 'node:test' +import ts from 'typescript' + +const root = new URL('..', import.meta.url) + +async function text(path) { + return readFile(new URL(path, root), 'utf8') +} + +async function importTypeScript(path) { + const source = await text(path) + const output = ts.transpileModule(source, { + compilerOptions: { + module: ts.ModuleKind.ESNext, + target: ts.ScriptTarget.ES2022 + }, + fileName: path + }).outputText + return import(`data:text/javascript;base64,${Buffer.from(output).toString('base64')}`) +} + +function analysisFixture(overrides = {}) { + return { + gameId: 'game-1', + moveNumber: 42, + boardSize: 19, + currentMove: { moveNumber: 42, color: 'W', point: 'dd', row: 3, col: 3, gtp: 'D16', pass: false }, + before: { winrate: 55, scoreLead: 1.5, topMoves: [{ move: 'Q4', winrate: 55, scoreLead: 1.5, visits: 120, order: 0, prior: 20, pv: [] }] }, + after: { winrate: 48, scoreLead: -0.4, topMoves: [] }, + playedMove: { move: 'D16', winrate: 48, scoreLead: -0.4, visits: 1200, source: 'forced', winrateLoss: 7, scoreLoss: 1.9 }, + judgement: 'mistake', + analysisQuality: { + phase: 'opening', totalVisits: 240, bestVisits: 120, actualVisits: 1200, + candidateSpreadWinrate: 2, candidateSpreadScore: 1, pvStable: true, + confidence: 'medium', reason: 'fixture', deepenRecommended: true + }, + moveClassification: { + severity: 'mistake', confidence: 'medium', phase: 'opening', winrateLoss: 7, scoreLoss: 1.9, + shouldTeach: true, shouldDeepen: true, reason: 'fixture', evidenceWarnings: [] + }, + ...overrides + } +} + +test('timeline issues require direct, non-low-confidence played-move evidence', async () => { + const trust = await importTypeScript('src/renderer/src/features/timeline/analysisTrust.ts') + const verified = analysisFixture() + assert.equal(trust.assessAnalysisTrust(verified).evidenceState, 'verified') + assert.equal(trust.analysisDisplaySeverity(verified), 'mistake') + assert.equal(trust.isVerifiedTimelineIssue(verified), true) + + const provisional = analysisFixture({ + playedMove: { move: 'D16', winrate: 48, scoreLead: -0.4, source: 'after-root', winrateLoss: 28, scoreLoss: 4.5 }, + analysisQuality: { ...analysisFixture().analysisQuality, bestVisits: 24, actualVisits: 0, confidence: 'low' }, + moveClassification: { ...analysisFixture().moveClassification, severity: 'blunder', confidence: 'low', winrateLoss: 28 } + }) + assert.equal(trust.assessAnalysisTrust(provisional).evidenceState, 'provisional') + assert.equal(trust.analysisDisplaySeverity(provisional), 'quiet') + assert.equal(trust.isVerifiedTimelineIssue(provisional), false) +}) + +test('candidate rank badge follows the side whose values are displayed', async () => { + const geometry = await importTypeScript('src/renderer/src/features/board/boardGeometry.ts') + const beforeWhite = analysisFixture() + assert.equal(geometry.candidatePerspectiveColor(beforeWhite), 'W') + assert.equal(geometry.candidatePerspectiveColor({ ...beforeWhite, before: { ...beforeWhite.before, topMoves: [] } }), 'B') + assert.equal(geometry.candidatePerspectiveColor({ ...beforeWhite, trialContext: { active: true, nextColor: 'B' } }), 'B') + + const board = await text('src/renderer/src/features/board/GoBoardV2.tsx') + const css = await text('src/renderer/src/features/board/board-v2.css') + assert.match(board, /candidatePerspectiveColor\(analysis\)/) + assert.match(board, /ks-candidate-perspective--\$\{perspectiveColor\}/) + assert.match(css, /\.ks-candidate-perspective--W \.ks-candidate-rank-badge/) +}) + +test('quick analysis and visible review UI share the evidence-aware classification contract', async () => { + const katago = await text('src/main/services/katago.ts') + const app = await text('src/renderer/src/App.tsx') + const timeline = await text('src/renderer/src/features/board/WinrateTimelineV2.tsx') + assert.match(katago, /analysisQuality: buildAnalysisQuality\(moveNumber, currentMove, beforeTopMoves, forcedActual\)/) + assert.match(katago, /QUICK_ANALYSIS_REFINE_VISITS = 180/) + assert.match(app, /isVerifiedTimelineIssue\(item, TIMELINE_ISSUE_MIN_LOSS\)/) + assert.match(app, /return analysisDisplaySeverity\(item\)/) + assert.match(app, /ANALYSIS_CACHE_SCHEMA_VERSION = 'v6-evidence-aware-issues'/) + assert.match(timeline, /analysisDisplaySeverity\(item\)/) +}) + +test('UI Gallery capture builds and self-hosts the current renderer', async () => { + const script = await text('scripts/capture_ui_gallery.mjs') + const docs = await text('docs/VISUAL_QA_CAPTURE.md') + const pkg = JSON.parse(await text('package.json')) + assert.equal(pkg.scripts['capture:ui-gallery'], 'node scripts/capture_ui_gallery.mjs') + assert.match(script, /startRendererServer/) + assert.match(script, /server\.listen\(0, '127\.0\.0\.1'/) + assert.match(script, /await run\('pnpm', \['build'\]/) + assert.match(script, /GOAGENT_BROWSER_PATH/) + assert.match(docs, /pnpm capture:ui-gallery/) +}) diff --git a/tests/katago-ranking-contract.test.mjs b/tests/katago-ranking-contract.test.mjs index 1474671..629e662 100644 --- a/tests/katago-ranking-contract.test.mjs +++ b/tests/katago-ranking-contract.test.mjs @@ -78,7 +78,8 @@ test('Board overlays display candidate values from the side-to-move perspective' assert.doesNotMatch(galleryMock, /winrate: 0\.56/) assert.match(geometry, /function playerWinrateValue/) assert.match(geometry, /function playerScoreValue/) - assert.match(geometry, /const displayColor = isBeforePosition \? currentColor : oppositeColor\(currentColor\)/) + assert.match(geometry, /export function candidatePerspectiveColor/) + assert.match(geometry, /const displayColor = candidatePerspectiveColor\(analysis\)/) assert.match(geometry, /const winrateValue = playerWinrateValue\(getCandidateWinrate\(candidate\), displayColor\)/) assert.match(geometry, /const scoreValue = playerScoreValue\(getCandidateScore\(candidate\), displayColor\)/) @@ -142,7 +143,7 @@ test('Quick winrate graph uses KaTrain-style fast visits and refines suspected m const katago = read('src/main/services/katago.ts') assert.match(katago, /QUICK_ANALYSIS_FAST_VISITS = 24/) assert.match(katago, /QUICK_ANALYSIS_MAX_SWEEP_VISITS = 80/) - assert.match(katago, /QUICK_ANALYSIS_REFINE_VISITS = 120/) + assert.match(katago, /QUICK_ANALYSIS_REFINE_VISITS = 180/) assert.match(katago, /QUICK_ANALYSIS_WIDE_ROOT_NOISE = 0/) assert.match(katago, /Math\.min\(QUICK_ANALYSIS_MAX_SWEEP_VISITS, Math\.round\(maxVisits\)\)/) assert.match(katago, /overrideSettings/) @@ -167,7 +168,7 @@ test('Quick winrate graph uses KaTrain-style fast visits and refines suspected m const app = read('src/renderer/src/App.tsx') assert.match(app, /QUICK_GRAPH_FAST_VISITS = 24/) - assert.match(app, /QUICK_GRAPH_REFINE_VISITS = 120/) + assert.match(app, /QUICK_GRAPH_REFINE_VISITS = 180/) assert.match(app, /quickGraphFastVisits/) assert.match(app, /refineVisits/) assert.doesNotMatch(app, /maxVisits: 12,/) @@ -180,7 +181,7 @@ test('Quick winrate graph uses KaTrain-style fast visits and refines suspected m test('Workbench reuses cache for automatic analysis but manual analysis refreshes the current move', () => { const app = read('src/renderer/src/App.tsx') assert.match(app, /ANALYSIS_CACHE_PREFIX/) - assert.match(app, /ANALYSIS_CACHE_SCHEMA_VERSION = 'v5-komi-and-stable-quick-winrate'/) + assert.match(app, /ANALYSIS_CACHE_SCHEMA_VERSION = 'v6-evidence-aware-issues'/) assert.match(app, /goagent\.analysisCache\.\$\{ANALYSIS_CACHE_SCHEMA_VERSION\}\./) assert.match(app, /function analysisQuality/) assert.match(app, /function preferAnalysis/)