diff --git a/.gitignore b/.gitignore index 74c204b..9ec4a58 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,8 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json # local design and implementation plans docs/plans/ .sisyphus + +openspec +.codegraph +.opencode +AGENTS.md \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index 0735514..14dd8be 100644 --- a/src/config.ts +++ b/src/config.ts @@ -59,11 +59,23 @@ interface OpenCodeMemConfig { deduplicationEnabled?: boolean; deduplicationSimilarityThreshold?: number; userProfileAnalysisInterval?: number; - userProfileMaxPreferences?: number; - userProfileMaxPatterns?: number; - userProfileMaxWorkflows?: number; + userProfileDisplayPreferences?: number; + userProfileDisplayPatterns?: number; + userProfileDisplayWorkflows?: number; + userProfileStaleDays?: number; + userProfileInjectPreferences?: number; + userProfileInjectPatterns?: number; + userProfileInjectWorkflows?: number; userProfileConfidenceDecayDays?: number; userProfileChangelogRetentionCount?: number; + userProfileEmbeddingThresholdSameCat?: number; + userProfileEmbeddingThresholdSameCatWeak?: number; + userProfileEmbeddingThresholdCrossCat?: number; + userProfileEmbeddingThresholdCrossCatWeak?: number; + userProfileCentroidDriftThreshold?: number; + userProfileEmbeddingMinDescriptionLength?: number; + userProfileMinEvidenceForRetention?: number; + userProfileValidationEnabled?: boolean; showAutoCaptureToasts?: boolean; showUserProfileToasts?: boolean; showErrorToasts?: boolean; @@ -139,11 +151,23 @@ const DEFAULTS: Required< deduplicationEnabled: true, deduplicationSimilarityThreshold: 0.9, userProfileAnalysisInterval: 10, - userProfileMaxPreferences: 20, - userProfileMaxPatterns: 15, - userProfileMaxWorkflows: 10, + userProfileDisplayPreferences: 20, + userProfileDisplayPatterns: 15, + userProfileDisplayWorkflows: 10, + userProfileStaleDays: 2, + userProfileInjectPreferences: 5, + userProfileInjectPatterns: 5, + userProfileInjectWorkflows: 3, userProfileConfidenceDecayDays: 30, userProfileChangelogRetentionCount: 5, + userProfileEmbeddingThresholdSameCat: 0.8, + userProfileEmbeddingThresholdSameCatWeak: 0.5, + userProfileEmbeddingThresholdCrossCat: 0.9, + userProfileEmbeddingThresholdCrossCatWeak: 0.8, + userProfileCentroidDriftThreshold: 0.65, + userProfileEmbeddingMinDescriptionLength: 5, + userProfileMinEvidenceForRetention: 3, + userProfileValidationEnabled: false, showAutoCaptureToasts: true, showUserProfileToasts: true, showErrorToasts: true, @@ -394,18 +418,28 @@ const CONFIG_TEMPLATE = `{ // - User workflows (development habits, sequences, learning style) // - Skill level (overall and per-domain assessment) "userProfileAnalysisInterval": 10, + + // Days before inactive items (all types) are eligible for removal + "userProfileStaleDays": 2, + + // Number of preferences shown in UI + "userProfileDisplayPreferences": 20, + + // Number of patterns shown in UI + "userProfileDisplayPatterns": 15, + + // Number of workflows shown in UI + "userProfileDisplayWorkflows": 10, - // Maximum number of preferences to keep in user profile (sorted by confidence) - // Preferences are things like "prefers code without comments", "likes concise responses" - "userProfileMaxPreferences": 20, + // Number of preferences injected into LLM conversation context + // Keep this small — the strongest signals are enough; more dilute LLM attention + "userProfileInjectPreferences": 5, - // Maximum number of patterns to keep in user profile (sorted by frequency) - // Patterns are recurring topics like "often asks about database optimization" - "userProfileMaxPatterns": 15, + // Number of patterns injected into LLM conversation context + "userProfileInjectPatterns": 5, - // Maximum number of workflows to keep in user profile (sorted by frequency) - // Workflows are sequences like "usually asks for tests after implementation" - "userProfileMaxWorkflows": 10, + // Number of workflows injected into LLM conversation context + "userProfileInjectWorkflows": 3, // Days before preference confidence starts to decay (if not reinforced) // Preferences that aren't seen again will gradually lose confidence and be removed @@ -414,7 +448,16 @@ const CONFIG_TEMPLATE = `{ // Number of profile versions to keep in changelog (for rollback/debugging) // Older versions are automatically cleaned up "userProfileChangelogRetentionCount": 5, - + + // Minimum evidence count for a preference/pattern to survive confidence decay + // Items confirmed fewer times are more likely to be pruned when confidence decays + "userProfileMinEvidenceForRetention": 3, + + // Enable LLM validation of existing preferences against recent behavior. + // When enabled, each analysis round checks if top-5 preferences still match recent prompts. + // Experimental — disabled by default. + "userProfileValidationEnabled": false, + // ============================================ // Search Settings // ============================================ @@ -543,14 +586,44 @@ function buildConfig(fileConfig: OpenCodeMemConfig) { fileConfig.deduplicationSimilarityThreshold ?? DEFAULTS.deduplicationSimilarityThreshold, userProfileAnalysisInterval: fileConfig.userProfileAnalysisInterval ?? DEFAULTS.userProfileAnalysisInterval, - userProfileMaxPreferences: - fileConfig.userProfileMaxPreferences ?? DEFAULTS.userProfileMaxPreferences, - userProfileMaxPatterns: fileConfig.userProfileMaxPatterns ?? DEFAULTS.userProfileMaxPatterns, - userProfileMaxWorkflows: fileConfig.userProfileMaxWorkflows ?? DEFAULTS.userProfileMaxWorkflows, + userProfileDisplayPreferences: + fileConfig.userProfileDisplayPreferences ?? DEFAULTS.userProfileDisplayPreferences, + userProfileDisplayPatterns: + fileConfig.userProfileDisplayPatterns ?? DEFAULTS.userProfileDisplayPatterns, + userProfileDisplayWorkflows: + fileConfig.userProfileDisplayWorkflows ?? DEFAULTS.userProfileDisplayWorkflows, + userProfileInjectPreferences: + fileConfig.userProfileInjectPreferences ?? DEFAULTS.userProfileInjectPreferences, + userProfileInjectPatterns: + fileConfig.userProfileInjectPatterns ?? DEFAULTS.userProfileInjectPatterns, + userProfileInjectWorkflows: + fileConfig.userProfileInjectWorkflows ?? DEFAULTS.userProfileInjectWorkflows, userProfileConfidenceDecayDays: fileConfig.userProfileConfidenceDecayDays ?? DEFAULTS.userProfileConfidenceDecayDays, userProfileChangelogRetentionCount: fileConfig.userProfileChangelogRetentionCount ?? DEFAULTS.userProfileChangelogRetentionCount, + userProfileEmbeddingThresholdSameCat: + fileConfig.userProfileEmbeddingThresholdSameCat ?? + DEFAULTS.userProfileEmbeddingThresholdSameCat, + userProfileEmbeddingThresholdSameCatWeak: + fileConfig.userProfileEmbeddingThresholdSameCatWeak ?? + DEFAULTS.userProfileEmbeddingThresholdSameCatWeak, + userProfileEmbeddingThresholdCrossCat: + fileConfig.userProfileEmbeddingThresholdCrossCat ?? + DEFAULTS.userProfileEmbeddingThresholdCrossCat, + userProfileEmbeddingThresholdCrossCatWeak: + fileConfig.userProfileEmbeddingThresholdCrossCatWeak ?? + DEFAULTS.userProfileEmbeddingThresholdCrossCatWeak, + userProfileCentroidDriftThreshold: + fileConfig.userProfileCentroidDriftThreshold ?? DEFAULTS.userProfileCentroidDriftThreshold, + userProfileEmbeddingMinDescriptionLength: + fileConfig.userProfileEmbeddingMinDescriptionLength ?? + DEFAULTS.userProfileEmbeddingMinDescriptionLength, + userProfileMinEvidenceForRetention: + fileConfig.userProfileMinEvidenceForRetention ?? DEFAULTS.userProfileMinEvidenceForRetention, + userProfileValidationEnabled: + fileConfig.userProfileValidationEnabled ?? DEFAULTS.userProfileValidationEnabled, + userProfileStaleDays: fileConfig.userProfileStaleDays ?? DEFAULTS.userProfileStaleDays, showAutoCaptureToasts: fileConfig.showAutoCaptureToasts ?? DEFAULTS.showAutoCaptureToasts, showUserProfileToasts: fileConfig.showUserProfileToasts ?? DEFAULTS.showUserProfileToasts, showErrorToasts: fileConfig.showErrorToasts ?? DEFAULTS.showErrorToasts, diff --git a/src/index.ts b/src/index.ts index d1419f4..815768c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -51,6 +51,14 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { const providerResult = await ctx.client.provider.list(); if (providerResult.data?.connected) { setConnectedProviders(providerResult.data.connected); + log("opencode providers connected", { + list: providerResult.data.connected, + configured: CONFIG.opencodeProvider || "(not set)", + }); + } else { + log("opencode provider list empty or failed", { + data: JSON.stringify(providerResult.data).substring(0, 100), + }); } } catch (error) { log("Failed to initialize opencode provider state", { error: String(error) }); @@ -398,17 +406,23 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => { category: "explicit", description: sanitizedContent, confidence: 1.0, + frequency: 1, evidence: ["manual-write"], - lastUpdated: Date.now(), + lastSeen: Date.now(), }; const existingProfile = userProfileManager.getActiveProfile(userId); if (existingProfile) { const existingData = JSON.parse(existingProfile.profileData); - const mergedData = userProfileManager.mergeProfileData(existingData, { - preferences: [newPreference], - }); + const mergedData = await userProfileManager.mergeProfileData( + existingData, + { + preferences: [newPreference], + }, + undefined, + existingProfile.id + ); userProfileManager.updateProfile( existingProfile.id, mergedData, diff --git a/src/services/ai/profile-llm-client.ts b/src/services/ai/profile-llm-client.ts new file mode 100644 index 0000000..14b6d04 --- /dev/null +++ b/src/services/ai/profile-llm-client.ts @@ -0,0 +1,37 @@ +import type { OpencodeClient } from "@opencode-ai/sdk/v2/client"; +import { CONFIG } from "../../config.js"; + +let _cachedClient: OpencodeClient | null = null; +let _cachedProvider: string | null = null; +let _cachedModel: string | null = null; + +export async function getOpenCodeClient(): Promise { + const provider = CONFIG.opencodeProvider!; + const model = CONFIG.opencodeModel!; + + if (!provider || !model) { + throw new Error("opencode-mem: opencodeProvider and opencodeModel must be configured"); + } + + if (_cachedClient && _cachedProvider === provider && _cachedModel === model) { + return _cachedClient; + } + + const { isProviderConnected, getV2Client } = await import("./opencode-provider.js"); + + if (!isProviderConnected(provider)) { + throw new Error( + `opencode provider '${provider}' is not connected. Check your opencode provider configuration.` + ); + } + + const client = getV2Client(); + if (!client) { + throw new Error("opencode-mem: v2 client not initialized"); + } + + _cachedClient = client; + _cachedProvider = provider; + _cachedModel = model; + return client; +} diff --git a/src/services/ai/providers/openai-chat-completion.ts b/src/services/ai/providers/openai-chat-completion.ts index e2e6a9f..5f1b554 100644 --- a/src/services/ai/providers/openai-chat-completion.ts +++ b/src/services/ai/providers/openai-chat-completion.ts @@ -85,6 +85,15 @@ function extractFirstJSON(raw: string): string | null { return null; } +function repairInnerQuotes(json: string): string { + let result = json.replace( + /([\u4e00-\u9fff\u3000-\u303f\uff00-\uffef])"([\u4e00-\u9fff\u3000-\u303f\uff00-\uffef])/g, + '$1\\"$2' + ); + result = result.replace(/([\u4e00-\u9fff\u3000-\u303f\uff00-\uffef])"(?=\s*[,}\]])/g, '$1\\"'); + return result; +} + export class OpenAIChatCompletionProvider extends BaseAIProvider { private readonly aiSessionManager: AISessionManager; @@ -372,7 +381,18 @@ export class OpenAIChatCompletionProvider extends BaseAIProvider { return JSON.parse(raw); } catch (e1) { const fixed = extractFirstJSON(raw); - if (fixed) return JSON.parse(fixed); + if (fixed) { + try { + return JSON.parse(fixed); + } catch { + const repaired = repairInnerQuotes(fixed); + if (repaired !== fixed) { + try { + return JSON.parse(repaired); + } catch {} + } + } + } throw e1; } })(); diff --git a/src/services/api-handlers.ts b/src/services/api-handlers.ts index c26714c..55afc1f 100644 --- a/src/services/api-handlers.ts +++ b/src/services/api-handlers.ts @@ -6,6 +6,8 @@ import { log } from "./logger.js"; import { CONFIG } from "../config.js"; import type { MemoryType } from "../types/index.js"; import { userPromptManager } from "./user-prompt/user-prompt-manager.js"; +import type { UserProfileData } from "./user-profile/types.js"; +import { sortProfileItems } from "../utils/profile.js"; interface ApiResponse { success: boolean; @@ -933,6 +935,9 @@ export async function handleGetUserProfile(userId?: string): Promise c.id === changelogId); + const changelog = userProfileManager.getChangelogById(changelogId); if (!changelog) return { success: false, error: "Changelog not found" }; const profileData = JSON.parse(changelog.profileDataSnapshot); return { @@ -1011,7 +1015,15 @@ export async function handleRefreshProfile(userId?: string): Promise(); + +export async function handleAICleanup( + userId?: string, + includeIds?: string[] +): Promise> { + try { + const { userProfileManager } = await import("./user-profile/user-profile-manager.js"); + const { getTags } = await import("./tags.js"); + const { aiCleanupProfile, aiCleanupProfileFromIndexed, filterProfileForCleanup } = + await import("./user-profile/ai-cleanup.js"); + + let targetUserId = userId; + if (!targetUserId) { + const tags = getTags(process.cwd()); + targetUserId = tags.user.userEmail || "unknown"; + } + + const profile = userProfileManager.getActiveProfile(targetUserId); + if (!profile) { + return { success: false, error: "No profile found to clean up" }; + } + + const profileData: UserProfileData = JSON.parse(profile.profileData); + + let indexed; + let result; + if (includeIds && includeIds.length > 0) { + indexed = filterProfileForCleanup(profileData, includeIds); + result = await aiCleanupProfileFromIndexed(indexed); + } else { + result = await aiCleanupProfile(profileData); + } + + pendingCleanups.set(targetUserId, { + cleaned: result.cleaned, + oldProfileData: profileData, + diff: result.diff, + allMergedIds: (result.diff?.merged || []).map((m: any) => m.ids || []), + allRemovedIds: (result.diff?.removed || []).map((r: any) => r.id), + expiresAt: Date.now() + 30 * 60 * 1000, + }); + + return { + success: true, + data: { + old: profileData, + new: result.cleaned, + changes: result.diff, + }, + }; + } catch (error) { + log("handleAICleanup: error", { error: String(error) }); + return { success: false, error: String(error) }; + } +} + +export async function handleApplyCleanup(userId?: string, body?: any): Promise> { + try { + const { userProfileManager } = await import("./user-profile/user-profile-manager.js"); + const { getTags } = await import("./tags.js"); + + let targetUserId = userId; + if (!targetUserId) { + const tags = getTags(process.cwd()); + targetUserId = tags.user.userEmail || "unknown"; + } + + const pending = pendingCleanups.get(targetUserId); + if (!pending) { + return { success: false, error: "No pending cleanup found. Run AI cleanup first." }; + } + + if (Date.now() > pending.expiresAt) { + pendingCleanups.delete(targetUserId); + return { success: false, error: "Cleanup session expired. Run AI cleanup again." }; + } + + const profile = userProfileManager.getActiveProfile(targetUserId); + if (!profile) { + return { success: false, error: "Profile not found" }; + } + + const cleanedData = body?.profile || pending.cleaned; + const acceptedMerged: string[][] = body?.acceptedMerged || []; + const acceptedRemoved: string[] = body?.acceptedRemoved || []; + + // Partial application: start from cleaned data (which has shrunk descriptions) + // and only apply removals for items the user unchecked. + if (acceptedMerged.length > 0 || acceptedRemoved.length > 0) { + const existingData: UserProfileData = JSON.parse(profile.profileData); + const result: UserProfileData = { + preferences: [...cleanedData.preferences], + patterns: [...cleanedData.patterns], + workflows: [...cleanedData.workflows], + }; + + // Remove items the user chose NOT to merge (revert to old descriptions) + for (const id of acceptedRemoved) { + const desc = findItemDesc(pending.oldProfileData, id); + if (desc) removeByDesc(result, desc, itemTypeFromId(id)); + } + + // For merges: just remove the source items; target is already in cleaned + for (const ids of acceptedMerged) { + for (let i = 1; i < ids.length; i++) { + const srcDesc = findItemDesc(pending.oldProfileData, ids[i] ?? ""); + if (srcDesc) removeByDesc(result, srcDesc, itemTypeFromId(ids[i] ?? "")); + } + } + + // Restore source items from unapproved merges + const acceptedTargetIds = new Set(acceptedMerged.map((g) => g[0])); + for (const groupIds of pending.allMergedIds || []) { + if (groupIds.length <= 1) continue; + if (acceptedTargetIds.has(groupIds[0])) continue; + for (let i = 1; i < groupIds.length; i++) { + const srcId = groupIds[i] ?? ""; + if (!srcId) continue; + const srcDesc = findItemDesc(pending.oldProfileData, srcId); + if (!srcDesc) continue; + const srcItem = findItemByDesc(pending.oldProfileData, srcDesc); + if (srcItem) { + const { id: _id, ...rest } = srcItem as any; + if (srcId.startsWith("pref_")) result.preferences.push(rest); + else if (srcId.startsWith("pat_")) result.patterns.push(rest); + else if (srcId.startsWith("wf_")) result.workflows.push(rest); + } + } + } + + // Restore items from unapproved removals + const acceptedRemovedSet = new Set(acceptedRemoved); + for (const removedId of pending.allRemovedIds || []) { + if (acceptedRemovedSet.has(removedId)) continue; + const desc = findItemDesc(pending.oldProfileData, removedId); + if (!desc) continue; + const srcItem = findItemByDesc(pending.oldProfileData, desc); + if (srcItem) { + const { id: _id, ...rest } = srcItem as any; + if (removedId.startsWith("pref_")) result.preferences.push(rest); + else if (removedId.startsWith("pat_")) result.patterns.push(rest); + else if (removedId.startsWith("wf_")) result.workflows.push(rest); + } + } + + const success = userProfileManager.updateProfile( + profile.id, + result, + 0, + "AI cleanup applied (partial)" + ); + if (!success) + return { success: false, error: "Profile was modified by another session. Please retry." }; + pendingCleanups.delete(targetUserId); + return { + success: true, + data: { message: "Partial cleanup applied", version: profile.version + 1 }, + }; + } + + const success = userProfileManager.updateProfile( + profile.id, + cleanedData, + 0, + "AI cleanup applied" + ); + + if (!success) { + return { success: false, error: "Profile was modified by another session. Please retry." }; + } + + pendingCleanups.delete(targetUserId); + + return { + success: true, + data: { message: "Cleanup applied successfully", version: profile.version + 1 }, + }; + } catch (error) { + log("handleApplyCleanup: error", { error: String(error) }); + return { success: false, error: String(error) }; + } +} + +function itemTypeFromId(id: string): string { + if (id.startsWith("pref_")) return "preferences"; + if (id.startsWith("pat_")) return "patterns"; + return "workflows"; +} +function findItemDesc(profile: UserProfileData, id: string): string | null { + if (typeof id !== "string" || !id.includes("_")) return null; + const parts = id.split("_"); + const prefix = parts[0]; + const idx = parseInt(parts[1] || "", 10); + if (isNaN(idx)) return null; + + if (prefix === "pref") return profile.preferences[idx]?.description || null; + if (prefix === "pat") return profile.patterns[idx]?.description || null; + if (prefix === "wf") return profile.workflows[idx]?.description || null; + + return null; +} +function findItemByDesc(profile: UserProfileData, desc: string): any | null { + for (const key of ["preferences", "patterns", "workflows"] as const) { + const found = (profile as any)[key].find((p: any) => p.description === desc); + if (found) return found; + } + return null; +} +function removeByDesc(profile: UserProfileData, desc: string, itemType?: string) { + if (!itemType || itemType === "preferences") { + profile.preferences = profile.preferences.filter((p) => p.description !== desc); + } + if (!itemType || itemType === "patterns") { + profile.patterns = profile.patterns.filter((p) => p.description !== desc); + } + if (!itemType || itemType === "workflows") { + profile.workflows = profile.workflows.filter((w) => w.description !== desc); + } +} + +export async function handleUpdateProfileItem(body?: any): Promise> { + try { + const { userProfileManager } = await import("./user-profile/user-profile-manager.js"); + const { getTags } = await import("./tags.js"); + + const tags = getTags(process.cwd()); + const userId = tags.user.userEmail || "unknown"; + if (!userId) return { success: false, error: "Unable to resolve user identity" }; + + const profile = userProfileManager.getActiveProfile(userId); + if (!profile) return { success: false, error: "No profile found" }; + + const { type, index, action, category, description, steps } = body || {}; + if (!type || index === undefined || !action) { + return { success: false, error: "type, index, and action are required" }; + } + if (!["preferences", "patterns", "workflows"].includes(type)) { + return { success: false, error: "type must be preferences, patterns, or workflows" }; + } + if (!["edit", "delete"].includes(action)) { + return { success: false, error: "action must be edit or delete" }; + } + + const profileData: UserProfileData = JSON.parse(profile.profileData); + const items: any[] = (profileData as any)[type] || []; + // Re-sort to match handleGetUserProfile's display order + const metric = type === "preferences" ? "confidence" : "frequency"; + const sorted = sortProfileItems(items as any[], metric); + + if (index < 0 || index >= sorted.length) { + return { success: false, error: "index out of range" }; + } + + if (action === "delete") { + sorted.splice(index, 1); + } else { + const item = sorted[index]; + if (!item) return { success: false, error: "Item not found" }; + if (category !== undefined && type !== "workflows") item.category = category; + if (description !== undefined && description !== item.description) { + item.description = description; + item.centroid = undefined; + item.anchor = undefined; + } + if (steps !== undefined && Array.isArray(steps) && type === "workflows") item.steps = steps; + } + + (profileData as any)[type] = sorted; + + const changeSummary = + action === "delete" + ? `Deleted ${type.slice(0, -1)} at index ${index}` + : `Edited ${type.slice(0, -1)} at index ${index}`; + + const success = userProfileManager.updateProfile(profile.id, profileData, 0, changeSummary); + if (!success) + return { success: false, error: "Profile was modified by another session. Please retry." }; + + return { + success: true, + data: { message: `${action} successful`, version: profile.version + 1 }, + }; + } catch (error) { + log("handleUpdateProfileItem: error", { error: String(error) }); + return { success: false, error: String(error) }; + } +} + export async function handleDetectTagMigration(): Promise< ApiResponse<{ needsMigration: boolean; count: number }> > { diff --git a/src/services/auto-capture.ts b/src/services/auto-capture.ts index 647ef47..ae8d788 100644 --- a/src/services/auto-capture.ts +++ b/src/services/auto-capture.ts @@ -298,34 +298,35 @@ async function generateSummary( ): Promise<{ summary: string; type: string; tags: string[] } | null> { // Opencode provider path (when opencodeProvider + opencodeModel configured) if (CONFIG.opencodeProvider && CONFIG.opencodeModel) { - if (CONFIG.memoryModel) { - log("opencodeProvider takes precedence over memoryModel for auto-capture"); - } + try { + if (CONFIG.memoryModel) { + log("opencodeProvider takes precedence over memoryModel for auto-capture"); + } - const { isProviderConnected, getV2Client, generateStructuredOutput } = - await import("./ai/opencode-provider.js"); + const { isProviderConnected, getV2Client, generateStructuredOutput } = + await import("./ai/opencode-provider.js"); - if (!isProviderConnected(CONFIG.opencodeProvider)) { - throw new Error( - `opencode provider '${CONFIG.opencodeProvider}' is not connected. Check your opencode provider configuration.` - ); - } + if (!isProviderConnected(CONFIG.opencodeProvider)) { + throw new Error( + `opencode provider '${CONFIG.opencodeProvider}' is not connected. Check your opencode provider configuration.` + ); + } - const v2Client = getV2Client(); - if (!v2Client) { - throw new Error( - "opencode-mem: v2 client not initialized; cannot perform structured-output capture" - ); - } + const v2Client = getV2Client(); + if (!v2Client) { + throw new Error( + "opencode-mem: v2 client not initialized; cannot perform structured-output capture" + ); + } - const { detectLanguage, getLanguageName } = await import("./language-detector.js"); - const targetLang = - CONFIG.autoCaptureLanguage === "auto" || !CONFIG.autoCaptureLanguage - ? detectLanguage(userPrompt) - : CONFIG.autoCaptureLanguage; - const langName = getLanguageName(targetLang); + const { detectLanguage, getLanguageName } = await import("./language-detector.js"); + const targetLang = + CONFIG.autoCaptureLanguage === "auto" || !CONFIG.autoCaptureLanguage + ? detectLanguage(userPrompt) + : CONFIG.autoCaptureLanguage; + const langName = getLanguageName(targetLang); - const systemPrompt = `You are a technical memory recorder for a software development project. + const systemPrompt = `You are a technical memory recorder for a software development project. RULES: 1. ONLY capture technical work (code, bugs, features, architecture, config) @@ -345,31 +346,36 @@ FORMAT: SKIP if: greetings, casual chat, no code/decisions made CAPTURE if: code changed, bug fixed, feature added, decision made`; - const aiPrompt = `${context} + const aiPrompt = `${context} Analyze this conversation. If it contains technical work (code, bugs, features, decisions), create a concise summary and relevant tags. If it's non-technical (greetings, casual chat, incomplete requests), return type="skip" with empty summary.`; - const { z } = await import("zod"); - const schema = z.object({ - summary: z.string(), - type: z.string(), - tags: z.array(z.string()), - }); - - const result = await generateStructuredOutput({ - client: v2Client, - providerID: CONFIG.opencodeProvider, - modelID: CONFIG.opencodeModel, - systemPrompt, - userPrompt: aiPrompt, - schema, - }); - - return { - summary: result.summary, - type: result.type, - tags: (result.tags || []).map((t: string) => t.toLowerCase().trim()), - }; + const { z } = await import("zod"); + const schema = z.object({ + summary: z.string(), + type: z.string(), + tags: z.array(z.string()), + }); + + const result = await generateStructuredOutput({ + client: v2Client, + providerID: CONFIG.opencodeProvider, + modelID: CONFIG.opencodeModel, + systemPrompt, + userPrompt: aiPrompt, + schema, + }); + + return { + summary: result.summary, + type: result.type, + tags: (result.tags || []).map((t: string) => t.toLowerCase().trim()), + }; + } catch (e) { + log("auto-capture: opencode provider failed, falling back to external API", { + error: String(e), + }); + } } // Existing manual config path diff --git a/src/services/deduplication-service.ts b/src/services/deduplication-service.ts index 0544a99..e885786 100644 --- a/src/services/deduplication-service.ts +++ b/src/services/deduplication-service.ts @@ -3,6 +3,7 @@ import { vectorSearch } from "./sqlite/vector-search.js"; import { connectionManager } from "./sqlite/connection-manager.js"; import { CONFIG } from "../config.js"; import { log } from "./logger.js"; +import { cosineSimilarity } from "../utils/math.js"; interface DuplicateGroup { representative: { @@ -103,7 +104,7 @@ export class DeduplicationService { if (mem1.container_tag !== mem2.container_tag) continue; const vector2 = new Float32Array(new Uint8Array(mem2.vector).buffer); - const similarity = this.cosineSimilarity(vector1, vector2); + const similarity = cosineSimilarity(vector1, vector2); if (similarity >= CONFIG.deduplicationSimilarityThreshold && similarity < 1.0) { similarGroup.duplicates.push({ @@ -130,26 +131,6 @@ export class DeduplicationService { } } - private cosineSimilarity(a: Float32Array, b: Float32Array): number { - if (a.length !== b.length) return 0; - - let dotProduct = 0; - let normA = 0; - let normB = 0; - - for (let i = 0; i < a.length; i++) { - const aVal = a[i] || 0; - const bVal = b[i] || 0; - dotProduct += aVal * bVal; - normA += aVal * aVal; - normB += bVal * bVal; - } - - if (normA === 0 || normB === 0) return 0; - - return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); - } - getStatus() { return { enabled: CONFIG.deduplicationEnabled, diff --git a/src/services/embedding.ts b/src/services/embedding.ts index a68a1a1..f0d132e 100644 --- a/src/services/embedding.ts +++ b/src/services/embedding.ts @@ -100,6 +100,7 @@ export class EmbeddingService { progress_callback: progressCallback, }); this.isWarmedUp = true; + log("Embedding model warmed up", { model: CONFIG.embeddingModel }); } catch (error) { this.initPromise = null; log("Failed to initialize embedding model", { error: String(error) }); diff --git a/src/services/logger.ts b/src/services/logger.ts index c53437c..606f02a 100644 --- a/src/services/logger.ts +++ b/src/services/logger.ts @@ -6,6 +6,7 @@ import { statSync, renameSync, unlinkSync, + readdirSync, } from "fs"; import { homedir } from "os"; import { join } from "path"; @@ -21,19 +22,79 @@ function getLogDirPath(): string { } const MAX_LOG_SIZE = 5 * 1024 * 1024; +const MAX_LOG_DAYS = 30; const GLOBAL_LOGGER_KEY = Symbol.for("opencode-mem.logger.initialized"); +const LAST_ROTATE_DATE_KEY = Symbol.for("opencode-mem.logger.lastRotateDate"); + +function formatTimestamp(d: Date): string { + const pad = (n: number, w = 2) => String(n).padStart(w, "0"); + const offset = -d.getTimezoneOffset(); + const sign = offset >= 0 ? "+" : "-"; + const oh = pad(Math.floor(Math.abs(offset) / 60)); + const om = pad(Math.abs(offset) % 60); + return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}T${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}.${pad(d.getMilliseconds(), 3)}${sign}${oh}:${om}`; +} + +function formatDate(d: Date): string { + const pad = (n: number) => String(n).padStart(2, "0"); + return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`; +} + +function getDateStamp(): string { + return formatDate(new Date()); +} function rotateLog() { const logFile = getLogFilePath(); try { if (!existsSync(logFile)) return; + const stats = statSync(logFile); - if (stats.size < MAX_LOG_SIZE) return; + const dateStamp = getDateStamp(); + const logDir = getLogDirPath(); + + const needRotate = (() => { + if (stats.size >= MAX_LOG_SIZE) return true; + const lastModified = formatDate(stats.mtime); + return dateStamp !== lastModified; + })(); + + if (!needRotate) return; + + const archiveName = join(logDir, `opencode-mem-${getArchiveDate(stats)}.log`); + if (!existsSync(archiveName)) { + renameSync(logFile, archiveName); + } else { + const oldLog = logFile + ".old"; + if (existsSync(oldLog)) unlinkSync(oldLog); + renameSync(logFile, oldLog); + } - const oldLog = logFile + ".old"; - if (existsSync(oldLog)) unlinkSync(oldLog); - renameSync(logFile, oldLog); + cleanupOldLogs(); + } catch {} +} + +function getArchiveDate(stats: { mtime: Date }): string { + return formatDate(stats.mtime); +} + +function cleanupOldLogs() { + const logDir = getLogDirPath(); + const cutoff = Date.now() - MAX_LOG_DAYS * 24 * 60 * 60 * 1000; + try { + if (!existsSync(logDir)) return; + const files = readdirSync(logDir); + for (const file of files) { + const match = file.match(/^opencode-mem-(\d{4}-\d{2}-\d{2})\.log$/); + if (!match) continue; + const dateStr = match[1]!; + const [y, m, d] = dateStr.split("-").map(Number) as [number, number, number]; + const fileDate = new Date(y, m - 1, d); + if (fileDate.getTime() < cutoff) { + unlinkSync(join(logDir, file)); + } + } } catch {} } @@ -45,7 +106,7 @@ function ensureLoggerInitialized() { mkdirSync(logDir, { recursive: true }); } rotateLog(); - writeFileSync(logFile, `\n--- Session started: ${new Date().toISOString()} ---\n`, { + writeFileSync(logFile, `\n--- Session started: ${formatTimestamp(new Date())} ---\n`, { flag: "a", }); (globalThis as any)[GLOBAL_LOGGER_KEY] = true; @@ -53,8 +114,15 @@ function ensureLoggerInitialized() { export function log(message: string, data?: unknown) { ensureLoggerInitialized(); + + const today = formatDate(new Date()); + if ((globalThis as any)[LAST_ROTATE_DATE_KEY] !== today) { + (globalThis as any)[LAST_ROTATE_DATE_KEY] = today; + rotateLog(); + } + const logFile = getLogFilePath(); - const timestamp = new Date().toISOString(); + const timestamp = formatTimestamp(new Date()); const line = data ? `[${timestamp}] ${message}: ${JSON.stringify(data)}\n` : `[${timestamp}] ${message}\n`; diff --git a/src/services/user-memory-learning.ts b/src/services/user-memory-learning.ts index af35d1a..bd571c1 100644 --- a/src/services/user-memory-learning.ts +++ b/src/services/user-memory-learning.ts @@ -5,6 +5,7 @@ import { CONFIG } from "../config.js"; import { userPromptManager } from "./user-prompt/user-prompt-manager.js"; import type { UserPrompt } from "./user-prompt/user-prompt-manager.js"; import { userProfileManager } from "./user-profile/user-profile-manager.js"; +import { sortProfileItems } from "../utils/profile.js"; import type { UserProfile, UserProfileData } from "./user-profile/types.js"; let isLearningRunning = false; @@ -19,6 +20,8 @@ export async function performUserProfileLearning( const count = userPromptManager.countUnanalyzedForUserLearning(); const threshold = CONFIG.userProfileAnalysisInterval; + log("user-profile-learning: check", { count, threshold }); + if (count < threshold) { return; } @@ -33,43 +36,179 @@ export async function performUserProfileLearning( const userId = tags.user.userEmail || "unknown"; let existingProfile = userProfileManager.getActiveProfile(userId); - if (existingProfile && userProfileManager.applyConfidenceDecay(existingProfile.id)) { - existingProfile = userProfileManager.getActiveProfile(userId); + const analysisStartTime = Date.now(); + + let validationPrompt: string | undefined; + let validationPrefKeys: string[] | undefined; + if (existingProfile && CONFIG.userProfileValidationEnabled) { + const profileData: UserProfileData = JSON.parse(existingProfile.profileData); + const { data: decayed } = userProfileManager.decayInMemory(profileData); + + for (const arr of [decayed.preferences, decayed.patterns, decayed.workflows] as any[][]) { + for (const item of arr) { + if (item.pendingValidation && (item.lastSeen || 0) < analysisStartTime) { + item.pendingValidation = false; + item.alpha = (item.alpha || 1) + 1; + userProfileManager.syncConfidence(item); + } + } + } + + existingProfile = { ...existingProfile, profileData: JSON.stringify(decayed) }; + + const topPrefs = ( + sortProfileItems(decayed.preferences as any[], "confidence") as any[] + ).slice(0, 5); + const topPats = (sortProfileItems(decayed.patterns as any[], "frequency") as any[]).slice( + 0, + 3 + ); + const hasValidator = topPrefs.length >= 5; + if (hasValidator) { + const allValidated = [...topPrefs, ...topPats]; + validationPrefKeys = allValidated.map( + (p: any) => `${p.category || "_"}||${(p.description || "").substring(0, 30)}` + ); + log("user-profile-learning: validation enabled", { + topPrefs: topPrefs.map( + (p: any) => `[${p.category}] ${(p.description || "").substring(0, 30)}` + ), + topPats: topPats.map( + (p: any) => `[${p.category}] ${(p.description || "").substring(0, 30)}` + ), + }); + validationPrompt = `## Task 3: Validate Existing Profile Entries + +CRITICAL: Complete Tasks 1-2 (new observations) FIRST. This task is separate — only check whether the entries below still match recent behavior. Do NOT let these descriptions influence your new observations. + +${allValidated.map((p: any, i: number) => `${i}. [${p.category || "_"}] ${(p.description || "").substring(0, 30)} (conf: ${Math.round((p.confidence || 0) * 100) / 100})`).join("\n")} + +For each entry above, judge whether recent prompts confirm or contradict it. Output: +{"validations": [{"index": 0, "verdict": "confirmed|contradicted|no_evidence|inaccurate|oversimplified", "reason": "one sentence"}]} + +Rules: +- confirmed: recent prompts show clear evidence +- contradicted: recent prompts show the user has changed +- inaccurate: the description is directionally wrong (opposite behavior seen) +- oversimplified: the description is too vague, missing important nuance +- no_evidence: recent prompts don't address this topic +- Only mark contradicted if there is explicit evidence the user's behavior has changed`; + } else { + log("user-profile-learning: validation skipped", { + prefCount: topPrefs.length, + reason: "needs ≥ 5 preferences", + }); + } + } else if (existingProfile) { + const profileData: UserProfileData = JSON.parse(existingProfile.profileData); + const { data: decayed } = userProfileManager.decayInMemory(profileData); + + for (const arr of [decayed.preferences, decayed.patterns, decayed.workflows] as any[][]) { + for (const item of arr) { + if (item.pendingValidation && (item.lastSeen || 0) < analysisStartTime) { + item.pendingValidation = false; + item.alpha = (item.alpha || 1) + 1; + userProfileManager.syncConfidence(item); + } + } + } + + existingProfile = { ...existingProfile, profileData: JSON.stringify(decayed) }; } - const context = buildUserAnalysisContext(prompts, existingProfile); + const context = buildUserAnalysisContext(prompts, existingProfile, validationPrompt); + + const analysisResult = await analyzeUserProfile(context, existingProfile); - const updatedProfileData = await analyzeUserProfile(context, existingProfile); + log("user-profile-learning: analyze done", { hasResult: !!analysisResult }); - if (!updatedProfileData) { + if (!analysisResult) { userPromptManager.markMultipleAsUserLearningCaptured(prompts.map((p) => p.id)); + if (prompts.length >= 10 && existingProfile) { + buildLearningPaths(prompts, existingProfile.id).catch(() => {}); + } return; } + const { raw: llmResult, merged: initialMerged } = analysisResult; + if (existingProfile) { - const changeSummary = generateChangeSummary( - JSON.parse(existingProfile.profileData), - updatedProfileData - ); - userProfileManager.updateProfile( - existingProfile.id, - updatedProfileData, - prompts.length, - changeSummary - ); + let updatedProfileData = initialMerged!; + const MAX_RETRIES = 2; + let retries = 0; + let success = false; + + while (!success && retries <= MAX_RETRIES) { + if (retries > 0) { + existingProfile = userProfileManager.getActiveProfile(userId); + if (!existingProfile) break; + const retryProfileData: UserProfileData = JSON.parse(existingProfile.profileData); + const { data: decayedRetry } = userProfileManager.decayInMemory(retryProfileData); + existingProfile = { ...existingProfile, profileData: JSON.stringify(decayedRetry) }; + updatedProfileData = await userProfileManager.mergeProfileData( + decayedRetry, + llmResult, + undefined, + existingProfile.id + ); + log("user-profile-learning: retry merge", { + retry: retries, + profileId: existingProfile.id, + }); + } + + let changeSummary = generateChangeSummary( + JSON.parse(existingProfile.profileData), + updatedProfileData + ); + + const validationSummary = applyValidations( + updatedProfileData, + llmResult, + existingProfile.id, + validationPrefKeys + ); + if (validationSummary) { + changeSummary = changeSummary + "; " + validationSummary; + } + + success = userProfileManager.updateProfile( + existingProfile.id, + updatedProfileData, + prompts.length, + changeSummary + ); + if (!success) { + log("User profile update conflict, retrying", { + profileId: existingProfile.id, + userId, + retry: retries, + }); + } + retries++; + } + + if (!success) { + log("User profile update conflict: exhausted retries", { + profileId: existingProfile?.id, + userId, + }); + return; + } + + userPromptManager.markMultipleAsUserLearningCaptured(prompts.map((p) => p.id)); } else { userProfileManager.createProfile( userId, tags.user.displayName || "Unknown", tags.user.userName || "unknown", tags.user.userEmail || "unknown", - updatedProfileData, + llmResult, prompts.length ); + userPromptManager.markMultipleAsUserLearningCaptured(prompts.map((p) => p.id)); } - userPromptManager.markMultipleAsUserLearningCaptured(prompts.map((p) => p.id)); - if (CONFIG.showUserProfileToasts) { await ctx.client?.tui .showToast({ @@ -102,127 +241,318 @@ function generateChangeSummary(oldProfile: UserProfileData, newProfile: UserProf return changes.length > 0 ? changes.join(", ") : "Profile refinement"; } -function buildUserAnalysisContext( - prompts: UserPrompt[], - existingProfile: UserProfile | null -): string { - const existingProfileSection = existingProfile - ? ` -## Existing User Profile +function buildCategorySummary(profileData: UserProfileData): string { + const parts: string[] = []; + + const prefCats = [...new Set(profileData.preferences.map((p) => p.category))]; + const patCats = [...new Set(profileData.patterns.map((p) => p.category))]; + + const catParts: string[] = []; + if (prefCats.length > 0) { + const catCounts = prefCats + .map((cat) => { + const cnt = profileData.preferences.filter((p) => p.category === cat).length; + return `${cat} (${cnt})`; + }) + .join(", "); + catParts.push(`Preference categories: ${catCounts}`); + } + if (patCats.length > 0) { + const catCounts = patCats + .map((cat) => { + const cnt = profileData.patterns.filter((p) => p.category === cat).length; + return `${cat} (${cnt})`; + }) + .join(", "); + catParts.push(`Pattern categories: ${catCounts}`); + } -${existingProfile.profileData} + const catSection = + catParts.length > 0 + ? `## Existing Categories\nUse these exact category names when your observation fits:\n\n${catParts.join("\n")}\n` + : ""; + + const prefCount = profileData.preferences.length; + const patCount = profileData.patterns.length; + const wfCount = profileData.workflows.length; + + const wfParts: string[] = []; + if (wfCount > 0) { + wfParts.push( + "## Existing Workflows\nFor reference — only report a workflow when recent prompts show a genuinely NEW sequence, NOT a minor variant of an existing one:" + ); + profileData.workflows.forEach((wf, i) => { + const steps = wf.steps?.length + ? ` (freq ${wf.frequency || 1}x: ${wf.steps.join(" → ")})` + : ""; + wfParts.push(`${i + 1}. ${wf.description}${steps}`); + }); + } -**Instructions**: Merge new insights with the existing profile. Update confidence scores for reinforced patterns, add new patterns, and refine existing ones.` - : ` -**Instructions**: Create a new user profile from scratch based on the prompts below.`; + const countSection = `## Profile Size\nPreferences: ${prefCount} | Patterns: ${patCount} | Workflows: ${wfCount}\n +New observations are matched via embedding cosine similarity — write descriptions in your own words; do not reuse existing wording.`; - return `# User Profile Analysis + return [catSection, ...wfParts, countSection].filter(Boolean).join("\n"); +} -Analyze ${prompts.length} user prompts to ${existingProfile ? "update" : "create"} the user profile. +function buildUserAnalysisContext( + prompts: UserPrompt[], + existingProfile: UserProfile | null, + validationPrompt?: string +): string { + const base = `# User Profile Analysis -${existingProfileSection} +Analyze ${prompts.length} user prompts to ${existingProfile ? "update" : "create"} the user profile. +${existingProfile ? `The merge system will automatically connect your observations to existing profile entries — you only need to describe what you see in these recent prompts.` : `Create a new user profile from scratch based on the prompts below.`} +${existingProfile ? buildCategorySummary(JSON.parse(existingProfile.profileData)) : ""} ## Recent Prompts ${prompts.map((p, i) => `${i + 1}. ${p.content}`).join("\n\n")} ## Analysis Guidelines -Identify and ${existingProfile ? "update" : "create"}: +Identify and ${existingProfile ? "report" : "create"}: -1. **Preferences** (max ${CONFIG.userProfileMaxPreferences}) + 1. **Preferences** - Code style, communication style, tool preferences - - Assign confidence 0.5-1.0 based on evidence strength + - Assign confidence 0.3-0.5 based on evidence strength in these recent prompts - Include 1-3 example prompts as evidence + - **Revealed preferences**: when the user chooses one approach over alternatives (e.g. picks simpler solution, skips certain steps), capture the choice as a lower-confidence preference (0.3-0.5). What the user does NOT do is also a signal. -2. **Patterns** (max ${CONFIG.userProfileMaxPatterns}) - - Recurring topics, problem domains, technical interests + 2. **Patterns** + - Recurring topics, problem domains, technical interests seen in these prompts - Track frequency of occurrence -3. **Workflows** (max ${CONFIG.userProfileMaxWorkflows}) - - Development sequences, habits, learning style - - Break down into steps if applicable + 3. **Workflows** + - Distinct, named step sequences the user follows repeatedly + - Each workflow should represent a DIFFERENT activity (different purpose, different steps) + - Break down into 3-6 concrete, observable steps, NOT abstract phases + - Do NOT repeat the same workflow every cycle — only output when you observe a NEW recurring sequence + - Examples of distinct workflows: "debugging workflow", "code review workflow", "learning workflow", "refactoring workflow" + +CRITICAL: Only output observations grounded in the RECENT PROMPTS above. Write descriptions in your own words — the system matches by embedding similarity, not exact wording. Do NOT output entries that lack evidence in recent prompts. Put the core semantics at the beginning of each description, keeping descriptions concise and specific (under 120 characters). Do NOT extract one-time debugging tasks, environment setup issues, or specific error investigations as preferences — these are transient events, not behavioral patterns. + +## Few-Shot Examples + +❌ Do NOT extract as preference: +- "User is debugging a NullPointerException in auth service" (one-time debugging task) +- "User installed Redis for the first time" (one-time setup event) +- "User ran npm audit fix" (routine maintenance, not a behavioral pattern) + +✅ DO extract as preference: +- "User prefers functional programming style over OOP" +- "User consistently writes tests before implementation" +- "User asks for explanations before accepting code changes" + +✅ DO extract as workflow (distinct, non-overlapping): +- Debugging workflow: "reproduce the error → check logs → grep source code → trace call chain → propose fix → verify fix" +- Code review workflow: "read the diff → check edge cases → verify consistency with existing patterns → report issues → suggest alternatives" +- Learning workflow: "ask for explanation → request examples → test understanding with a small task → apply to real problem" + +❌ Do NOT extract as workflow: +- "User analyzes problems and verifies solutions" (too abstract — not a concrete step sequence) +- "User writes code and tests it" (too generic — covers everything)`; + + if (validationPrompt) { + return base + "\n\n" + validationPrompt; + } + return base; +} + +type AnalysisResult = { raw: UserProfileData; merged: UserProfileData | null }; + +function applyValidations( + profileData: UserProfileData, + llmResult: UserProfileData, + profileId: string, + prefKeys?: string[] +): string | null { + const validations = (llmResult as any).validations as + | Array<{ + index: number; + verdict: string; + reason: string; + }> + | undefined; + if (!validations?.length || !prefKeys?.length) return null; + + const allItems = [...profileData.preferences, ...profileData.patterns]; + const results: string[] = []; + let confirmed = 0; + let contradicted = 0; + let inaccurate = 0; + let oversimplified = 0; + + for (const v of validations) { + const key = prefKeys[v.index]; + if (!key) continue; + const item = allItems.find( + (i) => `${i.category || "_"}||${(i.description || "").substring(0, 30)}` === key + ); + if (!item) { + log("user-profile-learning: validation match failed", { index: v.index, key }); + continue; + } + + if (v.verdict === "confirmed") { + item.alpha = (item.alpha || 1) + 0.5; + userProfileManager.syncConfidence(item); + confirmed++; + results.push(`confirmed [${v.index}] ${v.reason}`); + } else if (v.verdict === "contradicted") { + const oldAlpha = item.alpha || 1; + item.alpha = oldAlpha * 0.75; + item.beta = (item.beta || 1) + oldAlpha * 0.25; + userProfileManager.syncConfidence(item); + contradicted++; + results.push(`contradicted [${v.index}] ${v.reason}`); + } else if (v.verdict === "inaccurate") { + const oldAlpha = item.alpha || 1; + item.alpha = oldAlpha * 0.6; + item.beta = (item.beta || 1) + oldAlpha * 0.4; + userProfileManager.syncConfidence(item); + inaccurate++; + results.push(`inaccurate [${v.index}] ${v.reason}`); + } else if (v.verdict === "oversimplified") { + const oldAlpha = item.alpha || 1; + item.alpha = oldAlpha * 0.85; + item.beta = (item.beta || 1) + oldAlpha * 0.15; + userProfileManager.syncConfidence(item); + oversimplified++; + results.push(`oversimplified [${v.index}] ${v.reason}`); + const evidence = (item as any).evidence; + if (Array.isArray(evidence) && evidence.length >= 3) { + const itemType = profileData.preferences.includes(item) ? "preference" : "pattern"; + userProfileManager.evolveAndUpdate(item, itemType, profileId).catch(() => {}); + } + } else { + results.push(`no_evidence [${v.index}] ${v.reason}`); + } + } + + if (results.length > 0) { + log("user-profile-learning: validation results", { validated: results }); + } + if (confirmed === 0 && contradicted === 0 && inaccurate === 0 && oversimplified === 0) + return null; -${existingProfile ? "Merge with existing profile, incrementing frequencies and updating confidence scores." : "Create initial profile with conservative confidence scores."}`; + return `validated: ${confirmed} confirmed, ${contradicted} contradicted, ${inaccurate} inaccurate, ${oversimplified} oversimplified`; } async function analyzeUserProfile( context: string, existingProfile: UserProfile | null -): Promise { +): Promise { + log("user-profile-learning: analyze called", { hasProfile: !!existingProfile }); if (CONFIG.opencodeProvider && CONFIG.opencodeModel) { - const { isProviderConnected, getV2Client, generateStructuredOutput } = - await import("./ai/opencode-provider.js"); + log("user-profile-learning: trying opencode provider"); + try { + const { generateStructuredOutput } = await import("./ai/opencode-provider.js"); + const { getOpenCodeClient } = await import("./ai/profile-llm-client.js"); - if (!isProviderConnected(CONFIG.opencodeProvider)) { - throw new Error( - `opencode provider '${CONFIG.opencodeProvider}' is not connected. Check your opencode provider configuration.` - ); - } + log("user-profile-learning: opencode provider diag", { + provider: CONFIG.opencodeProvider, + model: CONFIG.opencodeModel, + }); - const v2Client = getV2Client(); - if (!v2Client) { - throw new Error( - "opencode-mem: v2 client not initialized; cannot perform user-profile learning" - ); - } + const v2Client = await getOpenCodeClient(); - const systemPrompt = `You are a user behavior analyst for a coding assistant. + const systemPrompt = `You are a user behavior analyst for a coding assistant. Your task is to analyze user prompts and ${existingProfile ? "update" : "create"} a comprehensive user profile. CRITICAL: Detect the language used by the user in their prompts. You MUST output all descriptions, categories, and text in the SAME language as the user's prompts. -Use the update_user_profile tool to save the ${existingProfile ? "updated" : "new"} profile.`; +CRITICAL: All JSON string values MUST escape double quotes with backslash. Do NOT use unescaped quotation marks inside string values. - const { z } = await import("zod"); - const schema = z.object({ - preferences: z.array( - z.object({ - category: z.string(), - description: z.string(), - confidence: z.number(), - evidence: z.array(z.string()), - }) - ), - patterns: z.array( - z.object({ - category: z.string(), - description: z.string(), - }) - ), - workflows: z.array( - z.object({ - description: z.string(), - steps: z.array(z.string()), - }) - ), - }); - - const result = await generateStructuredOutput({ - client: v2Client, - providerID: CONFIG.opencodeProvider, - modelID: CONFIG.opencodeModel, - systemPrompt, - userPrompt: context, - schema, - }); +Use the update_user_profile tool to save the ${existingProfile ? "updated" : "new"} profile.`; - if (existingProfile) { - const existingData: UserProfileData = JSON.parse(existingProfile.profileData); - return userProfileManager.mergeProfileData( - existingData, - result as unknown as Partial - ); + const { z } = await import("zod"); + const schema = z.object({ + preferences: z.array( + z.object({ + category: z.string(), + description: z.string(), + confidence: z.number().min(0).max(0.5), + evidence: z.array(z.string()), + }) + ), + patterns: z.array( + z.object({ + category: z.string(), + description: z.string(), + }) + ), + workflows: z.array( + z.object({ + description: z.string(), + steps: z.array(z.string()), + }) + ), + validations: z + .array( + z.object({ + index: z.number(), + verdict: z.enum([ + "confirmed", + "contradicted", + "no_evidence", + "inaccurate", + "oversimplified", + ]), + reason: z.string(), + }) + ) + .optional(), + }); + + log("user-profile-learning: calling LLM", { contextLen: context.length }); + + const result = await Promise.race([ + generateStructuredOutput({ + client: v2Client, + providerID: CONFIG.opencodeProvider, + modelID: CONFIG.opencodeModel, + systemPrompt, + userPrompt: context, + schema, + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error("user-profile-learning: timeout")), 120000) + ), + ]); + + log("user-profile-learning: LLM returned", { + prefCount: result.preferences?.length, + patCount: result.patterns?.length, + wfCount: result.workflows?.length, + }); + + const rawData = result as unknown as UserProfileData; + + if (existingProfile) { + const existingData: UserProfileData = JSON.parse(existingProfile.profileData); + const merged = await userProfileManager.mergeProfileData( + existingData, + rawData as unknown as Partial, + undefined, + existingProfile.id + ); + return { raw: rawData, merged }; + } + return { raw: rawData, merged: null }; + } catch (e) { + log("user-profile-learning: opencode provider failed, falling back to external API", { + error: String(e), + }); } - return result as UserProfileData; } if (!CONFIG.memoryModel || !CONFIG.memoryApiUrl) { log("User Profile Config Check Failed:", { memoryModel: CONFIG.memoryModel, memoryApiUrl: CONFIG.memoryApiUrl, - memoryApiKey: CONFIG.memoryApiKey, }); throw new Error("External API not configured for user memory learning"); } @@ -240,6 +570,8 @@ Your task is to analyze user prompts and ${existingProfile ? "update" : "create" CRITICAL: Detect the language used by the user in their prompts. You MUST output all descriptions, categories, and text in the SAME language as the user's prompts. +CRITICAL: All JSON string values MUST escape double quotes with backslash. Do NOT use unescaped quotation marks inside string values. + Use the update_user_profile tool to save the ${existingProfile ? "updated" : "new"} profile.`; const toolSchema = { @@ -259,7 +591,7 @@ Use the update_user_profile tool to save the ${existingProfile ? "updated" : "ne properties: { category: { type: "string" }, description: { type: "string" }, - confidence: { type: "number", minimum: 0, maximum: 1 }, + confidence: { type: "number", minimum: 0, maximum: 0.5 }, evidence: { type: "array", items: { type: "string" }, maxItems: 3 }, }, required: ["category", "description", "confidence", "evidence"], @@ -287,6 +619,27 @@ Use the update_user_profile tool to save the ${existingProfile ? "updated" : "ne required: ["description", "steps"], }, }, + validations: { + type: "array", + items: { + type: "object", + properties: { + index: { type: "number" }, + verdict: { + type: "string", + enum: [ + "confirmed", + "contradicted", + "no_evidence", + "inaccurate", + "oversimplified", + ], + }, + reason: { type: "string" }, + }, + required: ["index", "verdict", "reason"], + }, + }, }, required: ["preferences", "patterns", "workflows"], }, @@ -304,12 +657,127 @@ Use the update_user_profile tool to save the ${existingProfile ? "updated" : "ne throw new Error(result.error || "Failed to analyze user profile"); } - const rawData = result.data; + const rawData = result.data as UserProfileData; if (existingProfile) { const existingData: UserProfileData = JSON.parse(existingProfile.profileData); - return userProfileManager.mergeProfileData(existingData, rawData); + const merged = await userProfileManager.mergeProfileData( + existingData, + rawData, + undefined, + existingProfile.id + ); + return { raw: rawData, merged }; + } + + return { raw: rawData, merged: null }; +} + +type LearningPathsResult = { paths: { topic: string; chain: string[]; description: string }[] }; + +async function buildLearningPaths(prompts: UserPrompt[], profileId: string): Promise { + const promptTexts = prompts.map((p, i) => `${i + 1}. ${p.content}`).join("\n"); + const systemPrompt = + "You are a learning path analyst. Identify causal chains across a user's prompts. Output valid JSON."; + const userPrompt = `Analyze these user prompts for causal learning chains: + +${promptTexts} + +Identify sequences where earlier prompts led to later ones (e.g. "learned X → applied X → refined X"). Return JSON: +{ "paths": [{ "topic": "string", "chain": ["step1", "step2", "step3"], "description": "one sentence summary" }] } +If no clear chains, return { "paths": [] }.`; + + let result: LearningPathsResult | null = null; + + if (CONFIG.opencodeProvider && CONFIG.opencodeModel) { + try { + const { z } = await import("zod"); + const { generateStructuredOutput } = await import("./ai/opencode-provider.js"); + const { getOpenCodeClient } = await import("./ai/profile-llm-client.js"); + + let v2Client; + try { + v2Client = await getOpenCodeClient(); + } catch { + // provider not available, skip learning paths + } + if (v2Client) { + result = (await Promise.race([ + generateStructuredOutput({ + client: v2Client, + providerID: CONFIG.opencodeProvider, + modelID: CONFIG.opencodeModel, + systemPrompt, + userPrompt, + schema: z.object({ + paths: z.array( + z.object({ + topic: z.string(), + chain: z.array(z.string()), + description: z.string(), + }) + ), + }), + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error("learning paths: opencode timeout")), 120000) + ), + ])) as LearningPathsResult; + } + } catch (e) { + log("learning paths: native provider failed", { error: String(e) }); + } + } + + if (!result && CONFIG.memoryModel && CONFIG.memoryApiUrl) { + try { + const response = await fetch(`${CONFIG.memoryApiUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${CONFIG.memoryApiKey || ""}`, + }, + body: JSON.stringify({ + model: CONFIG.memoryModel, + messages: [ + { role: "system", content: systemPrompt }, + { role: "user", content: userPrompt }, + ], + temperature: 0.3, + response_format: { type: "json_object" }, + }), + signal: AbortSignal.timeout(60000), + }); + + if (response.ok) { + const data = (await response.json()) as { + choices?: { message?: { content?: string } }[]; + }; + const content = data.choices?.[0]?.message?.content; + if (content) result = JSON.parse(content) as LearningPathsResult; + } + } catch (e) { + log("learning paths: external API failed", { error: String(e) }); + } } - return rawData as UserProfileData; + if (!result?.paths?.length) return; + + log("learning paths: detected", { + profileId, + pathCount: result.paths.length, + topics: result.paths.map((p) => p.topic).join(", "), + }); + + const profile = userProfileManager.getProfileById(profileId); + if (!profile) return; + + const data: UserProfileData = JSON.parse(profile.profileData); + data.learning_paths = result.paths; + userProfileManager.updateProfile( + profileId, + data, + 0, + `Updated learning paths: ${result.paths.map((p) => p.topic).join(", ")}` + ); } diff --git a/src/services/user-profile/ai-cleanup.ts b/src/services/user-profile/ai-cleanup.ts new file mode 100644 index 0000000..14f1d52 --- /dev/null +++ b/src/services/user-profile/ai-cleanup.ts @@ -0,0 +1,542 @@ +import type { UserProfileData } from "./types.js"; +import { CONFIG } from "../../config.js"; +import { log } from "../logger.js"; + +export interface AICleanupResult { + cleaned: UserProfileData; + diff: CleanupDiff; +} + +export interface CleanupDiff { + kept: string[]; + merged: Array<{ ids: string[]; result: string }>; + removed: Array<{ id: string; reason: string }>; +} + +export async function aiCleanupProfile(profileData: UserProfileData): Promise { + const t0 = Date.now(); + const indexed = addIdsToProfile(profileData); + const prompt = buildAICleanupPrompt(indexed); + + log("AI cleanup: prompt built", { + prefCount: indexed.preferences.length, + patCount: indexed.patterns.length, + wfCount: indexed.workflows.length, + promptLen: prompt.length, + buildMs: Date.now() - t0, + }); + + const aiStart = Date.now(); + const result = await callAICleanup(prompt); + log("AI cleanup: AI response received", { callMs: Date.now() - aiStart }); + + const cleanedById = buildItemIndex(result.profile); + const originalById = buildItemIndex(indexed); + const counters = { cleaned: 0, original: 0 }; + + const cleaned = rebuildProfileUsing(result.mapping, cleanedById, originalById, counters); + const diff = generateDiff(indexed, result.mapping); + + const sampleCleaned = result.profile.preferences[0]; + log("AI cleanup: rebuild done", { + cleanedPrefCount: result.profile.preferences.length, + cleanedPatCount: result.profile.patterns.length, + cleanedWfCount: result.profile.workflows.length, + sampleId: sampleCleaned?.id, + sampleDescLen: sampleCleaned?.description?.length, + cleanedItemDescLen: cleanedById.get("pref_0")?.description?.length, + originalItemDescLen: originalById.get("pref_0")?.description?.length, + keptById: cleaned.preferences.length, + usedCleaned: counters.cleaned, + usedOriginal: counters.original, + }); + + log("AI cleanup: complete", { + totalMs: Date.now() - t0, + kept: diff.kept.length, + merged: diff.merged.length, + removed: diff.removed.length, + }); + + return { cleaned, diff }; +} + +export async function aiCleanupProfileFromIndexed( + indexed: IndexedProfile +): Promise { + const t0 = Date.now(); + const prompt = buildAICleanupPrompt(indexed); + + log("AI cleanup: prompt built (filtered)", { + prefCount: indexed.preferences.length, + patCount: indexed.patterns.length, + wfCount: indexed.workflows.length, + promptLen: prompt.length, + buildMs: Date.now() - t0, + }); + + const aiStart = Date.now(); + const result = await callAICleanup(prompt); + log("AI cleanup: AI response received (filtered)", { callMs: Date.now() - aiStart }); + + const cleanedById = buildItemIndex(result.profile); + const originalById = buildItemIndex(indexed); + const counters = { cleaned: 0, original: 0 }; + + const cleaned = rebuildProfileUsing(result.mapping, cleanedById, originalById, counters); + const diff = generateDiff(indexed, result.mapping); + + log("AI cleanup: complete (filtered)", { + totalMs: Date.now() - t0, + kept: diff.kept.length, + merged: diff.merged.length, + removed: diff.removed.length, + }); + + return { cleaned, diff }; +} + +export function filterProfileForCleanup( + profileData: UserProfileData, + includeIds: string[] +): IndexedProfile { + const idSet = new Set(includeIds); + const indexed = addIdsToProfile(profileData); + return { + preferences: indexed.preferences.filter((p) => idSet.has(p.id)), + patterns: indexed.patterns.filter((p) => idSet.has(p.id)), + workflows: indexed.workflows.filter((p) => idSet.has(p.id)), + }; +} + +interface IndexedProfileItem { + id: string; + category?: string; + description: string; + confidence?: number; + frequency?: number; + [key: string]: unknown; +} + +interface IndexedProfile { + preferences: IndexedProfileItem[]; + patterns: IndexedProfileItem[]; + workflows: IndexedProfileItem[]; +} + +interface AIMapping { + kept: string[]; + merged: string[][]; + removed: string[]; +} + +function addIdsToProfile(profile: UserProfileData): IndexedProfile { + const items = { + preferences: profile.preferences.map((p, i) => ({ ...p, id: `pref_${i}` })), + patterns: profile.patterns.map((p, i) => ({ ...p, id: `pat_${i}` })), + workflows: profile.workflows.map((w, i) => ({ ...w, id: `wf_${i}` })), + }; + return items; +} + +function buildAICleanupPrompt(profile: IndexedProfile): string { + const profileJSON = JSON.stringify( + { + preferences: profile.preferences.map(formatForAI), + patterns: profile.patterns.map(formatForAI), + workflows: profile.workflows.map(formatForAI), + }, + null, + 2 + ); + + return `You are a user profile analyst. The profile below contains duplicate entries within each category (preferences, patterns, workflows). Output a cleaned profile. + +Rules: +1. Merge semantically identical entries ONLY within the same section (pref_ with pref_, pat_ with pat_, wf_ with wf_) +2. Do NOT merge across sections — preferences and patterns are different things +3. When merging, keep the most specific description. Do not artificially shorten or inflate — the natural length of the original is fine. +4. Do not add new entries; only merge and remove +5. Prefer merging over removing. If entries share the same topic or describe similar behavior, merge them. Only remove truly irrelevant/generic items that add no value (e.g. "uses tools", "checks things"). +6. You MUST return a mapping showing the disposition of each id + +Current profile: +${profileJSON} + +Return JSON only (no markdown): +{ + "preferences": [ + { "id": "pref_0", "category": "...", "description": "..." } + ], + "patterns": [...], + "workflows": [...], + "mapping": { + "kept": ["pref_0", "pat_1"], + "merged": [["pref_2", "pref_5"], ["pat_3", "pat_8"]], + "removed": ["pref_4"] + } +} + +The first id in each merged group is the kept entry; the rest are merged into it.`; +} + +function formatForAI(item: IndexedProfileItem): Record { + const { id, category, description, frequency } = item; + return { id, category, description, frequency }; +} + +async function callAICleanup( + prompt: string +): Promise<{ profile: IndexedProfile; mapping: AIMapping }> { + // Use opencode internal session when opencodeProvider is configured (same pattern as auto-capture) + if (CONFIG.opencodeProvider && CONFIG.opencodeModel) { + try { + const { getV2Client } = await import("../ai/opencode-provider.js"); + const v2Client = getV2Client(); + if (v2Client) { + const result = await callViaOpencodeWithClient(v2Client, prompt); + if (result) return result; + } + } catch (e) { + log("AI cleanup: opencode session failed, falling back to external API", { + error: String(e), + }); + } + } + + if (CONFIG.memoryModel && CONFIG.memoryApiUrl) { + return callViaExternalAPI(prompt); + } + + throw new Error("No AI provider configured for profile cleanup"); +} + +async function callViaExternalAPI( + prompt: string +): Promise<{ profile: IndexedProfile; mapping: AIMapping }> { + const t0 = Date.now(); + const systemPrompt = + "You are a user profile cleanup assistant. Merge duplicate entries and return only JSON."; + + const response = await fetch(`${CONFIG.memoryApiUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${CONFIG.memoryApiKey}`, + }, + body: JSON.stringify({ + model: CONFIG.memoryModel, + messages: [ + { role: "system", content: systemPrompt }, + { role: "user", content: prompt }, + ], + temperature: 0.3, + response_format: { type: "json_object" }, + }), + signal: AbortSignal.timeout(60000), + }); + + log("AI cleanup: external API http done", { httpMs: Date.now() - t0, status: response.status }); + + if (!response.ok) { + throw new Error(`API request failed: ${response.status} ${response.statusText}`); + } + + const data: any = await response.json(); + const content = data.choices?.[0]?.message?.content; + if (!content) throw new Error("No content in API response"); + + log("AI cleanup: external API parsing done", { + totalMs: Date.now() - t0, + respLen: content.length, + }); + + const parsed = JSON.parse(content); + return { + profile: parsed as IndexedProfile, + mapping: parsed.mapping as AIMapping, + }; +} + +async function callViaOpencodeWithClient( + v2Client: any, + prompt: string +): Promise<{ profile: IndexedProfile; mapping: AIMapping }> { + const t0 = Date.now(); + const systemPrompt = + "You are a user profile cleanup assistant. Merge duplicate entries and return only JSON without markdown wrapping."; + + const created = (await Promise.race([ + v2Client.session.create({ + title: "opencode-mem profile cleanup", + directory: process.cwd(), + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error("session.create timeout")), 30000) + ), + ])) as any; + log("AI cleanup: session.create result", { + rawType: typeof created, + keys: Object.keys(created || {}), + hasData: !!created?.data, + dataId: created?.data?.id, + }); + + const sessionID = created?.data?.id || created?.id || created?.sessionID; + if (!sessionID) throw new Error("session.create returned no session id"); + + log("AI cleanup: session created", { sessionID, createMs: Date.now() - t0 }); + + try { + const TIMEOUT_MS = 120000; + const promptResult = await Promise.race([ + v2Client.session.prompt({ + sessionID, + model: { + providerID: CONFIG.opencodeProvider || "bs-aigw", + modelID: CONFIG.opencodeModel || "deepseek-v4-flash", + }, + system: systemPrompt, + parts: [{ type: "text", text: prompt }], + noReply: true, + }), + new Promise((_, reject) => + setTimeout( + () => reject(new Error(`opencodeClient prompt timeout after ${TIMEOUT_MS}ms`)), + TIMEOUT_MS + ) + ), + ]); + + log("AI cleanup: session.prompt done", { promptMs: Date.now() - t0 }); + + const info = ( + promptResult as { + data?: { info?: { text?: string; error?: { name: string; data?: { message?: string } } } }; + } + ).data?.info; + + if (!info) throw new Error("prompt response missing info"); + if (info.error) + throw new Error(`opencode reported ${info.error.name}: ${info.error.data?.message ?? ""}`); + + const rawText = info.text?.trim() || ""; + const jsonMatch = rawText.match(/\{[\s\S]*\}/); + if (!jsonMatch) throw new Error("AI response did not contain valid JSON"); + + const parsed = JSON.parse(jsonMatch[0]); + return { + profile: parsed as IndexedProfile, + mapping: parsed.mapping as AIMapping, + }; + } finally { + try { + await v2Client.session.delete({ sessionID }); + } catch {} + } +} + +function buildItemIndex(profile: IndexedProfile): Map { + const map = new Map(); + for (const item of profile.preferences) map.set(item.id, item); + for (const item of profile.patterns) map.set(item.id, item); + for (const item of profile.workflows) map.set(item.id, item); + return map; +} + +function rebuildProfileUsing( + mapping: AIMapping, + cleanedById: Map, + originalById: Map, + counters?: { cleaned: number; original: number } +): UserProfileData { + const keptIds = new Set( + [...mapping.kept, ...mapping.merged.map((g) => g[0] ?? "")].filter(Boolean) + ); + + const mergedGroups = mapping.merged.filter((g) => g.length > 1); + const mergedSourceIds = new Set(); + for (const group of mergedGroups) { + for (let i = 1; i < group.length; i++) { + mergedSourceIds.add(group[i] ?? ""); + } + } + + const result: UserProfileData = { preferences: [], patterns: [], workflows: [] }; + + for (const id of keptIds) { + const cleanedItem = cleanedById.get(id); + const originalItem = originalById.get(id); + const item = cleanedItem || originalItem; + if (!item) continue; + + if (counters) { + if (cleanedById.has(id)) counters.cleaned++; + else counters.original++; + } + + const resultItem = { ...item }; + delete (resultItem as any).id; + + if (originalItem) { + const preserveKeys = [ + "centroid", + "anchor", + "weakHitCount", + "lastWeakHitAt", + "driftBelowCount", + "frequency", + "evidence", + "steps", + "confidence", + "alpha", + "beta", + "weakAlpha", + "weakBeta", + "lastMatchTime", + "firstSeen", + "pendingValidation", + ]; + for (const key of preserveKeys) { + if ((originalItem as any)[key] !== undefined && (resultItem as any)[key] === undefined) { + (resultItem as any)[key] = (originalItem as any)[key]; + } + } + + if (cleanedItem && cleanedItem.description !== originalItem.description) { + (resultItem as any).centroid = undefined; + (resultItem as any).anchor = undefined; + } + } + + if (mergedGroups.some((g) => g[0] === id)) { + const group = mergedGroups.find((g) => g[0] === id)!; + let bestFreq = (originalItem as any).frequency || 0; + let bestCentroid = (originalItem as any).centroid; + let bestAnchor = (originalItem as any).anchor; + for (let i = 1; i < group.length; i++) { + const srcOriginal = originalById.get(group[i] ?? ""); + if (srcOriginal) { + const srcFreq = (srcOriginal as any).frequency || 0; + if (srcFreq > bestFreq) { + bestFreq = srcFreq; + bestCentroid = (srcOriginal as any).centroid; + bestAnchor = (srcOriginal as any).anchor; + } + if ((srcOriginal as any).evidence) { + const existingEvidence = (resultItem as any).evidence || []; + const merged = [ + ...new Set([...(srcOriginal as any).evidence, ...existingEvidence]), + ].slice(0, 10); + (resultItem as any).evidence = merged; + } + } + } + // Take sum frequency — accumulating confirmed merges + let totalFreq = (originalItem as any).frequency || 0; + for (let i = 1; i < group.length; i++) { + const srcOriginal = originalById.get(group[i] ?? ""); + if (srcOriginal) { + totalFreq += (srcOriginal as any).frequency || 0; + } + } + (resultItem as any).frequency = totalFreq; + // Accumulate alpha/beta from merged items + const keeperAlpha = (originalItem as any).alpha || 1; + const keeperBeta = (originalItem as any).beta || 1; + let mergedAlpha = keeperAlpha; + let mergedBeta = keeperBeta; + for (let i = 1; i < group.length; i++) { + const srcOriginal = originalById.get(group[i] ?? ""); + if (srcOriginal) { + mergedAlpha += (srcOriginal as any).alpha || 1; + mergedBeta += (srcOriginal as any).beta || 1; + } + } + (resultItem as any).alpha = mergedAlpha; + (resultItem as any).beta = mergedBeta; + (resultItem as any).lastMatchTime = Math.max( + (originalItem as any).lastMatchTime || 0, + ...group.slice(1).map((id) => (originalById.get(id ?? "") as any)?.lastMatchTime || 0) + ); + (resultItem as any).lastSeen = Math.max( + (originalItem as any).lastSeen || 0, + ...group.slice(1).map((id) => (originalById.get(id ?? "") as any)?.lastSeen || 0) + ); + (resultItem as any).pendingValidation = + !!(originalItem as any).pendingValidation && + group.slice(1).every((id) => !!(originalById.get(id ?? "") as any)?.pendingValidation); + let mergedWeakAlpha = (originalItem as any).weakAlpha || 1; + let mergedWeakBeta = (originalItem as any).weakBeta || 1; + for (let i = 1; i < group.length; i++) { + const srcOriginal = originalById.get(group[i] ?? ""); + if (srcOriginal) { + mergedWeakAlpha += ((srcOriginal as any).weakAlpha || 1) - 1; + mergedWeakBeta += ((srcOriginal as any).weakBeta || 1) - 1; + } + } + (resultItem as any).weakAlpha = mergedWeakAlpha; + (resultItem as any).weakBeta = mergedWeakBeta; + if (bestCentroid) (resultItem as any).centroid = bestCentroid; + if (bestAnchor) (resultItem as any).anchor = bestAnchor; + } + + if (id.startsWith("pref_")) result.preferences.push(resultItem as any); + else if (id.startsWith("pat_")) result.patterns.push(resultItem as any); + else if (id.startsWith("wf_")) result.workflows.push(resultItem as any); + } + + const allOriginalIds = new Set(); + for (const item of originalById.values()) { + if (item.id) allOriginalIds.add(item.id); + } + const unmentionedIds = new Set(); + for (const id of allOriginalIds) { + if (!keptIds.has(id) && !mapping.removed.includes(id)) { + unmentionedIds.add(id); + } + } + + for (const id of unmentionedIds) { + const originalItem = originalById.get(id); + if (!originalItem) continue; + const resultItem = { ...originalItem }; + delete (resultItem as any).id; + if (id.startsWith("pref_")) { + result.preferences.push(resultItem as any); + } else if (id.startsWith("pat_")) { + result.patterns.push(resultItem as any); + } else if (id.startsWith("wf_")) { + result.workflows.push(resultItem as any); + } else if (originalItem.category) { + result.preferences.push(resultItem as any); + } else { + result.preferences.push(resultItem as any); + } + } + + return result; +} + +function generateDiff(original: IndexedProfile, mapping: AIMapping): CleanupDiff { + const index = buildItemIndex(original); + + const diff: CleanupDiff = { + kept: mapping.kept.map((id) => index.get(id)?.description || id), + merged: mapping.merged.map((group) => { + const first = group[0] ?? ""; + return { + ids: group, + result: index.get(first)?.description || first, + }; + }), + removed: mapping.removed.map((id) => ({ + id, + reason: index.get(id) + ? "AI determined this is a duplicate or stale entry" + : "Entry no longer exists", + })), + }; + + return diff; +} diff --git a/src/services/user-profile/profile-context.ts b/src/services/user-profile/profile-context.ts index 67c7e65..4abb920 100644 --- a/src/services/user-profile/profile-context.ts +++ b/src/services/user-profile/profile-context.ts @@ -1,5 +1,32 @@ import { userProfileManager } from "./user-profile-manager.js"; +import { CONFIG } from "../../config.js"; import type { UserProfileData } from "./types.js"; +import { sortProfileItems } from "../../utils/profile.js"; +import { log } from "../logger.js"; + +function dedupByCategory(items: any[], topN: number): any[] { + if (items.length <= topN) return items; + const seen = new Set(); + const result: any[] = []; + for (const item of items) { + if (seen.has(item.category)) continue; + seen.add(item.category); + result.push(item); + if (result.length >= topN) break; + } + return result; +} + +function scoreByRecency(items: any[]): any[] { + const now = Date.now(); + return [...items].sort((a, b) => { + const ageA = (now - (a.lastSeen || 0)) / (24 * 60 * 60 * 1000); + const ageB = (now - (b.lastSeen || 0)) / (24 * 60 * 60 * 1000); + const scoreA = (a.confidence || 0) * 0.7 + Math.exp(-ageA / 90) * 0.3; + const scoreB = (b.confidence || 0) * 0.7 + Math.exp(-ageB / 90) * 0.3; + return scoreB - scoreA; + }); +} export function getUserProfileContext(userId: string): string | null { const profile = userProfileManager.getActiveProfile(userId); @@ -11,39 +38,74 @@ export function getUserProfileContext(userId: string): string | null { const profileData: UserProfileData = JSON.parse(profile.profileData); const parts: string[] = []; - if (profileData.preferences.length > 0) { + const injectPrefs = CONFIG.userProfileInjectPreferences ?? 5; + const injectPats = CONFIG.userProfileInjectPatterns ?? 5; + const injectWfs = CONFIG.userProfileInjectWorkflows ?? 3; + + const sortedPrefs = + profileData.preferences.length > 0 + ? (sortProfileItems(profileData.preferences as any[], "confidence") as any[]) + : []; + const sortedPats = + profileData.patterns.length > 0 + ? (sortProfileItems(profileData.patterns as any[], "frequency") as any[]) + : []; + const sortedWfs = + profileData.workflows.length > 0 + ? (sortProfileItems(profileData.workflows as any[], "frequency") as any[]) + : []; + + const topPrefs = dedupByCategory( + scoreByRecency(sortedPrefs.slice(0, injectPrefs * 2)), + injectPrefs + ); + const topPats = dedupByCategory(sortedPats, injectPats); + const topWfs = sortedWfs.slice(0, injectWfs); + + if (topPrefs.length > 0) { parts.push("User Preferences:"); - profileData.preferences - .sort((a, b) => b.confidence - a.confidence) - .slice(0, 5) - .forEach((pref) => { - parts.push(`- [${pref.category}] ${pref.description}`); - }); + topPrefs.forEach((pref: any) => { + parts.push(`- [${pref.category}] ${pref.description}`); + }); } - if (profileData.patterns.length > 0) { + if (topPats.length > 0) { parts.push("\nUser Patterns:"); - profileData.patterns - .sort((a, b) => b.frequency - a.frequency) - .slice(0, 5) - .forEach((pattern) => { - parts.push(`- [${pattern.category}] ${pattern.description}`); - }); + topPats.forEach((pattern: any) => { + parts.push(`- [${pattern.category}] ${pattern.description}`); + }); } - if (profileData.workflows.length > 0) { + if (topWfs.length > 0) { parts.push("\nUser Workflows:"); - profileData.workflows - .sort((a, b) => b.frequency - a.frequency) - .slice(0, 3) - .forEach((workflow) => { - parts.push(`- ${workflow.description}`); - }); + topWfs.forEach((workflow: any) => { + const steps = workflow.steps?.length + ? ` (${workflow.frequency || 1}x: ${workflow.steps.join(" → ")})` + : ` (${workflow.frequency || 1}x)`; + parts.push(`- ${workflow.description}${steps}`); + }); + } + + if ((profileData as any).learning_paths?.length > 0) { + parts.push("\nLearning Paths:"); + (profileData as any).learning_paths.slice(0, 3).forEach((path: any) => { + parts.push(`- ${path.topic}: ${path.description}`); + }); } if (parts.length === 0) { return null; } - return parts.join("\n"); + const text = parts.join("\n"); + + if (topPrefs.length + topPats.length + topWfs.length > 0) { + log("profile inject", { + prefs: topPrefs.map((p: any) => `[${p.category}] ${p.description}`.substring(0, 80)), + pats: topPats.map((p: any) => `[${p.category}] ${p.description}`.substring(0, 80)), + wfs: topWfs.map((w: any) => w.description.substring(0, 80)), + }); + } + + return text; } diff --git a/src/services/user-profile/types.ts b/src/services/user-profile/types.ts index 2705f33..7386677 100644 --- a/src/services/user-profile/types.ts +++ b/src/services/user-profile/types.ts @@ -2,27 +2,70 @@ export interface UserProfilePreference { category: string; description: string; confidence: number; + frequency: number; evidence: string[]; - lastUpdated: number; + lastSeen: number; + centroid?: number[]; + anchor?: number[]; + weakHitCount?: number; + lastWeakHitAt?: number; + driftBelowCount?: number; + alpha?: number; + beta?: number; + weakAlpha?: number; + weakBeta?: number; + pendingValidation?: boolean; + lastMatchTime?: number; + firstSeen?: number; } export interface UserProfilePattern { category: string; description: string; + confidence: number; frequency: number; + evidence: string[]; lastSeen: number; + centroid?: number[]; + anchor?: number[]; + weakHitCount?: number; + lastWeakHitAt?: number; + driftBelowCount?: number; + alpha?: number; + beta?: number; + weakAlpha?: number; + weakBeta?: number; + pendingValidation?: boolean; + lastMatchTime?: number; + firstSeen?: number; } export interface UserProfileWorkflow { description: string; steps: string[]; + confidence: number; frequency: number; + evidence: string[]; + lastSeen: number; + centroid?: number[]; + anchor?: number[]; + weakHitCount?: number; + lastWeakHitAt?: number; + driftBelowCount?: number; + alpha?: number; + beta?: number; + weakAlpha?: number; + weakBeta?: number; + pendingValidation?: boolean; + lastMatchTime?: number; + firstSeen?: number; } export interface UserProfileData { preferences: UserProfilePreference[]; patterns: UserProfilePattern[]; workflows: UserProfileWorkflow[]; + learning_paths?: { topic: string; chain: string[]; description: string }[]; } export interface UserProfile { diff --git a/src/services/user-profile/user-profile-manager.ts b/src/services/user-profile/user-profile-manager.ts index 54646c0..d17077c 100644 --- a/src/services/user-profile/user-profile-manager.ts +++ b/src/services/user-profile/user-profile-manager.ts @@ -1,9 +1,72 @@ import { getDatabase } from "../sqlite/sqlite-bootstrap.js"; import { join } from "node:path"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { connectionManager } from "../sqlite/connection-manager.js"; import { CONFIG } from "../../config.js"; import type { UserProfile, UserProfileChangelog, UserProfileData } from "./types.js"; -import { safeArray, safeObject } from "./profile-utils.js"; +import { safeArray } from "./profile-utils.js"; +import { EmbeddingService } from "../embedding.js"; +import { log } from "../logger.js"; +import { cosineSimilarityNumbers, l2Normalize } from "../../utils/math.js"; + +const CENTROID_EMA_WEIGHT = 0.85; +const CENTROID_EMA_WEIGHT_COMPLEMENT = 0.15; +const THREE_WAY_CENTROID_W1 = 0.45; +const THREE_WAY_CENTROID_W2 = 0.45; +const THREE_WAY_CENTROID_W3 = 0.1; +const THOMPSON_PRIOR_ALPHA = 0.5; +const THOMPSON_PRIOR_BETA = 1.5; +const THOMPSON_PRIOR_RECOVERY_RATE = 0.8; +const DIRECTION_VALIDATION_TOLERANCE = 0.03; + +/** + * Gamma sampler (Marsaglia-Tsang 2000). + * Used by sampleBeta for Thompson Sampling weak-hit upgrades. + */ +function sampleGamma(shape: number): number { + if (shape < 1) { + const u = Math.random(); + return sampleGamma(shape + 1) * Math.pow(u, 1 / shape); + } + const d = shape - 1 / 3; + const c = 1 / Math.sqrt(9 * d); + while (true) { + let x: number, v: number; + do { + x = randn(); + v = 1 + c * x; + } while (v <= 0); + v = v * v * v; + const u = Math.random(); + if (u < 1 - 0.0331 * x * x * x * x) return d * v; + if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v))) return d * v; + } +} +function randn(): number { + let u = 0, + v = 0; + while (u === 0) u = Math.random(); + while (v === 0) v = Math.random(); + return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v); +} +function sampleBeta(alpha: number, beta: number): number { + const x = sampleGamma(alpha); + const y = sampleGamma(beta); + return x / (x + y); +} + +/** + * Language-agnostic text normalization for embedding comparison. + * Strips punctuation and collapses whitespace — the embedding model + * handles semantic similarity naturally without word-level rules. + */ +function normalizeDescription(text: string): string { + return text + .trim() + .replace(/[，,。！？；;：:、\n\r.]/g, " ") + .replace(/\s+/g, " ") + .trim(); +} const Database = getDatabase(); type DatabaseType = typeof Database.prototype; @@ -11,13 +74,54 @@ type DatabaseType = typeof Database.prototype; const USER_PROFILES_DB_NAME = "user-profiles.db"; export class UserProfileManager { - private db: DatabaseType; + private db!: DatabaseType; private readonly dbPath: string; + private coldBuffer: { preferences: any[]; patterns: any[]; workflows: any[] }; + private coldBufferPath: string; + private dedupCheckedCache: Set = new Set(); constructor() { - this.dbPath = join(CONFIG.storagePath, USER_PROFILES_DB_NAME); - this.db = connectionManager.getConnection(this.dbPath); - this.initDatabase(); + this.dbPath = join(CONFIG.storagePath || "", USER_PROFILES_DB_NAME); + this.coldBufferPath = join(CONFIG.storagePath || "", "cold-buffer.json"); + this.coldBuffer = this.loadColdBuffer(); + try { + this.db = connectionManager.getConnection(this.dbPath); + this.initDatabase(); + } catch (e) { + log("user-profile-manager: db init failed, deferring", { error: String(e) }); + } + } + + private loadColdBuffer(): { preferences: any[]; patterns: any[]; workflows: any[] } { + try { + if (existsSync(this.coldBufferPath)) { + const raw = readFileSync(this.coldBufferPath, "utf-8"); + const data = JSON.parse(raw); + if (data.preferences?.length || data.patterns?.length || data.workflows?.length) { + log("profile cold buffer: loaded from disk", { + prefs: data.preferences?.length || 0, + pats: data.patterns?.length || 0, + wfs: data.workflows?.length || 0, + }); + } + return { + preferences: Array.isArray(data.preferences) ? data.preferences : [], + patterns: Array.isArray(data.patterns) ? data.patterns : [], + workflows: Array.isArray(data.workflows) ? data.workflows : [], + }; + } + } catch { + // 文件损坏或不存在，返回空缓冲 + } + return { preferences: [], patterns: [], workflows: [] }; + } + + private saveColdBuffer(): void { + try { + writeFileSync(this.coldBufferPath, JSON.stringify(this.coldBuffer), "utf-8"); + } catch { + // 磁盘满或无权限时静默失败 + } } private initDatabase(): void { @@ -86,7 +190,11 @@ export class UserProfileManager { const id = `profile_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`; const now = Date.now(); - const cleanedData = this.normalizeProfileData(profileData, now); + const cleanedData: UserProfileData = { + preferences: safeArray(profileData.preferences), + patterns: safeArray(profileData.patterns), + workflows: safeArray(profileData.workflows), + }; const stmt = this.db.prepare(` INSERT INTO user_profiles ( @@ -119,14 +227,19 @@ export class UserProfileManager { profileData: UserProfileData, additionalPromptsAnalyzed: number, changeSummary: string - ): void { + ): boolean { const now = Date.now(); - const cleanedData = this.normalizeProfileData(profileData, now); + const cleanedData: UserProfileData = { + preferences: safeArray(profileData.preferences), + patterns: safeArray(profileData.patterns), + workflows: safeArray(profileData.workflows), + }; const getVersionStmt = this.db.prepare(`SELECT version FROM user_profiles WHERE id = ?`); const versionRow = getVersionStmt.get(profileId) as any; - const newVersion = (versionRow?.version || 0) + 1; + const currentVersion = versionRow?.version || 0; + const newVersion = currentVersion + 1; const updateStmt = this.db.prepare(` UPDATE user_profiles @@ -134,20 +247,27 @@ export class UserProfileManager { version = ?, last_analyzed_at = ?, total_prompts_analyzed = total_prompts_analyzed + ? - WHERE id = ? + WHERE id = ? AND version = ? `); - updateStmt.run( + const result = updateStmt.run( JSON.stringify(cleanedData), newVersion, now, additionalPromptsAnalyzed, - profileId + profileId, + currentVersion ); + if (result.changes === 0) { + return false; + } + this.addChangelog(profileId, newVersion, "update", changeSummary, cleanedData); this.cleanupOldChangelogs(profileId); + + return true; } private addChangelog( @@ -200,58 +320,66 @@ export class UserProfileManager { return rows.map((row) => this.rowToChangelog(row)); } - applyConfidenceDecay(profileId: string): boolean { - const profile = this.getProfileById(profileId); - if (!profile) return false; + getChangelogById(id: string): UserProfileChangelog | undefined { + const stmt = this.db.prepare(`SELECT * FROM user_profile_changelogs WHERE id = ?`); + const row = stmt.get(id) as any; + if (!row) return undefined; + return this.rowToChangelog(row); + } - const profileData: UserProfileData = JSON.parse(profile.profileData); + decayInMemory(data: UserProfileData): { data: UserProfileData; hasChanges: boolean } { const now = Date.now(); - const decayThreshold = CONFIG.userProfileConfidenceDecayDays * 24 * 60 * 60 * 1000; + const prefResult = this.decayItems(data.preferences, now); + const patResult = this.decayItems(data.patterns, now); + const wfResult = this.decayItems(data.workflows, now); + + return { + data: { + ...data, + preferences: prefResult.items, + patterns: patResult.items, + workflows: wfResult.items, + }, + hasChanges: prefResult.hasChanges || patResult.hasChanges || wfResult.hasChanges, + }; + } + + private decayItems>( + items: T[], + now: number + ): { items: T[]; hasChanges: boolean; before: number; removed: number } { + const before = items.length; let hasChanges = false; - profileData.preferences = this.ensureArray(profileData.preferences) - .map((pref) => { - const lastUpdated = this.preferenceLastUpdated(pref, profile, now); - const evidence = this.ensureArray(pref.evidence); - const normalizedPref = { - ...pref, - confidence: this.normalizeConfidence(pref.confidence), - evidence, - lastUpdated, - }; + const filtered = items.filter((item) => { + this.lazyMigrateAlpha(item as any); - if ( - pref.lastUpdated !== lastUpdated || - pref.confidence !== normalizedPref.confidence || - !Array.isArray(pref.evidence) - ) { - hasChanges = true; - } + if ((item as any).lastSeen === undefined) (item as any).lastSeen = now; + if ((item as any).evidence === undefined) (item as any).evidence = []; - const age = now - lastUpdated; - if (age > decayThreshold) { - hasChanges = true; - const decayFactor = Math.max(0.5, 1 - (age - decayThreshold) / decayThreshold); - return { - ...normalizedPref, - confidence: normalizedPref.confidence * decayFactor, - lastUpdated: now, - }; - } - return normalizedPref; - }) - .filter((pref) => { - const keep = pref.confidence >= 0.3; - if (!keep) hasChanges = true; - return keep; - }); + const oldConf = (item as any).confidence; + this.syncConfidence(item as any); + if ((item as any).confidence !== oldConf) hasChanges = true; - if (hasChanges) { - this.updateProfile(profileId, profileData, 0, "Applied confidence decay to preferences"); - } + const age = now - ((item as any).lastSeen || now); + const ageDays = age / (24 * 60 * 60 * 1000); + const alpha = (item as any).alpha ?? 1; - return hasChanges; + if (alpha <= 2 && ageDays > 30) { + log("profile decay: removed stale", { + cat: (item as any).category || (item as any).description?.substring(0, 30), + alpha, + ageDays: Math.round(ageDays), + }); + hasChanges = true; + return false; + } + + return true; + }); + + return { items: filtered, hasChanges, before, removed: before - filtered.length }; } deleteProfile(profileId: string): void { @@ -300,7 +428,12 @@ export class UserProfileManager { }; } - mergeProfileData(existing: UserProfileData, updates: Partial): UserProfileData { + async mergeProfileData( + existing: UserProfileData, + updates: Partial, + embedService?: EmbeddingService, + profileId?: string + ): Promise { const merged: UserProfileData = { preferences: this.ensureArray(existing?.preferences), patterns: this.ensureArray(existing?.patterns), @@ -308,86 +441,726 @@ export class UserProfileManager { }; if (updates.preferences) { - const incomingPrefs = this.ensureArray(updates.preferences); - for (const newPref of incomingPrefs) { - const existingIndex = merged.preferences.findIndex( - (p) => p.category === newPref.category && p.description === newPref.description - ); - - if (existingIndex >= 0) { - const existingItem = merged.preferences[existingIndex]; - if (existingItem) { - merged.preferences[existingIndex] = { - ...newPref, - confidence: Math.min(1, (existingItem.confidence || 0) + 0.1), - evidence: [ - ...new Set([ - ...this.ensureArray(existingItem.evidence), - ...this.ensureArray(newPref.evidence), - ]), - ].slice(0, 5), - lastUpdated: Date.now(), - }; - } - } else { - merged.preferences.push({ ...newPref, lastUpdated: Date.now() }); - } + merged.preferences = await this.mergeItems( + merged.preferences, + this.ensureArray(updates.preferences), + "preference", + embedService, + profileId + ); + } + + if (updates.patterns) { + merged.patterns = await this.mergeItems( + merged.patterns, + this.ensureArray(updates.patterns), + "pattern", + embedService, + profileId + ); + } + + if (updates.workflows) { + merged.workflows = await this.mergeItems( + merged.workflows, + this.ensureArray(updates.workflows), + "workflow", + embedService, + profileId + ); + } + + if (profileId) { + if (merged.preferences.length >= 2) { + await this.detectConflicts(merged.preferences, "preference", profileId); + await this.deduplicateItems(merged.preferences, "preference", profileId); + } + if (merged.patterns.length >= 2) { + await this.detectConflicts(merged.patterns, "pattern", profileId); + await this.deduplicateItems(merged.patterns, "pattern", profileId); } + if (merged.workflows.length >= 2) { + await this.detectConflicts(merged.workflows, "workflow", profileId); + await this.deduplicateItems(merged.workflows, "workflow", profileId); + } + } + + return merged; + } + + private async mergeItems( + existing: T[], + incoming: T[], + itemType: "preference" | "pattern" | "workflow", + embedService?: EmbeddingService, + profileId?: string + ): Promise { + const embed = embedService ?? EmbeddingService.getInstance(); + const useEmbedding = embed.isWarmedUp; + const minDescLen = CONFIG.userProfileEmbeddingMinDescriptionLength; + const sameCatStrong = CONFIG.userProfileEmbeddingThresholdSameCat; + const sameCatWeak = CONFIG.userProfileEmbeddingThresholdSameCatWeak; + const crossCatStrong = CONFIG.userProfileEmbeddingThresholdCrossCat; + const crossCatWeak = CONFIG.userProfileEmbeddingThresholdCrossCatWeak; + const driftThreshold = CONFIG.userProfileCentroidDriftThreshold; - merged.preferences.sort((a, b) => (b.confidence || 0) - (a.confidence || 0)); - merged.preferences = merged.preferences.slice(0, CONFIG.userProfileMaxPreferences); + if (!useEmbedding) { + log("profile embedding skipped: model not warmed up"); } - if (updates.patterns) { - const incomingPatterns = this.ensureArray(updates.patterns); - for (const newPattern of incomingPatterns) { - const existingIndex = merged.patterns.findIndex( - (p) => p.category === newPattern.category && p.description === newPattern.description - ); - - if (existingIndex >= 0) { - const existingItem = merged.patterns[existingIndex]; - if (existingItem) { - merged.patterns[existingIndex] = { - ...newPattern, - frequency: (existingItem.frequency || 1) + 1, - lastSeen: Date.now(), - }; - } - } else { - merged.patterns.push({ ...newPattern, frequency: 1, lastSeen: Date.now() }); + for (const item of existing) { + this.lazyMigrateAlpha(item as any); + if ((item as any).lastSeen === undefined) + (item as any).lastSeen = (item as any).lastUpdated || Date.now(); + if ((item as any).evidence === undefined) (item as any).evidence = []; + + if (useEmbedding && item.description.length >= minDescLen && !(item as any).centroid) { + try { + const emb = await embed.embed(normalizeDescription(item.description)); + const arr = Array.from(emb); + (item as any).centroid = arr; + (item as any).anchor = arr; + log("profile centroid migrated", { desc: item.description.substring(0, 30) }); + } catch (e) { + log("profile centroid migration failed", { + desc: item.description.substring(0, 30), + error: String(e), + }); } } + } - merged.patterns.sort((a, b) => (b.frequency || 0) - (a.frequency || 0)); - merged.patterns = merged.patterns.slice(0, CONFIG.userProfileMaxPatterns); + let matchCount = 0; + let newCount = 0; + if (useEmbedding && (this.coldBuffer as any)[itemType + "s"].length > 0) { + const buffered = [...(this.coldBuffer as any)[itemType + "s"]]; + (this.coldBuffer as any)[itemType + "s"] = []; + this.saveColdBuffer(); + log("profile cold start: draining buffer", { + type: itemType, + bufferSize: buffered.length, + }); + incoming = [...buffered, ...incoming]; } + log("profile merge start", { + type: itemType, + existingCount: existing.length, + incomingCount: incoming.length, + embeddingReady: useEmbedding, + }); - if (updates.workflows) { - const incomingWorkflows = this.ensureArray(updates.workflows); - for (const newWorkflow of incomingWorkflows) { - const existingIndex = merged.workflows.findIndex( - (w) => w.description === newWorkflow.description - ); - - if (existingIndex >= 0) { - const existingItem = merged.workflows[existingIndex]; - if (existingItem) { - merged.workflows[existingIndex] = { - ...newWorkflow, - frequency: (existingItem.frequency || 1) + 1, - }; + for (const newItem of incoming) { + const exactIdx = existing.findIndex( + (e) => e.category === newItem.category && e.description === newItem.description + ); + + if (exactIdx >= 0) { + const oldFreq = (existing[exactIdx] as any).frequency || 1; + existing[exactIdx] = this.mergeConfirmedMatch(existing[exactIdx], newItem, itemType, true); + matchCount++; + log("profile matched: exact", { + type: itemType, + idx: exactIdx, + cat: newItem.category, + frequency: `${oldFreq}→${(existing[exactIdx] as any).frequency || "?"}`, + }); + continue; + } + + if (useEmbedding && newItem.description.length >= minDescLen) { + let top1Score = 0; + let top1Idx = -1; + let top1SameCat = false; + let top1Band: "strong" | "weak" | null = null; + let top2Score = 0; + let top2Idx = -1; + let top2SameCat = false; + let top2Band: "strong" | "weak" | null = null; + + const newEmb = await embed.embed(normalizeDescription(newItem.description)); + + for (let i = 0; i < existing.length; i++) { + const existingItem = existing[i]; + if (!existingItem) continue; + + const centroid = (existingItem as any).centroid as number[] | undefined; + if (!centroid) continue; + + const score = cosineSimilarityNumbers(Array.from(newEmb), centroid); + const sameCat = existingItem.category === newItem.category; + + const strongThreshold = sameCat ? sameCatStrong : crossCatStrong; + const weakThreshold = sameCat ? sameCatWeak : crossCatWeak; + + let band: "strong" | "weak" | null = null; + if (score >= strongThreshold) { + band = "strong"; + } else if (score >= weakThreshold) { + band = "weak"; + } else { + continue; } - } else { - merged.workflows.push({ ...newWorkflow, frequency: 1 }); + + if (score > top1Score) { + top2Score = top1Score; + top2Idx = top1Idx; + top2SameCat = top1SameCat; + top2Band = top1Band; + top1Score = score; + top1Idx = i; + top1SameCat = sameCat; + top1Band = band; + } else if (score > top2Score) { + top2Score = score; + top2Idx = i; + top2SameCat = sameCat; + top2Band = band; + } + } + + if ( + top1Idx >= 0 && + top1Band === "strong" && + top2Idx >= 0 && + top2Band === "strong" && + top1SameCat && + top2SameCat + ) { + const item1 = existing[top1Idx]!; + const item2 = existing[top2Idx]!; + const newEmbArr = Array.from(newEmb); + const threeWayResult = this.combineThree( + item1, + item2, + newItem, + itemType, + newEmbArr, + top1Score + ); + existing[top1Idx] = threeWayResult as T; + const removeIdx = top2Idx; + existing.splice(removeIdx, 1); + matchCount += 2; + log("profile matched: three-way merge", { + type: itemType, + idx1: top1Idx, + idx2: removeIdx, + cat: item1.category, + score1: Math.round(top1Score * 100) / 100, + score2: Math.round(top2Score * 100) / 100, + freq1: (item1 as any).frequency, + freq2: (item2 as any).frequency, + combinedFreq: (threeWayResult as any).frequency, + }); + if ( + this.isMilestone((threeWayResult as any).frequency || 1) && + (threeWayResult as any).evidence?.length >= this.minEvidenceForEvolve(itemType) + ) { + await this.evolveAndUpdate(threeWayResult as any, itemType, profileId); + } + continue; + } + + if (top1Idx >= 0 && top1Band === "strong") { + const existingItem = existing[top1Idx]!; + const oldFreq = (existingItem as any).frequency || 1; + const centroid = (existingItem as any).centroid as number[]; + const anchor = (existingItem as any).anchor as number[]; + + const newEmbArr = Array.from(newEmb); + const updatedCentroid = l2Normalize( + centroid.map( + (v, i) => + CENTROID_EMA_WEIGHT * v + CENTROID_EMA_WEIGHT_COMPLEMENT * (newEmbArr[i] ?? 0) + ) + ); + + let driftBelowCount = ((existingItem as any).driftBelowCount as number) || 0; + if (anchor && anchor.length === updatedCentroid.length) { + const driftScore = cosineSimilarityNumbers(updatedCentroid, anchor); + if (driftScore < driftThreshold) { + driftBelowCount++; + } else { + driftBelowCount = 0; + } + } + + if (driftBelowCount >= 2) { + const driftCentroid = Array.from(newEmb); + const driftAnchor = driftCentroid; + const frozenItem = { ...existingItem }; + (frozenItem as any).centroid = centroid; + (frozenItem as any).driftBelowCount = driftBelowCount; + existing[top1Idx] = frozenItem as T; + existing.push(this.initItem(newItem, itemType, driftCentroid, driftAnchor) as T); + newCount++; + log("profile drift fuse: frozen existing, new item created", { + type: itemType, + idx: top1Idx, + driftBelowCount, + driftScore: anchor + ? Math.round(cosineSimilarityNumbers(updatedCentroid, anchor) * 100) / 100 + : null, + }); + continue; + } + + const combined = this.mergeConfirmedMatch( + existingItem, + newItem, + itemType, + top1SameCat + ) as T; + (combined as any).centroid = updatedCentroid; + (combined as any).anchor = anchor; + (combined as any).driftBelowCount = driftBelowCount; + (combined as any).weakHitCount = 0; + (combined as any).lastWeakHitAt = null; + + if ( + top1Score > 0.9 && + top1SameCat && + newItem.description.length < (existingItem as any).description?.length + ) { + (combined as any).description = newItem.description; + } + + existing[top1Idx] = combined; + matchCount++; + + if (top1SameCat) { + const mergeIndices: number[] = []; + for (let j = 0; j < existing.length; j++) { + if (j === top1Idx) continue; + const other = existing[j]; + if (!other || other.category !== existingItem.category) continue; + const otherCentroid = (other as any).centroid as number[] | undefined; + if (!otherCentroid) continue; + const crossScore = cosineSimilarityNumbers(Array.from(newEmb), otherCentroid); + if (crossScore >= sameCatStrong) { + mergeIndices.push(j); + this.lazyMigrateAlpha(combined as any); + this.lazyMigrateAlpha(other as any); + (combined as any).alpha += (other as any).alpha || 1; + (combined as any).beta = ((combined as any).beta || 1) + ((other as any).beta || 1); + const oldFreq = (combined as any).frequency || 1; + (combined as any).frequency += (other as any).frequency || 1; + (combined as any).evidence = [ + ...new Set([ + ...this.ensureArray((combined as any).evidence), + ...this.ensureArray((other as any).evidence), + ]), + ].slice(0, 10); + const combinedCentroid = (combined as any).centroid as number[]; + const w1 = oldFreq / (oldFreq + ((other as any).frequency || 1)); + const w2 = + ((other as any).frequency || 1) / (oldFreq + ((other as any).frequency || 1)); + const mergedCentroid = l2Normalize( + combinedCentroid.map((v, i) => w1 * v + w2 * (otherCentroid[i] ?? 0)) + ); + (combined as any).centroid = mergedCentroid; + this.syncConfidence(combined as any); + matchCount++; + log("profile matched: cross-validation merge", { + type: itemType, + idx: top1Idx, + mergedIdx: j, + cat: other.category, + crossScore: Math.round(crossScore * 100) / 100, + mergedAlpha: Math.round((combined as any).alpha), + combinedFreq: (combined as any).frequency, + }); + } + } + for (let i = mergeIndices.length - 1; i >= 0; i--) { + existing.splice(mergeIndices[i]!, 1); + } + } + + const driftInfo = + anchor && anchor.length === updatedCentroid.length + ? { + driftScore: + Math.round(cosineSimilarityNumbers(updatedCentroid, anchor) * 100) / 100, + driftBelowCount, + } + : {}; + + log("profile matched: embedding strong", { + type: itemType, + idx: top1Idx, + cat: existingItem.category, + score: Math.round(top1Score * 100) / 100, + sameCat: top1SameCat, + frequency: `${oldFreq}→${(combined as any).frequency || "?"}`, + ...driftInfo, + }); + + const newFreq = (combined as any).frequency || 1; + if ( + this.isMilestone(newFreq) && + (combined as any).evidence?.length >= this.minEvidenceForEvolve(itemType) + ) { + await this.evolveAndUpdate(combined as any, itemType, profileId); + } + + continue; } + + if (top1Idx >= 0 && top1Band === "weak") { + const existingItem = existing[top1Idx]!; + + if (top1SameCat) { + const existingDriftBelow = ((existingItem as any).driftBelowCount as number) || 0; + const centroid = (existingItem as any).centroid as number[]; + const anchor = (existingItem as any).anchor as number[]; + const newEmbArr = Array.from(newEmb); + const updatedCentroid = l2Normalize( + centroid.map( + (v, i) => + CENTROID_EMA_WEIGHT * v + CENTROID_EMA_WEIGHT_COMPLEMENT * (newEmbArr[i] ?? 0) + ) + ); + + let driftBelowCount = existingDriftBelow; + if (anchor && anchor.length === updatedCentroid.length) { + const driftScore = cosineSimilarityNumbers(updatedCentroid, anchor); + if (driftScore < driftThreshold) { + driftBelowCount = existingDriftBelow + 1; + } else { + driftBelowCount = 0; + } + } + + if (driftBelowCount >= 2) { + const driftCentroid = Array.from(newEmb); + const driftAnchor = driftCentroid; + const frozenItem = { ...existingItem }; + (frozenItem as any).centroid = centroid; + (frozenItem as any).driftBelowCount = driftBelowCount; + existing[top1Idx] = frozenItem as T; + existing.push(this.initItem(newItem, itemType, driftCentroid, driftAnchor) as T); + newCount++; + log("profile drift fuse: forced merge path, frozen existing", { + type: itemType, + idx: top1Idx, + driftBelowCount, + driftScore: anchor + ? Math.round(cosineSimilarityNumbers(updatedCentroid, anchor) * 100) / 100 + : null, + }); + continue; + } + + const combined = { ...existingItem }; + (combined as any).alpha = ((existingItem as any).alpha || 1) + top1Score * 1.0; + (combined as any).frequency = ((existingItem as any).frequency || 1) + 1; + (combined as any).evidence = [ + ...new Set([ + (newItem as any).description || newItem.description, + ...this.ensureArray((newItem as any).evidence), + ...this.ensureArray((existingItem as any as any).evidence), + ]), + ].slice(0, 10); + (combined as any).weakAlpha = 1; + (combined as any).weakBeta = 1; + (combined as any).weakHitCount = 0; + (combined as any).lastWeakHitAt = null; + (combined as any).lastMatchTime = Date.now(); + (combined as any).centroid = updatedCentroid; + (combined as any).anchor = anchor; + (combined as any).driftBelowCount = driftBelowCount; + this.syncConfidence(combined as any); + + existing[top1Idx] = combined; + matchCount++; + + const driftInfo = + anchor && anchor.length === updatedCentroid.length + ? { + driftScore: + Math.round(cosineSimilarityNumbers(updatedCentroid, anchor) * 100) / 100, + driftBelowCount, + } + : {}; + + log("profile matched: same-cat forced merge", { + type: itemType, + idx: top1Idx, + cat: existingItem.category, + score: Math.round(top1Score * 100) / 100, + confidence: `${Math.round(((existingItem as any).confidence || 0) * 100) / 100}→${Math.round((combined as any).confidence * 100) / 100}`, + frequency: `${(existingItem as any).frequency || 1}→${(combined as any).frequency || "?"}`, + ...driftInfo, + }); + + const newFreq = (combined as any).frequency || 1; + if ( + this.isMilestone(newFreq) && + (combined as any).evidence?.length >= this.minEvidenceForEvolve(itemType) + ) { + await this.evolveAndUpdate(combined as any, itemType, profileId); + } + + continue; + } + + const weakAlpha = ((existingItem as any).weakAlpha || 1) + top1Score; + const weakBeta = ((existingItem as any).weakBeta || 1) + (1 - top1Score); + const effectiveAlpha = weakAlpha; + const effectiveBeta = weakBeta; + let upgraded = false; + if (effectiveAlpha + effectiveBeta > 7) { + upgraded = effectiveAlpha / (effectiveAlpha + effectiveBeta) >= 0.45; + } else { + upgraded = sampleBeta(effectiveAlpha, effectiveBeta) >= 0.5; + } + if (upgraded) { + const oldFreq = (existingItem as any).frequency || 1; + const centroid = (existingItem as any).centroid as number[]; + const anchor = (existingItem as any).anchor as number[]; + const newEmbArr = Array.from(newEmb); + const updatedCentroid = l2Normalize( + centroid.map( + (v, i) => + CENTROID_EMA_WEIGHT * v + CENTROID_EMA_WEIGHT_COMPLEMENT * (newEmbArr[i] ?? 0) + ) + ); + + const combined = this.mergeConfirmedMatch(existingItem, newItem, itemType, false) as T; + (combined as any).centroid = updatedCentroid; + (combined as any).anchor = anchor; + (combined as any).alpha += 0.5; + (combined as any).weakAlpha = 1; + (combined as any).weakBeta = 1; + (combined as any).driftBelowCount = (existingItem as any).driftBelowCount || 0; + this.syncConfidence(combined as any); + + existing[top1Idx] = combined; + matchCount++; + log("profile matched: weak upgrade (thompson)", { + type: itemType, + idx: top1Idx, + cat: existingItem.category, + score: Math.round(top1Score * 100) / 100, + weakAlpha: Math.round(weakAlpha * 100) / 100, + weakBeta: Math.round(weakBeta * 100) / 100, + forced: effectiveAlpha + effectiveBeta > 7, + frequency: `${oldFreq}→${(combined as any).frequency || "?"}`, + incomingDesc: (newItem.description || "").substring(0, 40), + }); + + const newFreq = (combined as any).frequency || 1; + if ( + this.isMilestone(newFreq) && + (combined as any).evidence?.length >= this.minEvidenceForEvolve(itemType) + ) { + await this.evolveAndUpdate(combined as any, itemType, profileId); + } + + continue; + } + if (effectiveAlpha + effectiveBeta <= 7) { + (existingItem as any).weakAlpha = effectiveAlpha; + (existingItem as any).weakBeta = effectiveBeta; + } else { + (existingItem as any).weakAlpha = 1; + (existingItem as any).weakBeta = 1; + } + (existingItem as any).lastSeen = Date.now(); + (existingItem as any).lastWeakHitAt = Date.now(); + log("profile weak hit (thompson)", { + type: itemType, + idx: top1Idx, + cat: existingItem.category, + score: Math.round(top1Score * 100) / 100, + weakAlpha: Math.round(weakAlpha * 100) / 100, + weakBeta: Math.round(weakBeta * 100) / 100, + forcedReset: effectiveAlpha + effectiveBeta > 7, + incomingDesc: (newItem.description || "").substring(0, 40), + }); + continue; + } + } + + if (!useEmbedding && newItem.description.length >= minDescLen) { + (this.coldBuffer as any)[itemType + "s"].push(newItem); + if ((this.coldBuffer as any)[itemType + "s"].length > 50) { + (this.coldBuffer as any)[itemType + "s"].shift(); + } + this.saveColdBuffer(); + log("profile cold start: buffered", { + type: itemType, + cat: newItem.category, + bufferSize: (this.coldBuffer as any)[itemType + "s"].length, + }); + continue; + } + + let initCentroid: number[] | undefined; + let initAnchor: number[] | undefined; + if (useEmbedding && newItem.description.length >= minDescLen) { + try { + const emb = await embed.embed(normalizeDescription(newItem.description)); + initCentroid = Array.from(emb); + initAnchor = initCentroid; + } catch {} } - merged.workflows.sort((a, b) => (b.frequency || 0) - (a.frequency || 0)); - merged.workflows = merged.workflows.slice(0, CONFIG.userProfileMaxWorkflows); + existing.push(this.initItem(newItem, itemType, initCentroid, initAnchor) as T); + newCount++; + log("profile no match: appended new", { + type: itemType, + cat: newItem.category, + desc: (newItem.description || "").substring(0, 40), + }); } - return merged; + log("profile merge done", { + type: itemType, + matched: matchCount, + appended: newCount, + existingAfter: existing.length, + }); + + return existing as T[]; + } + + /** + * Merge a new confirmed observation into an existing profile entry. + * ONLY call for confirmed matches (exact, strong sameCat/crossCat, Thompson upgrade). + * Do NOT call for forced merge or cross-validation — those handle fields directly. + */ + private mergeConfirmedMatch( + existing: any, + newItem: any, + itemType: string, + sameCat: boolean + ): any { + this.lazyMigrateAlpha(existing); + const evidence = [ + ...new Set([ + newItem.description, + ...this.ensureArray(newItem.evidence), + ...this.ensureArray(existing.evidence), + ]), + ].slice(0, 10); + + if (sameCat) { + const result = { + ...existing, + alpha: existing.alpha + 1, + beta: existing.beta ?? 1, + weakAlpha: 1, + weakBeta: 1, + frequency: (existing.frequency || 1) + 1, + evidence, + lastSeen: Date.now(), + lastMatchTime: Date.now(), + ...(itemType === "workflow" && newItem.steps?.length ? { steps: newItem.steps } : {}), + }; + this.syncConfidence(result); + return result; + } + + const result = { + ...existing, + alpha: existing.alpha + 0.5, + beta: existing.beta ?? 1, + weakAlpha: 1, + weakBeta: 1, + evidence, + lastSeen: Date.now(), + lastMatchTime: Date.now(), + ...(itemType === "workflow" && newItem.steps?.length ? { steps: newItem.steps } : {}), + }; + this.syncConfidence(result); + return result; + } + + private initItem(newItem: any, itemType: string, centroid?: number[], anchor?: number[]): any { + const hasEvidence = Array.isArray(newItem.evidence) && newItem.evidence.length > 0; + const item = { + ...newItem, + alpha: hasEvidence ? 1 : 0.3, + beta: hasEvidence ? 1 : 1.5, + weakAlpha: 1, + weakBeta: 1, + confidence: hasEvidence ? (newItem.confidence ?? 0.5) : 0.2, + frequency: 1, + lastSeen: Date.now(), + lastMatchTime: Date.now(), + firstSeen: Date.now(), + evidence: newItem.evidence ?? [], + weakHitCount: 0, + driftBelowCount: 0, + pendingValidation: true, + centroid, + anchor, + ...(itemType === "workflow" && newItem.steps?.length ? { steps: newItem.steps } : {}), + }; + this.syncConfidence(item); + return item; + } + + /** + * Three-way merge: item1 + item2 + newItem → single combined entry. + * Called when top-1 and top-2 both strongly match the same new observation + * within the same category, confirming they describe the same behavior. + */ + private combineThree( + item1: any, + item2: any, + newItem: any, + itemType: string, + newEmb: number[], + top1Score: number + ): any { + this.lazyMigrateAlpha(item1); + this.lazyMigrateAlpha(item2); + const freq = (item1.frequency || 1) + (item2.frequency || 1) + 1; + const evidenceSources = [ + ...this.ensureArray(newItem.evidence), + ...this.ensureArray(item1.evidence), + ...this.ensureArray(item2.evidence), + ]; + const evidence = [...new Set(evidenceSources)].slice(0, 10); + const centroid = l2Normalize( + item1.centroid.map( + (v: number, i: number) => + THREE_WAY_CENTROID_W1 * v + + THREE_WAY_CENTROID_W2 * (item2.centroid[i] ?? 0) + + THREE_WAY_CENTROID_W3 * (newEmb[i] ?? 0) + ) + ); + const anchor = item1.anchor; + const newIsShorter = newItem.description.length < (item1.description?.length || Infinity); + const description = top1Score > 0.9 && newIsShorter ? newItem.description : item1.description; + const steps = + itemType === "workflow" && (item1.steps || item2.steps) + ? item1.steps?.length && item1.steps.length >= (item2.steps?.length || 0) + ? item1.steps + : item2.steps + : newItem.steps; + const result = { + ...item1, + description, + frequency: freq, + evidence, + centroid, + anchor, + alpha: (item1.alpha || 1) + (item2.alpha || 1) + 1, + beta: (item1.beta || 1) + (item2.beta || 1), + weakAlpha: 1, + weakBeta: 1, + weakHitCount: 0, + driftBelowCount: 0, + lastSeen: Date.now(), + lastMatchTime: Date.now(), + ...(steps ? { steps } : {}), + }; + this.syncConfidence(result); + return result; } private ensureArray(val: any): any[] { @@ -395,62 +1168,659 @@ export class UserProfileManager { try { const parsed = JSON.parse(val); return Array.isArray(parsed) ? parsed : []; - } catch { + } catch (e) { + log("ensureArray: failed to parse JSON string, returning []", { + val: String(val).substring(0, 100), + error: String(e), + }); return []; } } return Array.isArray(val) ? val : []; } - private normalizeProfileData(profileData: UserProfileData, now: number): UserProfileData { - return { - preferences: safeArray(profileData.preferences).map((pref: any) => ({ - ...pref, - confidence: this.normalizeConfidence(pref.confidence), - evidence: this.ensureArray(pref.evidence), - lastUpdated: this.isValidTimestamp(pref.lastUpdated) ? pref.lastUpdated : now, - })), - patterns: safeArray(profileData.patterns).map((pattern: any) => ({ - ...pattern, - frequency: this.normalizePositiveNumber(pattern.frequency, 1), - lastSeen: this.isValidTimestamp(pattern.lastSeen) ? pattern.lastSeen : now, - })), - workflows: safeArray(profileData.workflows).map((workflow: any) => ({ - ...workflow, - frequency: this.normalizePositiveNumber(workflow.frequency, 1), - })), - }; + private isMilestone(frequency: number): boolean { + if (frequency <= 20) return [2, 5, 10, 20].includes(frequency); + return frequency % 20 === 0; + } + + private minEvidenceForEvolve(_itemType: string): number { + return 4; + } + + syncConfidence(item: any): void { + const alpha = item.alpha ?? 1; + const beta = item.beta ?? 1; + const betaMean = alpha / (alpha + beta); + const decayThreshold = CONFIG.userProfileConfidenceDecayDays * 24 * 60 * 60 * 1000; + const freq = item.frequency || 1; + const halfLife = decayThreshold * (1 + Math.log2(1 + freq)); + const age = Date.now() - (item.lastSeen || Date.now()); + const ageMs = Math.max(0, age - decayThreshold); + const timeFactor = Math.exp((-Math.LN2 * ageMs) / halfLife); + const matchTime = item.lastMatchTime || item.lastSeen || Date.now(); + const matchAge = (Date.now() - matchTime) / (24 * 60 * 60 * 1000); + const trendMultiplier = 0.8 + 0.2 * Math.exp(-matchAge / 30); + item.confidence = betaMean * Math.min(timeFactor, trendMultiplier); + } + + private lazyMigrateAlpha(item: any): void { + const needsAlphaMigration = + item.alpha === undefined || (item.alpha === 0.5 && item.beta === 1.5); + if (needsAlphaMigration) { + const conf = item.confidence ?? 1.0; + if (conf >= 1.0) { + item.alpha = 1 + (item.frequency || 1) * 0.5; + } else { + item.alpha = conf / (1 - conf + 0.01); + } + item.beta = 1; + } + item.weakAlpha = item.weakAlpha ?? 1; + item.weakBeta = item.weakBeta ?? 1; + item.lastMatchTime = item.lastMatchTime ?? item.lastSeen ?? Date.now(); + item.firstSeen = item.firstSeen ?? item.lastSeen ?? Date.now(); + if (item.pendingValidation === undefined) { + item.pendingValidation = false; + } + this.syncConfidence(item); + } + + private async detectConflicts(items: any[], itemType: string, profileId: string): Promise { + const candidates: { a: any; b: any; cos: number }[] = []; + const limit = items.length; + for (let i = 0; i < limit; i++) { + for (let j = i + 1; j < limit; j++) { + if (items[i].category !== items[j].category) continue; + const c1 = items[i].centroid as number[] | undefined; + const c2 = items[j].centroid as number[] | undefined; + if (!c1 || !c2) continue; + const cos = cosineSimilarityNumbers(c1, c2); + if (cos >= 0.65 && cos < 0.9) { + candidates.push({ a: items[i], b: items[j], cos }); + } + } + } + if (candidates.length === 0) return; + const maxChecks = 3; + candidates.sort((x, y) => y.cos - x.cos); + let checked = 0; + const removeIndices: number[] = []; + for (const { a, b, cos } of candidates) { + if (checked >= maxChecks) break; + if (removeIndices.includes(items.indexOf(a)) || removeIndices.includes(items.indexOf(b))) + continue; + const conflict = await this.checkConflict(a.description, b.description); + if (conflict) { + checked++; + const keeper = (a.frequency || 0) >= (b.frequency || 0) ? a : b; + const removed = keeper === a ? b : a; + this.lazyMigrateAlpha(keeper); + this.lazyMigrateAlpha(removed); + const oldAlpha = keeper.alpha || 1; + keeper.alpha *= 0.75; + keeper.beta = (keeper.beta || 1) + 0.25 * oldAlpha; + keeper.alpha += removed.alpha || 0; + keeper.frequency = (keeper.frequency || 0) + (removed.frequency || 0); + keeper.evidence = [ + ...new Set([...this.ensureArray(keeper.evidence), ...this.ensureArray(removed.evidence)]), + ].slice(0, 10); + if (itemType === "workflow") { + keeper.steps = + keeper.steps?.length >= (removed.steps?.length || 0) ? keeper.steps : removed.steps; + } + const removedIdx = items.indexOf(removed); + if (removedIdx >= 0) removeIndices.push(removedIdx); + this.syncConfidence(keeper); + log("profile conflict detected: resolved", { + type: itemType, + keeper: (keeper.description || "").substring(0, 40), + removed: (removed.description || "").substring(0, 40), + cos: Math.round(cos * 100) / 100, + alphaDrop: `${Math.round(oldAlpha)}→${Math.round(keeper.alpha)}`, + }); + } + } + for (let i = removeIndices.length - 1; i >= 0; i--) { + items.splice(removeIndices[i]!, 1); + } + } + + private async checkSemanticDuplicate(descA: string, descB: string): Promise { + const prompt = `Do these two descriptions refer to the same user behavior, preference, or pattern? Answer only whether they are semantically equivalent (same meaning, different wording). + +A: "${descA}" +B: "${descB}" + +Answer JSON only: { "duplicate": true|false, "reason": "one sentence explanation" }`; + + if (CONFIG.opencodeProvider && CONFIG.opencodeModel) { + try { + const { z } = await import("zod"); + const { generateStructuredOutput } = await import("../ai/opencode-provider.js"); + const { getOpenCodeClient } = await import("../ai/profile-llm-client.js"); + + let v2Client; + try { + v2Client = await getOpenCodeClient(); + } catch (e) { + log("profile dedup check: native provider not connected", { error: String(e) }); + } + + if (v2Client) { + const result: any = await Promise.race([ + generateStructuredOutput({ + client: v2Client, + providerID: CONFIG.opencodeProvider, + modelID: CONFIG.opencodeModel, + systemPrompt: "You are a semantic duplicate detector. Output valid JSON.", + userPrompt: prompt, + schema: z.object({ duplicate: z.boolean(), reason: z.string() }), + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error("dedup check timeout")), 30000) + ), + ]); + return result.duplicate || false; + } + } catch (e) { + log("profile dedup check: native provider failed", { error: String(e) }); + } + } + + if (CONFIG.memoryModel && CONFIG.memoryApiUrl) { + try { + const response = await fetch(`${CONFIG.memoryApiUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${CONFIG.memoryApiKey || ""}`, + }, + body: JSON.stringify({ + model: CONFIG.memoryModel, + messages: [ + { + role: "system", + content: "You are a semantic duplicate detector. Output valid JSON.", + }, + { role: "user", content: prompt }, + ], + temperature: 0, + response_format: { type: "json_object" }, + }), + signal: AbortSignal.timeout(30000), + }); + if (!response.ok) return false; + const data: any = await response.json(); + const content = data.choices?.[0]?.message?.content; + if (!content) return false; + const parsed = JSON.parse(content); + return parsed.duplicate || false; + } catch (e) { + log("profile dedup check: external API failed", { error: String(e) }); + } + } + + return false; + } + + private async deduplicateItems( + items: any[], + itemType: string, + profileId?: string + ): Promise { + if (!profileId || items.length < 2) return; + + const checkedPairs = this.dedupCheckedCache; + const maxLLMCalls = 5; + let llmCalls = 0; + let skippedCached = 0; + + const byCategory = new Map(); + for (const item of items) { + const cat = item.category || "_uncategorized"; + if (!byCategory.has(cat)) byCategory.set(cat, []); + byCategory.get(cat)!.push(item); + } + + for (const [, group] of byCategory) { + if (group.length < 2) continue; + + const candidates: { a: any; b: any; cos: number }[] = []; + for (let i = 0; i < group.length; i++) { + for (let j = i + 1; j < group.length; j++) { + const c1 = group[i].centroid as number[] | undefined; + const c2 = group[j].centroid as number[] | undefined; + if (!c1 || !c2) continue; + const cos = cosineSimilarityNumbers(c1, c2); + if (cos >= 0.5) { + candidates.push({ a: group[i], b: group[j], cos }); + } + } + } + + if (candidates.length === 0) continue; + candidates.sort((x, y) => y.cos - x.cos); + + const removeIndices: number[] = []; + let candidateIdx = 0; + for (const { a, b, cos } of candidates) { + candidateIdx++; + if (llmCalls >= maxLLMCalls) { + log("profile dedup: LLM call limit reached", { + itemType, + skipped: candidates.length - candidateIdx + 1, + }); + break; + } + if (removeIndices.includes(items.indexOf(a)) || removeIndices.includes(items.indexOf(b))) + continue; + + const pairKey = [a.description, b.description].sort().join("||").substring(0, 100); + if (checkedPairs.has(pairKey)) { + skippedCached++; + continue; + } + + llmCalls++; + const isDuplicate = await this.checkSemanticDuplicate(a.description, b.description); + + if (isDuplicate) { + const keeper = (a.frequency || 0) >= (b.frequency || 0) ? a : b; + const removed = keeper === a ? b : a; + keeper.frequency = (keeper.frequency || 0) + (removed.frequency || 0); + this.lazyMigrateAlpha(keeper); + this.lazyMigrateAlpha(removed); + keeper.alpha += removed.alpha || 0; + keeper.beta = (keeper.beta || 1) + (removed.beta || 1); + keeper.weakAlpha = (keeper.weakAlpha || 1) + ((removed.weakAlpha || 1) - 1); + keeper.weakBeta = (keeper.weakBeta || 1) + ((removed.weakBeta || 1) - 1); + keeper.lastSeen = Math.max(keeper.lastSeen || 0, removed.lastSeen || 0); + keeper.lastMatchTime = Math.max(keeper.lastMatchTime || 0, removed.lastMatchTime || 0); + this.syncConfidence(keeper); + keeper.evidence = [ + ...new Set([ + ...this.ensureArray(keeper.evidence), + ...this.ensureArray(removed.evidence), + ]), + ].slice(0, 10); + keeper.pendingValidation = !!keeper.pendingValidation && !!removed.pendingValidation; + if (itemType === "workflow") { + keeper.steps = + keeper.steps?.length >= (removed.steps?.length || 0) ? keeper.steps : removed.steps; + } + const removedIdx = items.indexOf(removed); + if (removedIdx >= 0) removeIndices.push(removedIdx); + checkedPairs.delete(pairKey); + log("profile dedup: merged duplicate", { + type: itemType, + keeper: (keeper.description || "").substring(0, 40), + removed: (removed.description || "").substring(0, 40), + cos: Math.round(cos * 100) / 100, + mergedFreq: keeper.frequency, + }); + } else { + checkedPairs.add(pairKey); + log("profile dedup: no duplicate confirmed", { + type: itemType, + cos: Math.round(cos * 100) / 100, + }); + } + } + + for (let i = removeIndices.length - 1; i >= 0; i--) { + items.splice(removeIndices[i]!, 1); + } + } + + log("profile dedup complete", { + type: itemType, + llmCalls, + skippedCached, + itemsAfter: items.length, + }); + + if (this.dedupCheckedCache.size > 1000) { + const entries = [...this.dedupCheckedCache]; + this.dedupCheckedCache = new Set(entries.slice(500)); + } } - private preferenceLastUpdated(pref: any, profile: UserProfile, fallback: number): number { - if (this.isValidTimestamp(pref.lastUpdated)) { - return pref.lastUpdated; + private async checkConflict(descA: string, descB: string): Promise { + const prompt = `Are these two user preferences contradictory (opposing, incompatible)? + +A: "${descA}" +B: "${descB}" + +Answer JSON only: { "conflict": true|false, "reason": "one sentence explanation" }`; + + if (CONFIG.opencodeProvider && CONFIG.opencodeModel) { + try { + const { z } = await import("zod"); + const { generateStructuredOutput } = await import("../ai/opencode-provider.js"); + const { getOpenCodeClient } = await import("../ai/profile-llm-client.js"); + + let v2Client; + try { + v2Client = await getOpenCodeClient(); + } catch (e) { + log("profile conflict check: native provider not connected", { error: String(e) }); + } + + if (v2Client) { + const result: any = await Promise.race([ + generateStructuredOutput({ + client: v2Client, + providerID: CONFIG.opencodeProvider, + modelID: CONFIG.opencodeModel, + systemPrompt: "You are a preference contradiction detector. Output valid JSON.", + userPrompt: prompt, + schema: z.object({ conflict: z.boolean(), reason: z.string() }), + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error("conflict check timeout")), 30000) + ), + ]); + return result.conflict || false; + } + } catch (e) { + log("profile conflict check: native provider failed", { error: String(e) }); + } + } + + if (CONFIG.memoryModel && CONFIG.memoryApiUrl) { + try { + const response = await fetch(`${CONFIG.memoryApiUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${CONFIG.memoryApiKey || ""}`, + }, + body: JSON.stringify({ + model: CONFIG.memoryModel, + messages: [ + { + role: "system", + content: "You are a preference contradiction detector. Output valid JSON.", + }, + { role: "user", content: prompt }, + ], + temperature: 0, + response_format: { type: "json_object" }, + }), + signal: AbortSignal.timeout(30000), + }); + if (!response.ok) return false; + const data: any = await response.json(); + const content = data.choices?.[0]?.message?.content; + if (!content) return false; + const parsed = (() => { + try { + return JSON.parse(content); + } catch { + const repaired = content + .replace( + /([\u4e00-\u9fff\u3000-\u303f\uff00-\uffef])"([\u4e00-\u9fff\u3000-\u303f\uff00-\uffef])/g, + '$1\\"$2' + ) + .replace(/([\u4e00-\u9fff\u3000-\u303f\uff00-\uffef])"(?=\s*[,}\]])/g, '$1\\"'); + return JSON.parse(repaired); + } + })(); + return parsed.conflict || false; + } catch (e) { + log("profile conflict check: external API failed", { error: String(e) }); + } } - if (this.isValidTimestamp(profile.lastAnalyzedAt)) { - return profile.lastAnalyzedAt; + + return false; + } + + private async evolveDescription( + item: any, + itemType: "preference" | "pattern" | "workflow", + profileId?: string + ): Promise { + if (!profileId) { + log("profile description evolution blocked: no profileId"); + return null; } - if (this.isValidTimestamp(profile.createdAt)) { - return profile.createdAt; + + const evidence = item.evidence; + if (!Array.isArray(evidence) || evidence.length < this.minEvidenceForEvolve(itemType)) { + if (Array.isArray(evidence) && evidence.length > 0) { + log("profile description evolution skipped: insufficient evidence", { + type: itemType, + evidenceCount: evidence.length, + required: this.minEvidenceForEvolve(itemType), + }); + } + return null; } - return fallback; + + log("profile description evolution attempt", { + type: itemType, + desc: (item.description || "").substring(0, 40), + frequency: item.frequency, + evidenceCount: evidence.length, + }); + + const evidenceList = evidence.map((e: string, i: number) => `${i + 1}. ${e}`).join("\n"); + + const systemPrompt = `You are a user profile description optimizer. Return ONLY a JSON object with a single "description" field: {"description": "..."} +Based on multiple independent observations of the same user behavior, generate a more precise description. +Rules: +- Describe the user's behavioral tendency in general terms +- Natural length — not artificially shortened or inflated +- Same language as the observations +- Do not over-infer beyond what the evidence shows +- Do not include technical implementation details, parameter values, algorithm names, tool names, product names, library names, file paths, error messages, or transient conversation content`; + + const userPrompt = `Current description: ${item.description} + +Independent observations: +${evidenceList} + +Generate a concise, abstract description of the user's general behavioral tendency.`; + + let newDescription: string | null = null; + + if (CONFIG.opencodeProvider && CONFIG.opencodeModel) { + try { + newDescription = await this.callOpencodeProvider(systemPrompt, userPrompt); + } catch (e) { + log("profile description evolution: native provider failed, trying external API", { + error: String(e), + }); + } + } + + if (!newDescription && CONFIG.memoryModel && CONFIG.memoryApiUrl) { + try { + newDescription = await this.callExternalAPI(systemPrompt, userPrompt); + } catch (e) { + log("profile description evolution: external API failed", { error: String(e) }); + return null; + } + } + + if (!newDescription || newDescription === item.description) { + if (!newDescription) { + log("profile description evolution blocked: no provider available"); + } else { + log("profile description evolution skipped: no change", { type: itemType }); + } + return null; + } + + return newDescription; } - private normalizeConfidence(value: any): number { - if (typeof value !== "number" || !Number.isFinite(value)) { - return 0.5; + async evolveAndUpdate(item: any, itemType: string, profileId?: string): Promise { + if (!profileId) return; + try { + const evolved = await this.evolveDescription(item, itemType as any, profileId); + if (evolved && evolved !== item.description) { + const embed = EmbeddingService.getInstance(); + + if (embed.isWarmedUp) { + const evidence = this.ensureArray(item.evidence) + .filter((e: any) => typeof e === "string" && e.length >= 10) + .slice(0, 8); + + if (evidence.length >= 3) { + const oldDesc = item.description; + const evEmbs = await Promise.all( + evidence.map((e: string) => embed.embed(normalizeDescription(e))) + ); + const sumVec = evEmbs.reduce( + (acc: number[], e: any) => { + const arr = Array.from(e) as number[]; + return acc.map((v, i) => v + (arr[i] ?? 0)); + }, + new Array((evEmbs[0] as any).length).fill(0) as number[] + ); + const evCentroid = l2Normalize(sumVec); + + let adopted = false; + const oldEmb = await embed.embed(normalizeDescription(item.description)); + const newEmb = await embed.embed(normalizeDescription(evolved)); + const cosOld = cosineSimilarityNumbers(Array.from(oldEmb) as number[], evCentroid); + const cosNew = cosineSimilarityNumbers(Array.from(newEmb) as number[], evCentroid); + + if (cosNew < cosOld - DIRECTION_VALIDATION_TOLERANCE) { + log("profile description evolution rejected: direction drift", { + type: itemType, + cosOld: Math.round(cosOld * 1000) / 1000, + cosNew: Math.round(cosNew * 1000) / 1000, + }); + } else { + item.description = evolved; + adopted = true; + } + + if (adopted) { + const newCentroid = evCentroid; + const oldCentroid = (item as any).centroid as number[] | undefined; + (item as any).centroid = newCentroid; + (item as any).anchor = newCentroid; + (item as any).driftBelowCount = 0; + + if (oldCentroid) { + log("profile centroid rebuilt from evidence after evolve", { + type: itemType, + cosShift: + Math.round(cosineSimilarityNumbers(oldCentroid, newCentroid) * 1000) / 1000, + evidenceCount: evidence.length, + validated: evidence.length >= 3, + }); + } + } + + log("profile description evolved", { + type: itemType, + oldDescription: oldDesc, + newDescription: item.description, + adopted: oldDesc !== item.description, + frequency: item.frequency, + }); + return; + } + } else { + log("profile description evolution: embedding not warmed up, skipping validation", { + type: itemType, + }); + } + + const oldDesc = item.description; + item.description = evolved; + log("profile description evolved", { + type: itemType, + oldDescription: oldDesc, + newDescription: evolved, + adopted: true, + frequency: item.frequency, + }); + } + } catch (e) { + log("profile description evolve error", { type: itemType, error: String(e) }); } - return Math.min(1, Math.max(0, value)); } - private normalizePositiveNumber(value: any, fallback: number): number { - if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { - return fallback; + private async callOpencodeProvider( + systemPrompt: string, + userPrompt: string + ): Promise { + const { generateStructuredOutput } = await import("../ai/opencode-provider.js"); + const { getOpenCodeClient } = await import("../ai/profile-llm-client.js"); + + let v2Client; + try { + v2Client = await getOpenCodeClient(); + } catch (e) { + log("profile description evolution: native provider not connected", { + provider: CONFIG.opencodeProvider, + error: String(e), + }); + return null; } - return value; + + const { z } = await import("zod"); + const schema = z.object({ description: z.string() }); + + const result: any = await Promise.race([ + generateStructuredOutput({ + client: v2Client, + providerID: CONFIG.opencodeProvider!, + modelID: CONFIG.opencodeModel!, + systemPrompt, + userPrompt, + schema, + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error("evolve description timeout")), 120000) + ), + ]); + + return result.description || null; } - private isValidTimestamp(value: any): value is number { - return typeof value === "number" && Number.isFinite(value) && value > 0; + private async callExternalAPI(systemPrompt: string, userPrompt: string): Promise { + const t0 = Date.now(); + const response = await fetch(`${CONFIG.memoryApiUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${CONFIG.memoryApiKey || ""}`, + }, + body: JSON.stringify({ + model: CONFIG.memoryModel, + messages: [ + { role: "system", content: systemPrompt }, + { role: "user", content: userPrompt }, + ], + temperature: 0.3, + response_format: { type: "json_object" }, + }), + signal: AbortSignal.timeout(60000), + }); + + log("profile description evolution: external API http done", { + httpMs: Date.now() - t0, + status: response.status, + }); + + if (!response.ok) { + const text = await response.text().catch(() => ""); + throw new Error(`External API error: ${response.status} ${text}`); + } + + const data: any = await response.json(); + const content = data.choices?.[0]?.message?.content; + if (!content) throw new Error("No content in API response"); + + const parsed = JSON.parse(content); + return parsed.description || null; } } diff --git a/src/services/vector-backends/exact-scan-backend.ts b/src/services/vector-backends/exact-scan-backend.ts index f10ea80..5fd1eee 100644 --- a/src/services/vector-backends/exact-scan-backend.ts +++ b/src/services/vector-backends/exact-scan-backend.ts @@ -6,6 +6,7 @@ import type { VectorKind, } from "./types.js"; import type { ShardInfo } from "../sqlite/types.js"; +import { cosineSimilarity } from "../../utils/math.js"; interface RankedRow { id: string; @@ -27,7 +28,7 @@ export class ExactScanBackend implements VectorBackend { return rows .map((row) => ({ id: row.id, - distance: 1 - this.cosineSimilarity(row.vector, queryVector), + distance: 1 - cosineSimilarity(row.vector, queryVector), })) .sort((a, b) => a.distance - b.distance) .slice(0, limit); @@ -93,28 +94,4 @@ export class ExactScanBackend implements VectorBackend { return new Float32Array(value); } - - private cosineSimilarity(a: Float32Array, b: Float32Array): number { - if (a.length !== b.length) { - return 0; - } - - let dot = 0; - let magA = 0; - let magB = 0; - - for (let i = 0; i < a.length; i++) { - const av = a[i] ?? 0; - const bv = b[i] ?? 0; - dot += av * bv; - magA += av * av; - magB += bv * bv; - } - - if (magA === 0 || magB === 0) { - return 0; - } - - return dot / (Math.sqrt(magA) * Math.sqrt(magB)); - } } diff --git a/src/services/web-server-worker.ts b/src/services/web-server-worker.ts index 489f609..9e148c9 100644 --- a/src/services/web-server-worker.ts +++ b/src/services/web-server-worker.ts @@ -25,6 +25,9 @@ import { handleGetProfileChangelog, handleGetProfileSnapshot, handleRefreshProfile, + handleAICleanup, + handleApplyCleanup, + handleUpdateProfileItem, } from "./api-handlers.js"; const __filename = fileURLToPath(import.meta.url); @@ -244,6 +247,26 @@ async function handleRequest(req: Request): Promise { return jsonResponse(result); } + if (path === "/api/user-profile/ai-cleanup" && method === "POST") { + const body = (await req.json().catch(() => ({}))) as any; + const userId = body.userId || undefined; + const result = await handleAICleanup(userId); + return jsonResponse(result); + } + + if (path === "/api/user-profile/ai-cleanup/apply" && method === "POST") { + const body = (await req.json().catch(() => ({}))) as any; + const userId = body.userId || undefined; + const result = await handleApplyCleanup(userId, body); + return jsonResponse(result); + } + + if (path === "/api/user-profile/item" && method === "PATCH") { + const body = (await req.json().catch(() => ({}))) as any; + const result = await handleUpdateProfileItem(body); + return jsonResponse(result); + } + return new Response("Not Found", { status: 404 }); } catch (error) { return jsonResponse( diff --git a/src/services/web-server.ts b/src/services/web-server.ts index f490179..99ea4f8 100644 --- a/src/services/web-server.ts +++ b/src/services/web-server.ts @@ -28,6 +28,9 @@ import { handleGetProfileChangelog, handleGetProfileSnapshot, handleRefreshProfile, + handleAICleanup, + handleApplyCleanup, + handleUpdateProfileItem, } from "./api-handlers.js"; /** @@ -69,6 +72,17 @@ function serveFetch(opts: { // Bodies stream both directions via the WHATWG Streams ↔ Node Streams // helpers that ship with Node 18+. const server = createServer(async (req: IncomingMessage, res: ServerResponse) => { + let destroyed = false; + const cleanup = () => { + if (destroyed) return; + destroyed = true; + if (!res.writableEnded) res.destroy(); + if (!req.socket.destroyed) req.socket.destroy(); + }; + req.on("close", cleanup); + req.socket.on("error", cleanup); + req.socket.on("close", cleanup); + try { const url = `http://${opts.hostname}:${opts.port}${req.url ?? "/"}`; const method = req.method ?? "GET"; @@ -77,20 +91,23 @@ function serveFetch(opts: { method, headers: req.headers as Record, body: hasBody ? (Readable.toWeb(req) as unknown as ReadableStream) : undefined, - // `duplex: "half"` is required by Node fetch when sending a body - // stream. Cast keeps TS happy on older lib.dom.d.ts revisions. ...(hasBody ? ({ duplex: "half" } as Record) : {}), }); const webRes = await opts.fetch(webReq); + if (destroyed) return; res.statusCode = webRes.status; webRes.headers.forEach((value, name) => res.setHeader(name, value)); res.setHeader("Connection", "close"); if (webRes.body) { - Readable.fromWeb(webRes.body as unknown as Parameters[0]).pipe( - res + const src = Readable.fromWeb( + webRes.body as unknown as Parameters[0] ); + res.on("close", () => { + if (!src.destroyed) src.destroy(); + }); + src.pipe(res); } else { res.end(); } @@ -99,7 +116,9 @@ function serveFetch(opts: { res.statusCode = 500; res.setHeader("Content-Type", "text/plain"); } - res.end(`Internal Server Error: ${error instanceof Error ? error.message : String(error)}`); + if (!res.writableEnded) { + res.end(`Internal Server Error: ${error instanceof Error ? error.message : String(error)}`); + } } }); @@ -111,7 +130,9 @@ function serveFetch(opts: { listenError = err; } }); - server.listen({ port: opts.port, host: opts.hostname, reuseAddr: true }); + // exclusive: false disables SO_EXCLUSIVEADDRUSE on Windows, allowing + // rebind after a crashed predecessor left orphaned sockets behind. + server.listen({ port: opts.port, host: opts.hostname, reuseAddr: true, exclusive: false }); server.unref(); server.timeout = 30000; server.keepAliveTimeout = 10000; @@ -488,6 +509,29 @@ export class WebServer { return this.jsonResponse(result); } + if (path === "/api/user-profile/ai-cleanup" && method === "POST") { + const body = (await req.json().catch(() => ({}))) as any; + const userId = body.userId || undefined; + const includeIds = Array.isArray(body.includeIds) + ? (body.includeIds as string[]) + : undefined; + const result = await handleAICleanup(userId, includeIds); + return this.jsonResponse(result); + } + + if (path === "/api/user-profile/ai-cleanup/apply" && method === "POST") { + const body = (await req.json().catch(() => ({}))) as any; + const userId = body.userId || undefined; + const result = await handleApplyCleanup(userId, body); + return this.jsonResponse(result); + } + + if (path === "/api/user-profile/item" && method === "PATCH") { + const body = (await req.json().catch(() => ({}))) as any; + const result = await handleUpdateProfileItem(body); + return this.jsonResponse(result); + } + return new Response("Not Found", { status: 404 }); } catch (error) { return this.jsonResponse( diff --git a/src/utils/math.ts b/src/utils/math.ts new file mode 100644 index 0000000..1bfb0a5 --- /dev/null +++ b/src/utils/math.ts @@ -0,0 +1,52 @@ +export function cosineSimilarity(a: Float32Array, b: Float32Array): number { + if (a.length !== b.length) return 0; + + let dot = 0; + let magA = 0; + let magB = 0; + + for (let i = 0; i < a.length; i++) { + const av = a[i] ?? 0; + const bv = b[i] ?? 0; + dot += av * bv; + magA += av * av; + magB += bv * bv; + } + + if (magA === 0 || magB === 0) return 0; + + return dot / (Math.sqrt(magA) * Math.sqrt(magB)); +} + +/** + * Cosine similarity for number[] (JSON-stored centroids), otherwise + * identical to the Float32Array version above. + */ +export function cosineSimilarityNumbers(a: number[], b: number[]): number { + if (a.length !== b.length) return 0; + + let dot = 0; + let magA = 0; + let magB = 0; + + for (let i = 0; i < a.length; i++) { + const av = a[i] ?? 0; + const bv = b[i] ?? 0; + dot += av * bv; + magA += av * av; + magB += bv * bv; + } + + if (magA === 0 || magB === 0) return 0; + return dot / (Math.sqrt(magA) * Math.sqrt(magB)); +} + +export function l2Normalize(vec: number[]): number[] { + let norm = 0; + for (let i = 0; i < vec.length; i++) { + norm += (vec[i] ?? 0) * (vec[i] ?? 0); + } + norm = Math.sqrt(norm); + if (norm === 0) return vec; + return vec.map((v) => v / norm); +} diff --git a/src/utils/profile.ts b/src/utils/profile.ts new file mode 100644 index 0000000..9244a10 --- /dev/null +++ b/src/utils/profile.ts @@ -0,0 +1,15 @@ +export function isFrozen(item: any): boolean { + return (item.driftBelowCount || 0) >= 2; +} + +export function sortProfileItems(items: any[], metric: "confidence" | "frequency"): any[] { + return [...items].sort((a, b) => { + const aFrozen = isFrozen(a) ? 1 : 0; + const bFrozen = isFrozen(b) ? 1 : 0; + if (aFrozen !== bFrozen) return aFrozen - bFrozen; + if (metric === "confidence") { + return (b.confidence || 0) - (a.confidence || 0) || (b.frequency || 1) - (a.frequency || 1); + } + return (b.frequency || 0) - (a.frequency || 0); + }); +} diff --git a/src/web/app.js b/src/web/app.js index f216da8..de6c887 100644 --- a/src/web/app.js +++ b/src/web/app.js @@ -14,6 +14,7 @@ const state = { selectedMemories: new Set(), autoRefreshInterval: null, userProfile: null, + profilePages: { pref: 1, pat: 1, wf: 1 }, }; marked.setOptions({ @@ -31,9 +32,13 @@ function renderMarkdown(markdown) { async function fetchAPI(endpoint, options = {}) { try { const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 60000); + const timeoutMs = + options.timeout || + (options.method === "POST" && endpoint.includes("/ai-cleanup") ? 180000 : 60000); + const { timeout: _, ...fetchOptions } = options; + const timeoutId = setTimeout(() => controller.abort(), timeoutMs); const response = await fetch(API_BASE + endpoint, { - ...options, + ...fetchOptions, signal: controller.signal, }); clearTimeout(timeoutId); @@ -925,10 +930,39 @@ function renderUserProfile() { return flattened; }; + const PAGE_SIZE = 20; + function paginate(items, page, type) { + const total = items.length; + const totalPages = Math.max(1, Math.ceil(total / PAGE_SIZE)); + const start = (page - 1) * PAGE_SIZE; + const pageItems = items.slice(start, start + PAGE_SIZE); + if (total <= PAGE_SIZE) return { items: pageItems, controls: "" }; + if (page > totalPages) page = totalPages; + const pages = []; + for (let i = 1; i <= totalPages; i++) { + pages.push( + `` + ); + } + const controls = ` +
+ ${start + 1}-${Math.min(start + PAGE_SIZE, total)} / ${total} + ${pages.join("")} +
`; + return { items: pageItems, controls }; + } + const preferences = parseField(data.preferences); const patterns = parseField(data.patterns); const workflows = parseField(data.workflows); + if (!state.profilePages) { + state.profilePages = { pref: 1, pat: 1, wf: 1 }; + } + const pp = paginate(preferences, state.profilePages.pref, "pref"); + const pt = paginate(patterns, state.profilePages.pat, "pat"); + const pw = paginate(workflows, state.profilePages.wf, "wf"); + container.innerHTML = `
@@ -961,35 +995,46 @@ function renderUserProfile() { ? `

${t("empty-preferences")}

` : `
- ${preferences - .sort((a, b) => (b.confidence || 0) - (a.confidence || 0)) + ${pp.items .map( (p) => `
${escapeHtml(p.category || "General")} -
- ${Math.round((p.confidence || 0) * 100)}% +
+ + +
+
+ ${Math.round((p.confidence || 0) * 1000) / 10}%

${escapeHtml(p.description || "")}

- ${ - p.evidence && p.evidence.length > 0 - ? ` + ${ + p.evidence || p.frequency + ? `
+ 🎯 ${p.frequency || 1} + ${ + p.evidence && p.evidence.length > 0 + ? ` + · ${Array.isArray(p.evidence) ? p.evidence.length : 1} evidence - + ` + : "" + }
` - : "" - } + : "" + }
` ) .join("")}
+ ${pp.controls} ` }
@@ -1001,21 +1046,41 @@ function renderUserProfile() { ? `

${t("empty-patterns")}

` : `
- ${patterns + ${pt.items .map( (p) => `
${escapeHtml(p.category || "General")} +
+ + +
+
+ ${Math.round((p.confidence || 0) * 1000) / 10}% +

${escapeHtml(p.description || "")}

+
+ 🎯 ${p.frequency || 1} + ${ + p.evidence && p.evidence.length > 0 + ? ` + · + + ${Array.isArray(p.evidence) ? p.evidence.length : 1} evidence + ` + : "" + } +
` ) .join("")}
+ ${pt.controls} ` }
@@ -1027,11 +1092,21 @@ function renderUserProfile() { ? `

${t("empty-workflows")}

` : `
- ${workflows + ${pw.items + .sort((a, b) => (b.frequency || 0) - (a.frequency || 0)) .map( (w) => `
-
${escapeHtml(w.description || "")}
+
+
${escapeHtml(w.description || "")}
+
+ + +
+
+ ${Math.round((w.confidence || 0) * 1000) / 10}% +
+
${(w.steps || []) .map( @@ -1045,11 +1120,24 @@ function renderUserProfile() { ) .join("")}
+
+ 🎯 ${w.frequency || 1} + ${ + w.evidence && w.evidence.length > 0 + ? ` + · + + ${Array.isArray(w.evidence) ? w.evidence.length : 1} evidence + ` + : "" + } +
` ) .join("")}
+ ${pw.controls} ` }
@@ -1105,6 +1193,598 @@ async function refreshProfile() { showToast(result.error || t("toast-update-failed"), "error"); } } +async function showAICleanup() { + const modal = document.getElementById("ai-cleanup-modal"); + const loading = document.getElementById("cleanup-loading"); + const diffV2 = document.getElementById("cleanup-diff-v2"); + const applyBtn = document.getElementById("cleanup-apply-btn"); + const sections = document.getElementById("cleanup-sections"); + + modal.classList.remove("hidden"); + loading.classList.add("hidden"); + diffV2.classList.remove("hidden"); + diffV2.style.cssText = "display:flex;flex-direction:column;flex:1;min-height:0;overflow:hidden;"; + applyBtn.classList.add("hidden"); + document.getElementById("cleanup-toolbar")?.classList.add("hidden"); + document.getElementById("cleanup-kept-section")?.classList.add("hidden"); + + if (!state.userProfile?.profileData) { + showToast("No profile data loaded", "error"); + modal.classList.add("hidden"); + return; + } + + const pd = state.userProfile.profileData; + const allItems = [ + ...(pd.preferences || []).map((p, i) => ({ ...p, _id: `pref_${i}`, _type: "pref" })), + ...(pd.patterns || []).map((p, i) => ({ ...p, _id: `pat_${i}`, _type: "pat" })), + ...(pd.workflows || []).map((w, i) => ({ ...w, _id: `wf_${i}`, _type: "wf" })), + ]; + + let html = `
+

${t("label-ai-cleanup-select")}

+
+ + + + +
+
`; + + const cats = {}; + for (const it of allItems) { + const cat = it.category || "(none)"; + if (!cats[cat]) cats[cat] = { pref: [], pat: [], wf: [] }; + cats[cat][it._type === "wf" ? "wf" : it._type === "pat" ? "pat" : "pref"].push(it); + } + + html += `
`; + for (const cat of Object.keys(cats).sort()) { + const items = [...cats[cat].pref, ...cats[cat].pat, ...cats[cat].wf]; + html += `
+
+ ${escapeHtml(cat)} + ${items.length} items +
`; + for (const it of items) { + const freq = it.frequency || 0; + const conf = it.confidence; + html += ``; + } + html += `
`; + } + html += `
+
+ ${t("label-ai-cleanup-selected", { count: allItems.filter((it) => (it.frequency || 0) <= 3).length })} + +
`; + + sections.innerHTML = html; + + const getSelected = () => + [...document.querySelectorAll(".cleanup-sel-item:checked")].map((cb) => cb.dataset.id); + + const updateCount = () => { + document.getElementById("cleanup-sel-count").textContent = t("label-ai-cleanup-selected", { + count: getSelected().length, + }); + }; + + sections.addEventListener("change", (e) => { + if (e.target.classList.contains("cleanup-sel-item")) updateCount(); + }); + + document.getElementById("sel-all").addEventListener("click", () => { + document.querySelectorAll(".cleanup-sel-item").forEach((cb) => { + cb.checked = true; + }); + updateCount(); + }); + document.getElementById("sel-none").addEventListener("click", () => { + document.querySelectorAll(".cleanup-sel-item").forEach((cb) => { + cb.checked = false; + }); + updateCount(); + }); + document.getElementById("sel-low").addEventListener("click", () => { + document.querySelectorAll(".cleanup-sel-item").forEach((cb) => { + const it = allItems.find((x) => x._id === cb.dataset.id); + cb.checked = it && (it.frequency || 0) <= 3; + }); + updateCount(); + }); + document.getElementById("sel-same-cat").addEventListener("click", () => { + document.querySelectorAll(".cleanup-sel-item").forEach((cb) => { + cb.checked = false; + }); + // Select items from categories with 3+ items + for (const cat of Object.keys(cats)) { + const count = [...cats[cat].pref, ...cats[cat].pat, ...cats[cat].wf].length; + if (count >= 3) { + [...cats[cat].pref, ...cats[cat].pat, ...cats[cat].wf].forEach((it) => { + const cb = document.querySelector(`.cleanup-sel-item[data-id="${it._id}"]`); + if (cb) cb.checked = true; + }); + } + } + updateCount(); + }); + + document.getElementById("sel-analyze").addEventListener("click", async () => { + const ids = getSelected(); + if (ids.length === 0) { + showToast("No items selected", "warn"); + return; + } + sections.innerHTML = ""; + loading.classList.remove("hidden"); + try { + const result = await fetchAPI("/api/user-profile/ai-cleanup", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ includeIds: ids }), + timeout: 180000, + }); + loading.classList.add("hidden"); + if (!result.success) { + showToast(result.error || "Cleanup failed", "error"); + return; + } + renderCleanupDiffV2(result.data); + diffV2.classList.remove("hidden"); + state.pendingCleanup = result.data; + } catch (e) { + loading.classList.add("hidden"); + showToast("Cleanup failed: " + e.message, "error"); + } + }); + + setTimeout(() => applyLanguage?.(), 0); + lucide?.createIcons?.(); +} + +function renderCleanupDiffV2(data) { + const changes = data.changes; + const sections = document.getElementById("cleanup-sections"); + const diffV2 = document.getElementById("cleanup-diff-v2"); + diffV2.style.cssText = ""; + sections.style.cssText = ""; + + let html = renderMergeSection(changes.merged || [], data.old); + html += renderRemoveSection(changes.removed || [], data.old); + sections.innerHTML = html; + + renderKeptSection(changes.kept || []); + bindCleanupToolbar(); + + const applyBtn = document.getElementById("cleanup-apply-btn"); + applyBtn.classList.remove("hidden"); + applyBtn.textContent = t("label-ai-cleanup-apply") || "Apply Changes"; + + setTimeout(() => applyLanguage?.(), 0); + lucide?.createIcons?.(); +} + +function renderMergeSection(merged, old) { + if (merged.length === 0) return ""; + + let html = `

${t("label-ai-cleanup-merged-header", { count: merged.length })}

`; + + merged.forEach((m, mi) => { + const mainId = m.ids[0]; + const mergedFrom = m.ids.slice(1); + const mainDesc = m.result || ""; + const mainSteps = findStepsById(mainId, old); + const mergedDescs = mergedFrom + .map((id) => { + const desc = findDescById(id, old); + const typeLabel = getTypeLabel(id); + const steps = findStepsById(id, old); + return desc ? { desc, typeLabel, steps } : null; + }) + .filter(Boolean); + + const stepsAfter = mainSteps?.length + ? `
${renderStepsInline(mainSteps)}
` + : ""; + + html += `
+ +
+
+ ${mergedDescs + .map((d) => { + const stepsHtml = d.steps?.length + ? `
${renderStepsInline(d.steps)}
` + : ""; + return `
${d.typeLabel} ${escapeHtml(d.desc.substring(0, 80))}${d.desc.length > 80 ? "..." : ""}${stepsHtml}
`; + }) + .join("")} +
+
▶
+
${escapeHtml(mainDesc.substring(0, 120))}${mainDesc.length > 120 ? "..." : ""}${stepsAfter}
+
+
`; + }); + + return html; +} + +function renderRemoveSection(removed, old) { + if (removed.length === 0) return ""; + + let html = `

${t("label-ai-cleanup-removed-header", { count: removed.length })}

`; + + removed.forEach((r, ri) => { + const desc = findDescById(r.id, old); + const steps = findStepsById(r.id, old); + const stepsHtml = steps?.length + ? `
${renderStepsInline(steps)}
` + : ""; + html += `
+ +
+
${escapeHtml(desc || r.id)}${stepsHtml}
+
${escapeHtml(r.reason)}
+
+
`; + }); + + return html; +} + +function renderKeptSection(kept) { + const keptCount = document.getElementById("cleanup-kept-count"); + const keptList = document.getElementById("cleanup-kept-list"); + const keptItems = kept || []; + + keptCount.textContent = `(${keptItems.length})`; + keptList.innerHTML = keptItems + .map((k) => `
${escapeHtml(k)}
`) + .join(""); + + if (keptItems.length === 0) { + document.getElementById("cleanup-kept-section").classList.add("hidden"); + } else { + document.getElementById("cleanup-kept-section").classList.remove("hidden"); + keptList.classList.add("hidden"); + } + + document.getElementById("cleanup-kept-toggle").onclick = () => { + keptList.classList.toggle("hidden"); + const icon = document.getElementById("cleanup-kept-toggle").querySelector("i"); + if (icon) { + icon.setAttribute( + "data-lucide", + keptList.classList.contains("hidden") ? "chevron-down" : "chevron-up" + ); + } + lucide?.createIcons?.(); + }; +} + +function bindCleanupToolbar() { + document.querySelectorAll(".diff-checkbox").forEach((cb) => { + cb.addEventListener("change", updateCleanupStats); + }); + + updateCleanupStats(); + + document.getElementById("cleanup-select-all").onclick = () => { + document.querySelectorAll(".diff-checkbox").forEach((cb) => { + cb.checked = true; + }); + updateCleanupStats(); + }; + document.getElementById("cleanup-deselect-all").onclick = () => { + document.querySelectorAll(".diff-checkbox").forEach((cb) => { + cb.checked = false; + }); + updateCleanupStats(); + }; +} + +function getTypeLabel(id) { + if (typeof id !== "string" || !id.includes("_")) return "?"; + const prefix = id.split("_")[0]; + if (prefix === "pref") return t("profile-type-pref") || "Pref"; + if (prefix === "pat") return t("profile-type-pat") || "Pat"; + if (prefix === "wf") return t("profile-type-wf") || "Wf"; + return "?"; +} + +function findDescById(id, profileData) { + if (!profileData) return null; + if (typeof id === "string" && id.includes("_")) { + const parts = id.split("_"); + const prefix = parts[0]; + const idx = parseInt(parts[1], 10); + if (!isNaN(idx)) { + if (prefix === "pref" && profileData.preferences?.[idx]) { + return profileData.preferences[idx].description; + } + if (prefix === "pat" && profileData.patterns?.[idx]) { + return profileData.patterns[idx].description; + } + if (prefix === "wf" && profileData.workflows?.[idx]) { + return profileData.workflows[idx].description; + } + } + } + return null; +} + +function findStepsById(id, profileData) { + if (!profileData || typeof id !== "string" || !id.includes("_")) return null; + const parts = id.split("_"); + const idx = parseInt(parts[1], 10); + if (isNaN(idx) || parts[0] !== "wf") return null; + return profileData.workflows?.[idx]?.steps || null; +} + +function renderStepsInline(steps) { + return steps + .map( + (s, i) => + `${i + 1} ${escapeHtml(s)}` + ) + .join('→'); +} + +function updateCleanupStats() { + const checkboxes = document.querySelectorAll(".diff-checkbox"); + let selected = 0; + checkboxes.forEach((cb) => { + if (cb.checked) selected++; + }); + const total = checkboxes.length; + document.getElementById("cleanup-stats").textContent = t("label-ai-cleanup-changes-selected", { + selected, + total, + }); + const btn = document.getElementById("cleanup-apply-btn"); + if (btn) { + btn.textContent = + selected > 0 + ? `${t("label-ai-cleanup-apply")} (${selected})` + : t("label-ai-cleanup-apply") || "Apply"; + btn.disabled = selected === 0; + } +} + +async function applyAICleanup() { + const applyBtn = document.getElementById("cleanup-apply-btn"); + applyBtn.disabled = true; + + const acceptedMerged = []; + const acceptedRemoved = []; + document.querySelectorAll(".diff-checkbox").forEach((cb) => { + if (!cb.checked) return; + if (cb.dataset.type === "merged") { + const mi = parseInt(cb.dataset.index, 10); + const change = state.pendingCleanup?.changes?.merged?.[mi]; + if (change) acceptedMerged.push(change.ids); + } else if (cb.dataset.type === "removed") { + const ri = parseInt(cb.dataset.index, 10); + const change = state.pendingCleanup?.changes?.removed?.[ri]; + if (change) acceptedRemoved.push(change.id); + } + }); + + try { + const result = await fetchAPI("/api/user-profile/ai-cleanup/apply", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + profile: state.pendingCleanup?.new, + acceptedMerged, + acceptedRemoved, + }), + }); + + if (result.success) { + showToast(t("toast-cleanup-success"), "success"); + closeCleanupModal(); + delete state.pendingCleanup; + await loadUserProfile(); + } else { + showToast(result.error || t("toast-cleanup-apply-failed"), "error"); + } + } catch (e) { + showToast(t("toast-cleanup-apply-failed") + ": " + e.message, "error"); + } + + applyBtn.disabled = false; +} + +function closeCleanupModal() { + document.getElementById("ai-cleanup-modal").classList.add("hidden"); + delete state.pendingCleanup; +} + +function showProfileItemModal(type, index, action) { + const current = getCurrentItem(type, index); + if (!current) return; + + const isEdit = action === "edit"; + const isWorkflow = type === "workflows"; + const modal = document.getElementById("profile-item-modal"); + + document.getElementById("profile-item-modal-title").textContent = isEdit + ? t("btn-edit") || "Edit Item" + : t("confirm-delete") || "Delete Item?"; + document.getElementById("profile-item-category").value = current.category || ""; + document.getElementById("profile-item-category").disabled = !isEdit; + document.getElementById("profile-item-description").value = current.description || ""; + document.getElementById("profile-item-description").disabled = !isEdit; + document.getElementById("profile-item-category").closest(".form-group").style.display = isWorkflow + ? "none" + : "block"; + document.getElementById("profile-item-save").textContent = isEdit + ? t("btn-save") || "Save" + : t("btn-delete") || "Delete"; + + const stepsSection = document.getElementById("steps-section"); + if (isWorkflow && isEdit) { + stepsSection.style.display = "block"; + renderStepsEditor(current.steps || []); + } else { + stepsSection.style.display = "none"; + } + + const saveBtn = document.getElementById("profile-item-save"); + if (isEdit) saveBtn.classList.remove("danger"); + else saveBtn.classList.add("danger"); + + modal.dataset.deleteStep = "1"; + modal.classList.remove("hidden"); + modal._profileAction = { type, index, action }; +} + +async function editProfileItem(type, index) { + showProfileItemModal(type, index, "edit"); +} + +async function deleteProfileItem(type, index) { + showProfileItemModal(type, index, "delete"); +} + +function submitProfileItemForm(e) { + e.preventDefault(); + const modal = document.getElementById("profile-item-modal"); + const { type, index, action } = modal._profileAction || {}; + if (!type) return; + + if (action === "edit") { + submitProfileEdit(type, index); + } else if (action === "delete") { + if (modal.dataset.deleteStep === "2") { + submitProfileDelete(type, index); + } else { + showDeleteConfirmation(); + } + } +} + +async function submitProfileEdit(type, index) { + const isWorkflow = type === "workflows"; + const category = isWorkflow ? undefined : document.getElementById("profile-item-category").value; + const description = document.getElementById("profile-item-description").value; + const body = { type, index, action: "edit", category, description }; + + if (type === "workflows") { + body.steps = collectSteps(); + } + + const result = await fetchAPI("/api/user-profile/item", { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (result.success) { + showToast(t("toast-update-success") || "Item updated", "success"); + closeProfileItemModal(); + await loadUserProfile(); + } else { + showToast(result.error || t("toast-update-failed") || "Update failed", "error"); + } +} + +function showDeleteConfirmation() { + document.getElementById("profile-item-modal-title").textContent = + t("confirm-delete-title") || "Confirm Delete"; + document.getElementById("profile-item-category").closest(".form-group").style.display = "none"; + document.getElementById("profile-item-description").closest(".form-group").style.display = "none"; + document.getElementById("profile-item-save").textContent = t("btn-delete") || "Delete"; + document.getElementById("profile-item-save").classList.add("danger"); + document.getElementById("profile-item-modal").dataset.deleteStep = "2"; +} + +async function submitProfileDelete(type, index) { + const result = await fetchAPI("/api/user-profile/item", { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ type, index, action: "delete" }), + }); + + if (result.success) { + showToast(t("toast-delete-success") || "Item deleted", "success"); + closeProfileItemModal(); + await loadUserProfile(); + } else { + showToast(result.error || t("toast-delete-failed") || "Delete failed", "error"); + } +} + +function closeProfileItemModal() { + const modal = document.getElementById("profile-item-modal"); + modal.classList.add("hidden"); + document.getElementById("profile-item-category").disabled = false; + document.getElementById("profile-item-category").closest(".form-group").style.display = "block"; + document.getElementById("profile-item-description").disabled = false; + document.getElementById("profile-item-description").closest(".form-group").style.display = + "block"; + document.getElementById("steps-section").style.display = "none"; + const saveBtn = document.getElementById("profile-item-save"); + saveBtn.classList.remove("danger"); + modal.dataset.deleteStep = "1"; + delete modal._profileAction; +} + +function getCurrentItem(type, index) { + if (!state.userProfile?.profileData) return null; + return state.userProfile.profileData[type]?.[index] || null; +} + +function renderStepsEditor(steps) { + const container = document.getElementById("steps-container"); + container.innerHTML = ""; + (steps || []).forEach((step, i) => addStepRow(step, i)); + if (!steps?.length) addStepRow(""); +} + +function addStepRow(text = "") { + const container = document.getElementById("steps-container"); + const i = container.children.length; + const row = document.createElement("div"); + row.className = "step-row"; + row.innerHTML = `${i + 1} + + `; + row.querySelector(".btn-remove-step").addEventListener("click", () => removeStepRow(row)); + container.appendChild(row); +} + +function removeStepRow(row) { + row.remove(); + updateStepNumbers(); +} + +function updateStepNumbers() { + const rows = document.querySelectorAll("#steps-container .step-row"); + rows.forEach((row, i) => { + row.querySelector(".step-num").textContent = i + 1; + row.querySelector(".step-input").placeholder = `Step ${i + 1}...`; + }); +} + +function collectSteps() { + return [...document.querySelectorAll("#steps-container .step-input")] + .map((el) => el.value.trim()) + .filter(Boolean); +} function switchView(view) { state.currentView = view; @@ -1154,6 +1834,7 @@ document.addEventListener("DOMContentLoaded", async () => { }); document.getElementById("lang-toggle").textContent = getLanguage().toUpperCase(); + setLanguage(getLanguage()); document.getElementById("tag-filter").addEventListener("change", () => { state.selectedTag = document.getElementById("tag-filter").value; @@ -1202,6 +1883,64 @@ document.addEventListener("DOMContentLoaded", async () => { if (e.target.id === "edit-modal") closeModal(); }); + document.getElementById("ai-cleanup-btn")?.addEventListener("click", showAICleanup); + document.getElementById("cleanup-modal-close")?.addEventListener("click", closeCleanupModal); + document.getElementById("cleanup-cancel-btn")?.addEventListener("click", closeCleanupModal); + document.getElementById("cleanup-apply-btn")?.addEventListener("click", applyAICleanup); + + // Re-render diff on language change if visible + document.addEventListener("langchange", () => { + if ( + state.pendingCleanup && + !document.getElementById("ai-cleanup-modal").classList.contains("hidden") + ) { + const diffV2 = document.getElementById("cleanup-diff-v2"); + if (!diffV2.classList.contains("hidden")) { + // Save checkbox states + const checkedStates = []; + document.querySelectorAll(".diff-checkbox").forEach((cb, i) => { + checkedStates[i] = cb.checked; + }); + renderCleanupDiffV2(state.pendingCleanup); + // Restore checkbox states + document.querySelectorAll(".diff-checkbox").forEach((cb, i) => { + if (checkedStates[i] !== undefined) cb.checked = checkedStates[i]; + }); + updateCleanupStats(); + } + } + }); + + document.getElementById("profile-item-form")?.addEventListener("submit", submitProfileItemForm); + document + .getElementById("profile-item-modal-close") + ?.addEventListener("click", closeProfileItemModal); + document.getElementById("profile-item-cancel")?.addEventListener("click", closeProfileItemModal); + document.getElementById("btn-add-step")?.addEventListener("click", () => addStepRow("")); + + // Event delegation for dynamically generated profile edit/delete buttons + document.getElementById("profile-content").addEventListener("click", (e) => { + const editBtn = e.target.closest(".btn-edit-profile-item"); + const deleteBtn = e.target.closest(".btn-delete-profile-item"); + const pageBtn = e.target.closest(".btn-page"); + if (editBtn) { + const type = editBtn.dataset.type; + const index = parseInt(editBtn.dataset.index, 10); + editProfileItem(type, index); + } + if (deleteBtn) { + const type = deleteBtn.dataset.type; + const index = parseInt(deleteBtn.dataset.index, 10); + deleteProfileItem(type, index); + } + if (pageBtn && !pageBtn.disabled) { + const pageType = pageBtn.dataset.pageType; + const page = parseInt(pageBtn.dataset.page, 10); + state.profilePages[pageType] = page; + refreshProfile(); + } + }); + await loadTags(); await loadMemories(); await loadStats(); diff --git a/src/web/i18n.js b/src/web/i18n.js index 80255c5..f5d28d9 100644 --- a/src/web/i18n.js +++ b/src/web/i18n.js @@ -65,6 +65,7 @@ const translations = { "toast-fresh-start-success": "Fresh start completed successfully", "toast-fresh-start-failed": "Fresh start failed", "confirm-delete": "Delete this memory?", + "confirm-delete-title": "Confirm Delete", "confirm-delete-pair": "Delete this memory AND its linked prompt?", "confirm-delete-prompt": "Delete this prompt AND its linked memory?", "confirm-bulk-delete": "Delete {count} selected memories?", @@ -83,8 +84,12 @@ const translations = { "profile-prompts": "PROMPTS", "profile-updated": "LAST UPDATED", "profile-preferences": "PREFERENCES", + "profile-type-pref": "Pref", + "profile-type-pat": "Pat", + "profile-type-wf": "Wf", "profile-patterns": "PATTERNS", "profile-workflows": "WORKFLOWS", + "label-evidence-tooltip": "AI analysis hit {count} times", "badge-prompt": "USER PROMPT", "badge-memory": "MEMORY", "badge-pinned": "PINNED", @@ -95,10 +100,41 @@ const translations = { "empty-patterns": "No patterns detected yet", "empty-workflows": "No workflows identified yet", "btn-delete-pair": "Delete Pair", + "btn-save": "Save", + "btn-edit": "Edit", "btn-delete": "Delete", + "label-category": "Category", + "label-description": "Description", + "label-steps": "Steps", + "btn-add-step": "+ Add Step", "text-generated-above": "Generated memory above", "text-from-below": "From prompt below", "btn-refresh": "Refresh", + "btn-ai-cleanup": "AI Cleanup", + "label-ai-cleanup-title": "AI Profile Cleanup", + "label-ai-cleanup-loading": "AI is analyzing profile...", + "label-ai-cleanup-merged": "Merged", + "label-ai-cleanup-removed": "Removed", + "label-ai-cleanup-kept": "Remaining", + "label-ai-cleanup-none": "None", + "label-ai-cleanup-select-all": "Select All", + "label-ai-cleanup-deselect-all": "Deselect All", + "label-ai-cleanup-apply": "Apply Changes", + "label-ai-cleanup-merge-check": "Merge", + "label-ai-cleanup-remove-check": "Remove", + "label-ai-cleanup-changes-selected": "{selected}/{total} changes selected", + "label-ai-cleanup-merged-header": "Merged ({count} groups)", + "label-ai-cleanup-removed-header": "Removed ({count} items)", + "btn-apply": "Apply", + "btn-cancel": "Cancel", + "toast-cleanup-success": "Profile cleanup completed", + "toast-cleanup-failed": "AI cleanup failed", + "toast-cleanup-apply-failed": "Apply cleanup failed", + "label-ai-cleanup-select": "Select items for cleanup analysis", + "label-ai-cleanup-select-low": "Low Freq (≤3)", + "label-ai-cleanup-select-same-cat": "Same Category Pairs", + "label-ai-cleanup-analyze": "Analyze Selected", + "label-ai-cleanup-selected": "{count} selected", "migration-found-tags": "Found {count} memories needing technical tags.", "migration-stopped": "Migration stopped: maximum attempts reached", "migration-shards-mismatch": "{count} shard(s) have different dimensions", @@ -170,6 +206,7 @@ const translations = { "toast-fresh-start-success": "重新开始完成", "toast-fresh-start-failed": "重新开始失败", "confirm-delete": "删除这条记忆？", + "confirm-delete-title": "确认删除", "confirm-delete-pair": "删除这条记忆及其关联的提示词？", "confirm-delete-prompt": "删除这条提示词及其关联的记忆？", "confirm-bulk-delete": "删除选中的 {count} 条记忆？", @@ -188,8 +225,12 @@ const translations = { "profile-prompts": "提示词数", "profile-updated": "最后更新", "profile-preferences": "偏好设置", + "profile-type-pref": "偏好", + "profile-type-pat": "模式", + "profile-type-wf": "流程", "profile-patterns": "行为模式", "profile-workflows": "工作流程", + "label-evidence-tooltip": "AI 分析命中 {count} 次", "badge-prompt": "用户提示词", "badge-memory": "记忆", "badge-pinned": "已置顶", @@ -200,10 +241,42 @@ const translations = { "empty-patterns": "尚未检测到行为模式", "empty-workflows": "尚未识别出工作流程", "btn-delete-pair": "删除组合", + "btn-save": "保存", + "btn-edit": "编辑", "btn-delete": "删除", + "label-category": "类别", + "label-description": "描述", + "label-steps": "步骤", + "btn-add-step": "+ 添加步骤", "text-generated-above": "由上方记忆生成", "text-from-below": "来自下方提示词", "btn-refresh": "刷新", + "btn-ai-cleanup": "AI 清理", + "label-ai-cleanup-title": "AI 画像清理", + "label-ai-cleanup-loading": "AI 正在分析画像...", + "label-ai-cleanup-merged": "合并项", + "label-ai-cleanup-removed": "删除项", + "label-ai-cleanup-kept": "保留", + "label-ai-cleanup-none": "无", + "label-ai-cleanup-select-all": "全选", + "label-ai-cleanup-deselect-all": "清空", + "label-ai-cleanup-apply": "应用变更", + "label-ai-cleanup-merge-check": "合并", + "label-ai-cleanup-remove-check": "删除", + "label-ai-cleanup-changes-selected": "已选 {selected}/{total} 项", + "label-ai-cleanup-merged-header": "合并 ({count} 组)", + "label-ai-cleanup-removed-header": "删除 ({count} 项)", + "btn-apply": "确认替换", + "btn-cancel": "取消", + "toast-cleanup-success": "画像清理完成", + "toast-cleanup-failed": "AI 清理失败", + "toast-cleanup-apply-failed": "应用清理失败", + "label-ai-cleanup-select": "选择需要清理分析的条目", + "label-ai-cleanup-select-low": "低频 (≤3)", + "label-ai-cleanup-select-same-cat": "同分类多条目", + "label-ai-cleanup-analyze": "分析选中项", + "label-ai-cleanup-selected": "已选 {count} 项", + "profile-actions": "操作", "migration-found-tags": "发现 {count} 条需要技术标签的记忆。", "migration-stopped": "迁移已停止：达到最大尝试次数", "migration-shards-mismatch": "{count} 个分片具有不同的维度", @@ -297,6 +370,7 @@ const translations = { "toast-fresh-start-failed": "فشلت البداية الجديدة", "confirm-delete": "هل تريد حذف هذه الذكرى؟", + "confirm-delete-title": "تأكيد الحذف", "confirm-delete-pair": "هل تريد حذف هذه الذكرى والموجه المرتبط بها؟", "confirm-delete-prompt": "هل تريد حذف هذا الموجه والذكرى المرتبطة به؟", @@ -322,8 +396,12 @@ const translations = { "profile-prompts": "الموجهات", "profile-updated": "آخر تحديث", "profile-preferences": "التفضيلات", + "profile-type-pref": "تفضيل", + "profile-type-pat": "نمط", + "profile-type-wf": "سير", "profile-patterns": "الأنماط", "profile-workflows": "سير العمل", + "label-evidence-tooltip": "الذكاء الاصطناعي حدد هذا {count} مرات", "badge-prompt": "موجه المستخدم", "badge-memory": "ذكرى", @@ -338,12 +416,43 @@ const translations = { "empty-workflows": "لم يتم التعرف على أي سير عمل بعد", "btn-delete-pair": "حذف الزوج", + "btn-save": "حفظ", + "btn-edit": "تعديل", "btn-delete": "حذف", + "label-category": "الفئة", + "label-description": "الوصف", + "label-steps": "الخطوات", + "btn-add-step": "+ أضف خطوة", "text-generated-above": "تم إنشاء الذكرى أعلاه", "text-from-below": "من الموجه أدناه", "btn-refresh": "تحديث", + "btn-ai-cleanup": "تنظيف بالذكاء الاصطناعي", + "label-ai-cleanup-title": "تنظيف الملف الشخصي", + "label-ai-cleanup-loading": "الذكاء الاصطناعي يحلل الملف الشخصي...", + "label-ai-cleanup-merged": "مدموجة", + "label-ai-cleanup-removed": "محذوفة", + "label-ai-cleanup-kept": "متبقية", + "label-ai-cleanup-none": "لا يوجد", + "label-ai-cleanup-select-all": "تحديد الكل", + "label-ai-cleanup-deselect-all": "إلغاء الكل", + "label-ai-cleanup-apply": "تطبيق التغييرات", + "label-ai-cleanup-merge-check": "دمج", + "label-ai-cleanup-remove-check": "حذف", + "label-ai-cleanup-changes-selected": "تم تحديد {selected}/{total}", + "label-ai-cleanup-merged-header": "دمج ({count} مجموعة)", + "label-ai-cleanup-removed-header": "حذف ({count} عنصر)", + "btn-apply": "تأكيد الاستبدال", + "btn-cancel": "إلغاء", + "toast-cleanup-success": "اكتمل تنظيف الملف الشخصي", + "toast-cleanup-failed": "فشل تنظيف الملف الشخصي", + "toast-cleanup-apply-failed": "فشل تطبيق التنظيف", + "label-ai-cleanup-select": "حدد العناصر للتحليل", + "label-ai-cleanup-select-low": "تردد منخفض (≤3)", + "label-ai-cleanup-select-same-cat": "نفس الفئة", + "label-ai-cleanup-analyze": "تحليل المحدد", + "label-ai-cleanup-selected": "{count} محدد", "migration-found-tags": "تم العثور على {count} من الذكريات التي تحتاج إلى وسوم تقنية.", @@ -370,6 +479,7 @@ function setLanguage(lang) { document.documentElement.lang = lang; applyLanguage(); + document.dispatchEvent(new CustomEvent("langchange")); } function t(key, params = {}) { diff --git a/src/web/index.html b/src/web/index.html index 05504e2..0879b28 100644 --- a/src/web/index.html +++ b/src/web/index.html @@ -151,9 +151,19 @@

└─ PROJECT MEMORIES (0) + + + + diff --git a/src/web/styles.css b/src/web/styles.css index 46462d3..4808a75 100644 --- a/src/web/styles.css +++ b/src/web/styles.css @@ -414,6 +414,48 @@ button:disabled { justify-content: center; } +.pagination-bar { + display: flex; + align-items: center; + justify-content: center; + gap: 8px; + margin-top: 10px; + padding-top: 8px; + border-top: 1px solid var(--border, #333); +} + +.pagination-bar .pagination-info { + font-size: 11px; + color: var(--text-secondary, #888); + margin-right: 4px; +} + +.pagination-bar .btn-page { + background: var(--bg-secondary, #2a2a2a); + border: 1px solid var(--border, #444); + color: var(--text-primary, #eee); + font-size: 12px; + padding: 2px 8px; + border-radius: 4px; + cursor: pointer; +} + +.pagination-bar .btn-page:hover:not([disabled]) { + background: var(--accent, #4a90d9); + color: #fff; +} + +.pagination-bar .btn-page[disabled] { + opacity: 0.35; + cursor: default; +} + +.pagination-bar .btn-page.active { + background: var(--accent, #4a90d9); + color: #fff; + border-color: var(--accent, #4a90d9); +} + .memories-list { display: flex; flex-direction: column; @@ -1429,21 +1471,52 @@ textarea:focus-visible { border-top: 1px solid #1a1a1a; padding-top: 6px; margin-top: auto; + display: flex; + align-items: center; + gap: 6px; + font-size: 11px; + color: #555; +} +.workflow-footer { + border-top: 1px solid #1a1a1a; + padding-top: 6px; + margin-top: 8px; + display: flex; + align-items: center; + gap: 6px; + font-size: 11px; + color: #555; +} + +.evidence-count { + display: inline-flex; + align-items: center; + cursor: help; + color: #666; + font-size: 11px; +} + +.evidence-sep { + color: #444; } .evidence-toggle { - font-size: 10px; + font-size: 11px; color: #555; cursor: help; - display: flex; + display: inline-flex; align-items: center; - gap: 4px; + gap: 3px; } .evidence-toggle:hover { color: #888; } +.evidence-count:hover { + color: #888; +} + .workflows-grid { display: flex; flex-direction: column; @@ -1570,6 +1643,52 @@ textarea:focus-visible { opacity: 0.5; } +/* Profile item actions */ +.card-actions { + display: flex; + gap: 2px; + margin-right: 8px; +} + +.btn-icon { + background: none; + border: none; + cursor: pointer; + padding: 2px 4px; + border-radius: 3px; + color: #666; + transition: + color 0.15s, + background 0.15s; +} + +.btn-icon:hover { + color: #00ff00; + background: rgba(0, 255, 0, 0.08); +} + +.btn-delete-profile-item:hover { + color: #ff4444; + background: rgba(255, 50, 50, 0.08); +} + +.card-top { + display: flex; + align-items: center; + gap: 8px; +} + +.workflow-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 8px; +} + +.workflow-header .card-actions { + flex-shrink: 0; +} + @media (max-width: 768px) { .profile-header { flex-direction: column; @@ -1629,3 +1748,529 @@ textarea:focus-visible { background: #00ff00; color: #0a0a0a; } + +/* AI Cleanup Modal */ +.cleanup-modal-content { + max-width: 700px; + max-height: 80vh; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.cleanup-loading { + text-align: center; + padding: 40px; + color: #888; +} + +.cleanup-loading i { + display: block; + margin-bottom: 10px; +} + +.cleanup-summary { + padding: 12px 16px; + background: #1a1a2e; + border-radius: 6px; + margin-bottom: 16px; + color: #00ff00; + font-weight: 600; +} + +.cleanup-section { + margin-bottom: 16px; +} + +.cleanup-section h4 { + color: #888; + margin-bottom: 8px; + font-size: 0.85rem; + text-transform: uppercase; + letter-spacing: 1px; +} + +.diff-group { + margin-bottom: 12px; + border-bottom: 1px solid #2a2a4a; + padding-bottom: 8px; +} + +.diff-item { + padding: 6px 10px; + border-radius: 4px; + font-size: 0.85rem; + line-height: 1.5; +} + +.diff-marker { + font-weight: 700; + margin-right: 6px; +} + +.kept-item { + background: rgba(0, 255, 0, 0.05); +} + +.kept-item .diff-marker { + color: #00ff00; +} + +.removed-item { + background: rgba(255, 50, 50, 0.05); + color: #999; + text-decoration: line-through; +} + +.removed-item .diff-marker { + color: #ff3232; +} + +/* Profile item modal */ +.profile-item-modal-content { + max-width: 480px; +} + +#profile-item-form .form-group { + margin-bottom: 12px; +} + +#profile-item-form label { + display: block; + font-size: 0.8rem; + color: #888; + margin-bottom: 4px; +} + +#profile-item-form input, +#profile-item-form textarea { + width: 100%; + background: #1a1a2e; + border: 1px solid #333; + color: #c0e0c0; + padding: 8px; + border-radius: 4px; + font-family: inherit; +} + +#profile-item-form input:disabled, +#profile-item-form textarea:disabled { + opacity: 0.6; +} + +#steps-container { + display: flex; + flex-direction: column; + gap: 6px; + margin-bottom: 8px; +} + +.step-row { + display: flex; + align-items: center; + gap: 6px; +} + +.step-num { + color: var(--text-secondary); + font-size: 12px; + width: 20px; + text-align: center; + flex-shrink: 0; +} + +.step-input { + flex: 1; + padding: 6px 8px; + font-size: 13px; + border: 1px solid var(--border-light); + border-radius: 4px; + background: var(--bg-input); + color: var(--text-primary); + font-family: inherit; +} + +.step-input:focus { + outline: none; + border-color: var(--accent-blue); +} + +.btn-remove-step { + background: none; + border: none; + color: var(--text-muted); + font-size: 16px; + cursor: pointer; + padding: 0 4px; + line-height: 1; +} + +.btn-remove-step:hover { + color: #cc2222; +} + +.btn-primary.danger { + background: #cc2222; + border-color: #ff4444; + color: #fff; +} + +.btn-primary.danger:hover { + background: #dd3333; +} + +.diff-reason { + display: block; + color: #666; + font-size: 0.75rem; + margin-top: 2px; + text-decoration: none; +} + +.profile-actions { + display: flex; + gap: 8px; +} + +/* V2 Diff Layout */ +.cleanup-diff-v2 { + overflow-y: auto; + max-height: 55vh; +} + +.cleanup-toolbar { + display: flex; + justify-content: space-between; + align-items: center; + padding: 8px 0; + border-bottom: 1px solid #2a2a4a; + margin-bottom: 12px; + font-size: 0.85rem; + color: #aaa; + position: sticky; + top: 0; + background: #0d0d1a; + z-index: 1; +} + +.toolbar-actions { + display: flex; + gap: 12px; +} + +.btn-link { + background: none; + border: none; + color: #00ff00; + cursor: pointer; + font-size: 0.8rem; + padding: 2px 0; +} +.btn-link:hover { + text-decoration: underline; +} + +.section-divider { + font-size: 0.8rem; + color: #888; + text-transform: uppercase; + letter-spacing: 1px; + margin: 16px 0 8px; + padding-bottom: 6px; + border-bottom: 1px solid #1a1a2e; +} +.section-divider i { + vertical-align: middle; + margin-right: 4px; +} + +.diff-card { + border-radius: 6px; + padding: 12px; + margin-bottom: 10px; +} + +.merge-card { + border-left: 3px solid #00ff00; + background: rgba(0, 255, 0, 0.02); +} + +.remove-card { + border-left: 3px solid #ff4444; + background: rgba(255, 50, 50, 0.02); +} + +.diff-card-check { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 10px; + cursor: pointer; +} + +.diff-card-check input[type="checkbox"] { + accent-color: #00ff00; + width: 16px; + height: 16px; +} + +.remove-card .diff-card-check input[type="checkbox"] { + accent-color: #ff4444; +} + +.check-label { + font-size: 0.8rem; + font-weight: 600; + color: #00ff00; +} + +.remove-card .check-label { + color: #ff4444; +} + +.merge-body { + display: grid; + grid-template-columns: 1fr auto 1fr; + gap: 10px; + align-items: flex-start; +} + +.merge-before, +.merge-after { + background: #111122; + border-radius: 4px; + padding: 8px; + font-size: 0.8rem; + line-height: 1.5; + color: #c0e0c0; +} + +.merge-source { + padding: 4px 0; + border-bottom: 1px solid #1a1a2e; + color: #999; + text-decoration: line-through; +} +.merge-source:last-child { + border-bottom: none; +} + +.type-badge { + display: inline-block; + background: #1a1a2e; + color: #888; + font-size: 0.65rem; + font-weight: 600; + padding: 1px 5px; + border-radius: 3px; + margin-right: 4px; + vertical-align: middle; + text-decoration: none; + text-transform: uppercase; +} + +.merge-arrow { + align-self: center; + color: #00ff00; + font-size: 0.9rem; + padding-top: 12px; +} + +.remove-body { + margin-left: 26px; +} + +.remove-desc { + color: #c0e0c0; + font-size: 0.85rem; + text-decoration: line-through; + margin-bottom: 4px; +} + +.remove-reason { + color: #ff8888; + font-size: 0.7rem; +} + +.collapse-header { + cursor: pointer; + font-size: 0.8rem; + color: #888; + text-transform: uppercase; + letter-spacing: 1px; + margin-top: 12px; + user-select: none; +} +.collapse-header:hover { + color: #aaa; +} +.collapse-header i { + vertical-align: middle; + margin-left: 4px; + transition: transform 0.2s; +} + +.cleanup-kept-list { + margin-top: 4px; + display: grid; + grid-template-columns: 1fr 1fr; + gap: 4px; +} + +.kept-item { + background: rgba(0, 255, 0, 0.03); + padding: 6px 8px; + border-radius: 3px; + font-size: 0.78rem; + color: #aaa; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.kept-item::before { + content: "✓ "; + color: #00ff00; + font-weight: 700; +} + +/* Inline steps in AI cleanup diff */ +.cleanup-steps { + margin-top: 4px; + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 2px; +} + +.step-inline { + display: inline-flex; + align-items: center; + background: rgba(255, 255, 255, 0.06); + border-radius: 3px; + padding: 1px 5px 1px 2px; + font-size: 0.7rem; + color: #aaa; +} + +.step-inline-num { + display: inline-flex; + align-items: center; + justify-content: center; + width: 14px; + height: 14px; + border-radius: 50%; + background: rgba(255, 255, 255, 0.1); + font-size: 0.6rem; + margin-right: 3px; + flex-shrink: 0; +} + +.step-arrow-inline { + color: #555; + font-size: 0.7rem; + margin: 0 1px; +} + +/* ============================================ + AI Cleanup selection UI + ============================================ */ +.cleanup-select-header { + flex-shrink: 0; + padding: 12px 0 8px; +} +.cleanup-select-header h3 { + font-size: 14px; + margin-bottom: 8px; + color: var(--text-primary, #eee); +} +.cleanup-select-actions { + display: flex; + gap: 6px; + flex-wrap: wrap; + margin-bottom: 8px; +} +.cleanup-select-actions .btn { + font-size: 11px; + padding: 3px 10px; +} +.cleanup-select-grid { + flex: 1; + overflow-y: auto; + border: 1px solid var(--border, #333); + border-radius: 6px; + padding: 4px; + min-height: 0; +} + +#cleanup-sections { + display: flex; + flex-direction: column; + flex: 1; + overflow: hidden; +} +.cleanup-cat-group { + margin-bottom: 4px; +} +.cleanup-cat-header { + display: flex; + align-items: center; + gap: 6px; + padding: 4px 8px; + background: var(--bg-secondary, #222); + border-radius: 4px; + position: sticky; + top: 0; + z-index: 1; +} +.cleanup-cat-header .cat-count { + font-size: 11px; + color: var(--text-secondary, #888); +} +.cleanup-item-row { + display: flex; + align-items: center; + gap: 6px; + padding: 3px 8px; + cursor: pointer; + font-size: 12px; + border-radius: 3px; +} +.cleanup-item-row:hover { + background: var(--bg-hover, #2a2a2a); +} +.cleanup-item-row input[type="checkbox"] { + flex-shrink: 0; +} +.cleanup-item-type { + font-size: 10px; + font-weight: 600; + color: var(--accent, #4a90d9); + width: 14px; + text-align: center; + flex-shrink: 0; +} +.cleanup-item-desc { + flex: 1; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--text-primary, #ddd); +} +.cleanup-item-stats { + font-size: 10px; + color: var(--text-secondary, #888); + flex-shrink: 0; + white-space: nowrap; +} +.cleanup-select-footer { + flex-shrink: 0; + display: flex; + align-items: center; + justify-content: space-between; + padding: 10px 0 0; +} +.cleanup-select-footer span { + font-size: 12px; + color: var(--text-secondary, #888); +} diff --git a/tests/profile-tool-runtime.test.ts b/tests/profile-tool-runtime.test.ts index 847fe6d..613f128 100644 --- a/tests/profile-tool-runtime.test.ts +++ b/tests/profile-tool-runtime.test.ts @@ -85,7 +85,7 @@ mock.module(${JSON.stringify(userProfileManagerUrl)}, () => ({ const existing = preferences.find((item) => item.description === pref.description); if (existing) { existing.confidence = Math.max(existing.confidence ?? 0, pref.confidence ?? 0); - existing.lastUpdated = pref.lastUpdated; + existing.lastSeen = pref.lastSeen; } else { preferences.push(pref); } diff --git a/tests/profile-write.test.ts b/tests/profile-write.test.ts index b612e35..23d9f14 100644 --- a/tests/profile-write.test.ts +++ b/tests/profile-write.test.ts @@ -3,19 +3,14 @@ * Exercises the write path added to src/index.ts `profile` mode * by testing the underlying manager directly (no live plugin context needed). */ -import { describe, it, expect, beforeAll, beforeEach, afterAll, afterEach } from "bun:test"; -import { mkdirSync, mkdtempSync } from "node:fs"; +import { describe, it, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { connectionManager } from "../src/services/sqlite/connection-manager.js"; -import { removeDirWithRetries } from "./helpers/temp-dir.mjs"; // We patch CONFIG.storagePath before importing the manager so the DB lands in tmp. -let suiteTmpDir: string; let tmpDir: string; -let testCounter = 0; - -const WINDOWS_CLEANUP_LOCK_ERRORS = new Set(["EBUSY", "ENOTEMPTY", "EPERM"]); async function makeManager() { // Dynamic import after setting storagePath so the constructor picks up the temp dir. @@ -29,30 +24,16 @@ async function makeManager() { } describe("UserProfileManager – explicit preference writes", () => { - beforeAll(() => { - suiteTmpDir = mkdtempSync(join(tmpdir(), "opencode-mem-profile-write-")); - }); - beforeEach(() => { - testCounter += 1; - tmpDir = join(suiteTmpDir, `case-${testCounter}`); - mkdirSync(tmpDir, { recursive: true }); - }); - - afterEach(() => { - connectionManager.closeAll(); + tmpDir = mkdtempSync(join(tmpdir(), "opencode-mem-test-")); }); - afterAll(async () => { + afterEach(async () => { connectionManager.closeAll(); + await new Promise((r) => setTimeout(r, 100)); try { - await removeDirWithRetries(suiteTmpDir, 8); - } catch (error: any) { - // Windows can briefly keep SQLite temp dirs locked after closeAll(). - if (!WINDOWS_CLEANUP_LOCK_ERRORS.has(error?.code)) { - throw error; - } - } + rmSync(tmpDir, { recursive: true, force: true }); + } catch {} }); it("creates a profile with an explicit preference when none exists", async () => { @@ -71,7 +52,7 @@ describe("UserProfileManager – explicit preference writes", () => { description: "Prefer concise answers", confidence: 1.0, evidence: ["manual-write"], - lastUpdated: Date.now(), + lastSeen: Date.now(), }, ], patterns: [], @@ -89,123 +70,6 @@ describe("UserProfileManager – explicit preference writes", () => { expect(data.preferences[0].evidence).toContain("manual-write"); }); - it("adds lastUpdated to generated preferences when creating a profile", async () => { - const mgr = await makeManager(); - const userId = "test@example.com"; - const before = Date.now(); - - mgr.createProfile( - userId, - "Test User", - "testuser", - userId, - { - preferences: [ - { - category: "style", - description: "Prefers concise answers", - confidence: 0.7, - evidence: ["observed"], - } as any, - ], - patterns: [], - workflows: [], - }, - 10 - ); - - const after = Date.now(); - const profile = mgr.getActiveProfile(userId)!; - const data = JSON.parse(profile.profileData); - const lastUpdated = data.preferences[0].lastUpdated; - - expect(typeof lastUpdated).toBe("number"); - expect(lastUpdated).toBeGreaterThanOrEqual(before); - expect(lastUpdated).toBeLessThanOrEqual(after); - }); - - it("applies confidence decay to stale preferences", async () => { - const mgr = await makeManager(); - const userId = "test@example.com"; - const staleTimestamp = Date.now() - 61 * 24 * 60 * 60 * 1000; - - mgr.createProfile( - userId, - "Test User", - "testuser", - userId, - { - preferences: [ - { - category: "style", - description: "Prefers concise answers", - confidence: 0.8, - evidence: ["observed"], - lastUpdated: staleTimestamp, - }, - ], - patterns: [], - workflows: [], - }, - 10 - ); - - const profile = mgr.getActiveProfile(userId)!; - const changed = mgr.applyConfidenceDecay(profile.id); - - expect(changed).toBe(true); - - const updated = mgr.getActiveProfile(userId)!; - const data = JSON.parse(updated.profileData); - - expect(updated.version).toBe(2); - expect(data.preferences[0].confidence).toBeCloseTo(0.4, 2); - expect(data.preferences[0].lastUpdated).toBeGreaterThan(staleTimestamp); - - const changedAgain = mgr.applyConfidenceDecay(updated.id); - const unchanged = mgr.getActiveProfile(userId)!; - - expect(changedAgain).toBe(false); - expect(unchanged.version).toBe(2); - }); - - it("removes preferences that decay below the confidence floor", async () => { - const mgr = await makeManager(); - const userId = "test@example.com"; - const staleTimestamp = Date.now() - 61 * 24 * 60 * 60 * 1000; - - mgr.createProfile( - userId, - "Test User", - "testuser", - userId, - { - preferences: [ - { - category: "style", - description: "Weak stale preference", - confidence: 0.4, - evidence: ["observed"], - lastUpdated: staleTimestamp, - }, - ], - patterns: [], - workflows: [], - }, - 10 - ); - - const profile = mgr.getActiveProfile(userId)!; - const changed = mgr.applyConfidenceDecay(profile.id); - - expect(changed).toBe(true); - - const updated = mgr.getActiveProfile(userId)!; - const data = JSON.parse(updated.profileData); - - expect(data.preferences).toHaveLength(0); - }); - it("merges a new explicit preference into an existing profile without clobbering other prefs", async () => { const mgr = await makeManager(); const userId = "test@example.com"; @@ -223,7 +87,7 @@ describe("UserProfileManager – explicit preference writes", () => { description: "Uses TypeScript", confidence: 0.8, evidence: ["observed"], - lastUpdated: Date.now(), + lastSeen: Date.now(), }, ], patterns: [], @@ -240,10 +104,10 @@ describe("UserProfileManager – explicit preference writes", () => { description: "Always use numbered lists", confidence: 1.0, evidence: ["manual-write"], - lastUpdated: Date.now(), + lastSeen: Date.now(), }; - const merged = mgr.mergeProfileData(existingData, { preferences: [newPref] }); + const merged = await mgr.mergeProfileData(existingData, { preferences: [newPref] }); mgr.updateProfile( existingProfile.id, merged, @@ -271,7 +135,7 @@ describe("UserProfileManager – explicit preference writes", () => { description, confidence: 1.0, evidence: ["manual-write"], - lastUpdated: Date.now(), + lastSeen: Date.now(), }; mgr.createProfile( @@ -290,8 +154,8 @@ describe("UserProfileManager – explicit preference writes", () => { // Write the same preference again (simulates calling profile+content twice) const p1 = mgr.getActiveProfile(userId)!; const d1 = JSON.parse(p1.profileData); - const merged = mgr.mergeProfileData(d1, { - preferences: [{ ...pref, lastUpdated: Date.now() }], + const merged = await mgr.mergeProfileData(d1, { + preferences: [{ ...pref, lastSeen: Date.now() }], }); mgr.updateProfile(p1.id, merged, 0, "Explicit preference added: Prefer short answers"); @@ -331,14 +195,14 @@ describe("UserProfileManager – explicit preference writes", () => { const p = mgr.getActiveProfile(userId)!; const d = JSON.parse(p.profileData); - const merged = mgr.mergeProfileData(d, { + const merged = await mgr.mergeProfileData(d, { preferences: [ { category: "explicit", description: "Use snake_case", confidence: 1.0, evidence: ["manual-write"], - lastUpdated: Date.now(), + lastSeen: Date.now(), }, ], });