feat: Enhance embedding service with comprehensive analytics tracking

  - Add detailed embedding text and model tracking for vector generation analysis
  - Capture query text and embedding dimensions for search insights
  - Track retrieved document content including embedding text, prompts, and code snippets
  - Enhance vector and hybrid search with comprehensive document metadata
  - Add performance metrics for embedding operations (latency, vector size, model)
  - Include similarity scores (vector, keyword, combined) for retrieval optimization
  - Store first 200 chars of code from retrieved documents for quick reference

  These enhancements enable better understanding of:
  - What text is being embedded vs original content
  - Query-to-document matching patterns
  - Retrieval effectiveness and ranking quality
  - Opportunities for vector database improvement

Author: jeffdyer

src/lib/embedding-service.ts

@@ -80,13 +80,16 @@ export async function generateEmbedding(
       ragLog(options.rid, "embedding.complete", {
         model,
         elapsedMs,
+        textLength: cleanedText.length,
       });
 
-      // Track in analytics
+      // Track in analytics with enhanced information
       safeRAGAnalytics.trackEmbedding(
         options.rid,
         response.data[0].embedding,
-        elapsedMs
+        elapsedMs,
+        cleanedText,  // Pass the actual text that was embedded
+        model         // Pass the model used
       );
     }
 
@@ -335,6 +338,15 @@ export async function vectorSearch({
       ? new VectorValue(queryEmbedding)
       : queryEmbedding;
 
+    // Log the query text for analysis
+    if (rid) {
+      ragLog(rid, "vectorSearch.queryText", {
+        query: query,
+        queryLength: query.length,
+        embeddingDimensions: queryEmbedding.length,
+      });
+    }
+
     // Perform vector similarity search
     // Note: Firebase requires a specific index for vector search
     const results = await searchQuery
@@ -368,13 +380,16 @@ export async function vectorSearch({
         })),
       });
 
-      // Track retrieval in analytics
+      // Track retrieval in analytics with enhanced information
       safeRAGAnalytics.trackRetrieval(
         rid,
         documents.map(doc => ({
           id: doc.id,
           similarity: doc.similarity,
           features: buildFeatureTags(doc),
+          embeddingText: createEmbeddingText(doc),  // Include embedding text
+          code: doc.code || doc.src,                // Include code
+          prompt: doc.prompt || doc.task,           // Include prompt
         })),
         "vector",
         Date.now() - startTime,
@@ -437,6 +452,8 @@ export async function hybridSearch({
         k: limit,
         lang,
         vectorWeight,
+        query: query,  // Log the query text
+        queryLength: query.length,
       });
     }
 
@@ -500,7 +517,7 @@ export async function hybridSearch({
         })),
       });
 
-      // Track hybrid search in analytics
+      // Track hybrid search in analytics with enhanced information
       safeRAGAnalytics.trackRetrieval(
         rid,
         topResults.map(doc => ({
@@ -509,6 +526,9 @@ export async function hybridSearch({
           keywordScore: doc.keywordScore,
           combinedScore: doc.combinedScore,
           features: buildFeatureTags(doc),
+          embeddingText: createEmbeddingText(doc),  // Include embedding text
+          code: doc.code || doc.src,                // Include code
+          prompt: doc.prompt || doc.task,           // Include prompt
         })),
         "hybrid",
         Date.now() - startTime,

src/lib/rag-analytics-safe.ts

@@ -59,11 +59,11 @@ class SafeRAGAnalytics {
   /**
    * Safely track embedding
    */
-  trackEmbedding(requestId: string, embeddingVector: number[], latencyMs: number): void {
+  trackEmbedding(requestId: string, embeddingVector: number[], latencyMs: number, embeddingText?: string, model?: string): void {
     if (!this.isEnabled()) return;
 
     try {
-      ragAnalytics.trackEmbedding(requestId, embeddingVector, latencyMs);
+      ragAnalytics.trackEmbedding(requestId, embeddingVector, latencyMs, embeddingText, model);
     } catch (error) {
       this.handleError("trackEmbedding", error);
     }

src/lib/rag-analytics.ts

@@ -32,6 +32,9 @@ export interface RetrievedDocument {
   position: number;
   wasUsedInPrompt: boolean;
   features?: string[];
+  embeddingText?: string;  // Text used for embedding
+  prompt?: string;         // Original prompt/task from the document
+  codeSnippet?: string;    // Snippet of code from the document
 }
 
 // Generation metrics
@@ -89,6 +92,16 @@ export interface RAGAnalyticsRecord {
     language: string;
     keywords?: string[];
     embeddingVector?: number[]; // Store first N dimensions for analysis
+    embeddingDimensions?: number; // Total number of dimensions
+    embeddingText?: string; // The actual text that was embedded
+    embeddingModel?: string; // Model used for embedding
+  };
+
+  // Embedding information
+  embedding?: {
+    latencyMs: number;
+    vectorSize: number;
+    model: string;
   };
 
   // Retrieval information
@@ -273,16 +286,36 @@ export class RAGAnalyticsService {
   public trackEmbedding(
     requestId: string,
     embeddingVector: number[],
-    latencyMs: number
+    latencyMs: number,
+    embeddingText?: string,
+    model?: string
   ): void {
     const record = this.activeRequests.get(requestId);
     if (!record) return;
 
-    // Store first 10 dimensions for analysis (to save space)
+    // Store embedding details for analysis
     if (record.query) {
-      record.query.embeddingVector = embeddingVector.slice(0, 10);
+      record.query.embeddingVector = embeddingVector.slice(0, 10); // Store first 10 dimensions
+      record.query.embeddingDimensions = embeddingVector.length;
+
+      // Store the actual text that was embedded (for debugging and enhancement)
+      if (embeddingText) {
+        record.query.embeddingText = embeddingText;
+      }
+
+      // Store model information
+      if (model) {
+        record.query.embeddingModel = model;
+      }
     }
 
+    // Track embedding performance metrics
+    record.embedding = {
+      latencyMs: latencyMs,
+      vectorSize: embeddingVector.length,
+      model: model || "text-embedding-3-small"
+    };
+
     this.endStage(requestId, "embedding");
   }
 
@@ -297,6 +330,9 @@ export class RAGAnalyticsService {
       keywordScore?: number;
       combinedScore?: number;
       features?: string[];
+      embeddingText?: string;
+      code?: string;
+      prompt?: string;
     }>,
     strategy: "vector" | "hybrid" | "keyword",
     latencyMs: number,
@@ -313,6 +349,9 @@ export class RAGAnalyticsService {
       position: index,
       wasUsedInPrompt: false, // Will be updated later
       features: doc.features,
+      embeddingText: doc.embeddingText,  // Text used for embedding
+      prompt: doc.prompt,                 // Original prompt/task from the document
+      codeSnippet: doc.code ? doc.code.substring(0, 200) : undefined, // First 200 chars of code
     }));
 
     record.retrieval = {