feat: prompt improvements, CLI Dockerfile, and sanitize AI generation logs

Prompt improvements (based on eval run analysis):
- Acknowledge dynamic pages that load content on scroll (remove false
  "no scrolling needed" claim)
- Add guidance for click failures, autocomplete/combobox fields, date
  pickers, stale element refs, and PDF-avoidance (use HTML versions)
- Fix bug: hasWebSearch was not passed to actionLoopSystemPromptTemplate,
  so the webSearch tool instruction was never shown to the model
- Add step-error recovery guidance (stale refs, retrying with different
  approach, no repeated actions)
- Refine abort() vs done(): only abort when core task fails; done() is
  acceptable when supplementary details are inaccessible
- Require explicit criterion confirmation in done() answers

CLI Dockerfile (for eval/standalone use):
- Uses --ignore-scripts + pnpm rebuild esbuild to avoid prepare
  lifecycle running before source is copied
- Installs Playwright system deps as root, browser as node user
- INSTALL_PLAYWRIGHT_BROWSERS ARG to skip browser install in CI/tests
- Stub config files for both root and node users

Strip binary image data from ai:generation logs by default:
- JSONConsoleLogger sanitizes image content blocks in AI_GENERATION
  events (replaces raw bytes with { type, mediaType, size, data: '<omitted>' })
- Avoids 50-100MB stdout.txt files from embedded JPEG data
- Opt-out via sanitizeGenerationImages: false

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>

Author: lmorchard

packages/cli/Dockerfile

@@ -0,0 +1,69 @@
+FROM node:22-slim
+
+# Install pnpm
+RUN npm install -g pnpm
+
+# Build argument to control Playwright browser installation
+ARG INSTALL_PLAYWRIGHT_BROWSERS=true
+
+# Set working directory to repo root
+WORKDIR /app
+
+# Copy workspace manifests so pnpm can resolve the workspace graph
+COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
+
+# Copy all package manifests before installing so pnpm can link the workspace
+COPY packages/core/package.json ./packages/core/package.json
+COPY packages/cli/package.json ./packages/cli/package.json
+COPY packages/server/package.json ./packages/server/package.json
+COPY packages/extension/package.json ./packages/extension/package.json
+
+# Install all workspace dependencies.
+# --ignore-scripts prevents the root prepare lifecycle script from running
+# pilo-core build before its source has been copied into the image.
+# We then selectively rebuild only packages with native binaries (esbuild)
+# so their postinstall scripts run without triggering the root prepare.
+RUN pnpm install --frozen-lockfile --ignore-scripts
+RUN pnpm rebuild esbuild
+
+# Copy core source and build it (CLI depends on pilo-core)
+COPY packages/core/ ./packages/core/
+RUN pnpm --filter pilo-core run build
+
+# Install Playwright system dependencies as root (requires apt)
+RUN if [ "$INSTALL_PLAYWRIGHT_BROWSERS" = "true" ]; then \
+    npx playwright install-deps chromium; \
+    fi
+
+# Copy CLI source and build
+COPY packages/cli/ ./packages/cli/
+RUN pnpm --filter pilo-cli run build
+
+# Set environment variables
+ENV NODE_ENV=production
+
+# Create minimal stub config for root user (container may run as root)
+RUN mkdir -p /root/.config/pilo && echo '{}' > /root/.config/pilo/config.json
+
+# Ensure node user owns the app files
+RUN chown -R node:node /app
+
+# Switch to non-root user
+USER node
+
+# Create minimal stub configs to satisfy the CLI config guard.
+# The guard only checks for file existence; actual provider/key config
+# is supplied via environment variables (dev-mode env var loading).
+# Both root and node user paths are created because callers may run
+# the container as either user.
+RUN mkdir -p /home/node/.config/pilo && echo '{}' > /home/node/.config/pilo/config.json
+
+# Install Playwright browsers as node user (so they end up in /home/node/.cache)
+# ARG must be redeclared after USER to be accessible (with same default)
+ARG INSTALL_PLAYWRIGHT_BROWSERS=true
+RUN if [ "$INSTALL_PLAYWRIGHT_BROWSERS" = "true" ]; then \
+    npx playwright install chromium; \
+    fi
+
+# Run pilo CLI - subcommand args are appended by the caller
+ENTRYPOINT ["node", "/app/packages/cli/dist/cli/src/cli.js"]

packages/core/src/loggers/json.ts

@@ -13,16 +13,63 @@ const DEFAULT_EXCLUDED_EVENTS: readonly WebAgentEventType[] = [
 export interface JSONConsoleLoggerOptions {
   /** Include screenshot image events (default: false) */
   includeScreenshotImages?: boolean;
+  /**
+   * Strip binary image data from ai:generation message history (default: true).
+   * Images are replaced with { type, mediaType, size, data: '<omitted>' }.
+   * Disable only if you need the raw bytes for debugging.
+   */
+  sanitizeGenerationImages?: boolean;
+}
+
+/**
+ * Strip binary image data from ai:generation event data before logging.
+ * Image content blocks are replaced with a lightweight descriptor:
+ *   { type: 'image', mediaType, size, data: '<omitted>' }
+ * The actual JPEG files are already saved separately alongside stdout.txt.
+ */
+function sanitizeAiGenerationData(data: unknown): unknown {
+  if (!data || typeof data !== "object") return data;
+  const d = data as Record<string, unknown>;
+  if (!Array.isArray(d.messages)) return data;
+
+  return {
+    ...d,
+    messages: d.messages.map((msg: unknown) => {
+      if (!msg || typeof msg !== "object") return msg;
+      const m = msg as Record<string, unknown>;
+      if (!Array.isArray(m.content)) return msg;
+
+      return {
+        ...m,
+        content: m.content.map((block: unknown) => {
+          if (!block || typeof block !== "object") return block;
+          const b = block as Record<string, unknown>;
+          if (b.type === "image" && ArrayBuffer.isView(b.image)) {
+            const buf = b.image as ArrayBufferView;
+            return {
+              type: "image",
+              mediaType: b.mediaType,
+              size: buf.byteLength,
+              data: "<omitted>",
+            };
+          }
+          return block;
+        }),
+      };
+    }),
+  };
 }
 
 export class JSONConsoleLogger implements Logger {
   private emitter: WebAgentEventEmitter | null = null;
   private handlers: [WebAgentEventType, (data: any) => void][] = [];
   private excludedEvents: Set<WebAgentEventType>;
+  private sanitizeGenerationImages: boolean;
 
   constructor(options: JSONConsoleLoggerOptions = {}) {
     // Build excluded events set
     this.excludedEvents = new Set(options.includeScreenshotImages ? [] : DEFAULT_EXCLUDED_EVENTS);
+    this.sanitizeGenerationImages = options.sanitizeGenerationImages !== false; // default: true
   }
 
   initialize(emitter: WebAgentEventEmitter): void {
@@ -55,7 +102,15 @@ export class JSONConsoleLogger implements Logger {
         return;
       }
 
-      const json = JSON.stringify({ event: type, data }, null, 0);
+      // Strip raw image bytes from AI generation messages to keep stdout.txt manageable.
+      // Screenshots are already saved separately as JPEG files; the binary arrays
+      // in the message history serve no purpose in the log and can reach 50-100MB.
+      const safeData =
+        this.sanitizeGenerationImages && type === WebAgentEventType.AI_GENERATION
+          ? sanitizeAiGenerationData(data)
+          : data;
+
+      const json = JSON.stringify({ event: type, data: safeData }, null, 0);
       console.log(json);
     };
   }

packages/core/src/prompts.ts

@@ -122,7 +122,7 @@ You adapt to situations and find creative ways to complete tasks without getting
 
 IMPORTANT:
 - You can see the entire page content through the accessibility tree snapshot.
-- You do not need to scroll or click links to navigate within a page - all content is visible to you.
+- The accessibility tree shows all currently loaded page elements. On dynamic pages, some content may only appear after scrolling or interaction — if expected data isn't visible, try scrolling or interacting to trigger loading.
 - Focus on the elements you need to interact with directly.
 `.trim();
 
@@ -285,15 +285,21 @@ Analyze the current page state and determine your next action based on previous
 - extract() if you need more information
 
 **Best Practices:**
-- Full page content is visible - no scrolling needed
+- The accessibility tree shows currently loaded elements; dynamic pages may load more content on scroll
 - Clear obstructing modals/popups first
 - Prefer click() over goto() for page navigation
 - Submit forms via enter() or submit button after filling
 - Find alternative elements if primary ones aren't available
+- When click() fails due to element interception, try focus() first, then keyboard navigation (Tab, Enter, arrow keys), or press Escape to dismiss overlapping overlays
+- For autocomplete/combobox search fields (e.g., flight origin/destination, location pickers): after fill(), use focus() on a visible suggestion in the dropdown followed by enter() to select it — click() on autocomplete suggestions often times out
+- For date pickers and calendar widgets: prefer typing dates directly into the date input field using fill() rather than clicking through calendar months; if the field doesn't respond to fill(), try focus() on it first; avoid repeated calendar navigation clicks — if clicking "next month" fails twice, try filling the date field directly or using keyboard input
+- When you receive an 'Invalid element reference' error, the page DOM has changed — read the updated page snapshot on your next turn and use the new element refs; do not retry old ref IDs
 - Adapt your approach based on what's actually available
 - If you don't find relevant links or buttons, and the site has a search form, prioritize using it for navigation
-- Use abort() only after trying reasonable alternatives (site down, access blocked, required data unavailable)
+- If you have found the core information requested but cannot access supplementary details due to site limitations, use done() with what you have — only use abort() when the core task cannot be completed at all
 - For research: Use extract() immediately when finding relevant data
+- For academic papers or documents that require reading, counting, or extracting content (e.g., counting figures/tables, reading body text): PDFs are often unscrollable and unreadable — use webSearch to find an HTML version (e.g., ACL Anthology, Semantic Scholar) or the abstract page before attempting the PDF
+{% if hasWebSearch %}- If you need to search the web, use webSearch({query}) directly rather than filling in a browser search engine (DuckDuckGo, Google, Bing, etc.) — webSearch avoids CAPTCHA and bot detection that will block browser-based searches{% endif %}
 {% if hasGuardrails %}- Verify guardrail compliance before each action{% endif %}
 
 **When using done():**
@@ -304,6 +310,7 @@ Provide your final answer:
 - Include all requested information
 - Format results as VALID Markdown
 - NEVER return raw JSON - ALWAYS format structured data as VALID Markdown
+- If the task required finding content that meets specific criteria (minimum rating, review count, price range, ingredient count, etc.), explicitly confirm each criterion was met in your answer
 
 {% if hasGuardrails %}
 🚨 **GUARDRAIL COMPLIANCE:** Any action violating the provided guardrails is FORBIDDEN.
@@ -317,6 +324,7 @@ ${toolCallInstruction}
 const buildActionLoopSystemPrompt = (hasGuardrails: boolean, hasWebSearch: boolean = false) =>
   actionLoopSystemPromptTemplate({
     hasGuardrails,
+    hasWebSearch,
     toolExamples: buildToolExamples(hasWebSearch),
     currentDate: getCurrentFormattedDate(),
   });
@@ -428,6 +436,11 @@ const stepErrorFeedbackTemplate = buildPromptTemplate(
 CRITICAL: ALL TOOL CALLS MUST COMPLY WITH THE PROVIDED GUARDRAILS
 {% endif %}
 
+**Recovery guidance:**
+- If the error mentions "Invalid element reference" or "does not exist on the current page": the page DOM has changed and your cached ref IDs are stale. Do NOT retry the same ref. Wait for the next page snapshot (it will arrive automatically) and use only the new ref IDs shown there.
+- If an action keeps failing after 2 attempts, try a different approach: use a different element, navigate differently, or use extract() to re-read the current state.
+- Do NOT repeat the same action with the same arguments.
+
 **Available Tools:**
 {{ toolExamples }}
 
@@ -512,6 +525,7 @@ const taskValidationFeedbackTemplate = buildPromptTemplate(
 **Feedback:** {{ feedback }}
 
 Do not repeat your previous answer. Address the issues identified above.
+If you cannot address the feedback due to genuine site limitations (disabled UI, inaccessible content), call done() with the best answer available rather than aborting.
 `.trim(),
 );
 

packages/core/test/loggers.test.ts

@@ -829,6 +829,91 @@ describe("JSONConsoleLogger", () => {
       expect(parsed.data.temperature).toBe(0.7);
     });
 
+    it("should sanitize image Buffer data in AI_GENERATION messages", () => {
+      const fakeJpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]); // JPEG header bytes
+      const eventData: AIGenerationEventData = {
+        timestamp: Date.now(),
+        iterationId: "test-1",
+        prompt: "Analyze the page",
+        schema: {},
+        messages: [
+          {
+            role: "user",
+            content: [
+              { type: "text", text: "What do you see?" },
+              { type: "image", image: fakeJpeg, mimeType: "image/jpeg" },
+            ] as any,
+          },
+        ],
+        finishReason: "stop",
+        usage: {
+          totalTokens: 200,
+          inputTokens: 150,
+          outputTokens: 50,
+          inputTokenDetails: { noCacheTokens: 150, cacheReadTokens: 0, cacheWriteTokens: 0 },
+          outputTokenDetails: { textTokens: 50, reasoningTokens: 0 },
+        },
+        providerMetadata: {},
+        warnings: [],
+      };
+
+      emitter.emitEvent({
+        type: WebAgentEventType.AI_GENERATION,
+        data: eventData,
+      });
+
+      expect(mockConsole.log).toHaveBeenCalledTimes(1);
+      const output = mockConsole.log.mock.calls[0][0];
+      const parsed = JSON.parse(output);
+
+      const imageBlock = parsed.data.messages[0].content[1];
+      expect(imageBlock.type).toBe("image");
+      expect(imageBlock.data).toBe("<omitted>");
+      expect(imageBlock.size).toBe(fakeJpeg.byteLength);
+      // Raw byte array must not appear in the output
+      expect(output).not.toContain("255"); // 0xff byte value
+    });
+
+    it("should preserve raw image data when sanitizeGenerationImages is false", () => {
+      logger.dispose();
+      const rawLogger = new JSONConsoleLogger({ sanitizeGenerationImages: false });
+      rawLogger.initialize(emitter);
+
+      const fakeJpeg = Buffer.from([0xff, 0xd8]);
+      const eventData: AIGenerationEventData = {
+        timestamp: Date.now(),
+        iterationId: "test-1",
+        prompt: "Analyze the page",
+        schema: {},
+        messages: [
+          {
+            role: "user",
+            content: [{ type: "image", image: fakeJpeg, mimeType: "image/jpeg" }] as any,
+          },
+        ],
+        finishReason: "stop",
+        usage: {
+          totalTokens: 100,
+          inputTokens: 80,
+          outputTokens: 20,
+          inputTokenDetails: { noCacheTokens: 80, cacheReadTokens: 0, cacheWriteTokens: 0 },
+          outputTokenDetails: { textTokens: 20, reasoningTokens: 0 },
+        },
+        providerMetadata: {},
+        warnings: [],
+      };
+
+      emitter.emitEvent({ type: WebAgentEventType.AI_GENERATION, data: eventData });
+
+      expect(mockConsole.log).toHaveBeenCalledTimes(1);
+      const parsed = JSON.parse(mockConsole.log.mock.calls[0][0]);
+      const imageBlock = parsed.data.messages[0].content[0];
+      // data field should NOT be '<omitted>' when sanitization is disabled
+      expect(imageBlock.data).not.toBe("<omitted>");
+
+      rawLogger.dispose();
+    });
+
     it("should output JSON for AI_GENERATION_ERROR events", () => {
       const eventData: AIGenerationErrorEventData = {
         timestamp: Date.now(),