Merge branch 'main' into stafford/tab-976-harden-pilo-against-web-content-prompt-injection

Resolve import conflict in packages/cli/src/utils.ts (keep both BROWSERS and
FIELDS/PiloConfig — all are used). Add fail-safe action-firewall introspection
(getFieldMetadata/getFormSubmissionContext) to main's new BiDiBrowser so it
satisfies the extended AriaBrowser interface; real BiDi introspection is a
follow-up.

Author: srbiv

.github/workflows/build-test.yml

@@ -356,3 +356,108 @@ jobs:
           path: packages/extension/test-results/
           retention-days: 30
           if-no-files-found: ignore
+
+  smoke-eval:
+    name: Smoke eval (Pilo agent on example.com)
+    runs-on: ubuntu-latest
+    needs: [detect-changes, setup]
+    if: needs.detect-changes.outputs.cli == 'true' || needs.detect-changes.outputs.core == 'true' || needs.detect-changes.outputs.workflows == 'true'
+    # id-token: write is required if/when this job is switched to Workload
+    # Identity Federation (see the auth step below).
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 9
+
+      - name: Get pnpm store directory
+        shell: bash
+        run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+
+      - name: Cache pnpm store
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Cache core build
+        uses: actions/cache@v4
+        with:
+          path: |
+            packages/core/dist
+            packages/core/src/browser/ariaTree/bundle.ts
+          key: ${{ runner.os }}-core-dist-${{ github.sha }}
+
+      - name: Build CLI
+        run: pnpm --filter pilo-cli run build
+
+      - name: Get Playwright version
+        id: playwright-version
+        run: |
+          set -euo pipefail
+          version=$(jq -re '.dependencies["playwright"] // empty' package.json)
+          if [ -z "$version" ]; then
+            echo "Error: playwright dependency not found in root package.json" >&2
+            exit 1
+          fi
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+
+      - name: Cache Playwright browsers
+        uses: actions/cache@v4
+        id: playwright-cache
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}
+
+      - name: Install Playwright browsers
+        if: steps.playwright-cache.outputs.cache-hit != 'true'
+        run: pnpm exec playwright install --with-deps chromium
+
+      - name: Install Playwright system dependencies
+        if: steps.playwright-cache.outputs.cache-hit == 'true'
+        run: pnpm exec playwright install-deps chromium
+
+      # GCP auth for Vertex AI (Gemini). Default path uses a service-account
+      # JSON secret; once Workload Identity Federation is provisioned for
+      # nonprod Vertex access, swap `credentials_json` for the WIF inputs
+      # shown in the commented block below.
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_SMOKE_EVAL_SA_KEY }}
+          # workload_identity_provider: ${{ vars.GCP_SMOKE_EVAL_WIF_PROVIDER }}
+          # service_account: ${{ vars.GCP_SMOKE_EVAL_SERVICE_ACCOUNT }}
+
+      - name: Run smoke eval
+        env:
+          GOOGLE_CLOUD_PROJECT: moz-fx-tabs-nonprod
+          GOOGLE_CLOUD_REGION: us-central1
+          SMOKE_EVAL_PROVIDER: vertex
+          SMOKE_EVAL_MODEL: gemini-2.5-flash
+          SMOKE_EVAL_TIMEOUT_MS: "180000"
+        run: pnpm smoke-eval
+
+      - name: Upload smoke-eval log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: smoke-eval-log
+          path: .smoke-eval-output/
+          retention-days: 14
+          if-no-files-found: warn

.gitignore

@@ -7,6 +7,7 @@ dist
 scripts/docker-build.sh
 debug
 coverage
+.smoke-eval-output
 
 # Auto-generated files
 packages/core/src/browser/ariaTree/bundle.ts

README.md

@@ -143,6 +143,26 @@ try {
 }
 ```
 
+### WebDriver BiDi (experimental)
+
+Pilo can connect directly to any browser that supports the [WebDriver BiDi](https://w3c.github.io/webdriver-bidi/) protocol, without requiring Playwright. This is distinct from Pilo's `--channel moz-firefox` option, which uses BiDi _through_ Playwright — the `--browser bidi` mode speaks the BiDi protocol directly over WebSocket with no Playwright dependency in the chain.
+
+Start Firefox with remote debugging enabled:
+
+```bash
+# Headless
+firefox --remote-debugging-port 9222 --headless --no-remote --profile "$(mktemp -d)"
+
+# Visible (watch the agent work)
+firefox --remote-debugging-port 9222 --no-remote --profile "$(mktemp -d)"
+```
+
+Then point Pilo at it:
+
+```bash
+pilo run --browser bidi --bidi-url "ws://127.0.0.1:9222/session" "what's the weather in Tokyo?"
+```
+
 ## Features
 
 - 🤖 **Natural Language Control**: Just describe what you want to do in plain English

package.json

@@ -36,6 +36,7 @@
     "test": "pnpm -r run test",
     "check": "pnpm run typecheck && pnpm run test",
     "pilo": "pnpm --filter pilo-cli run pilo",
+    "smoke-eval": "node scripts/smoke-eval/run.mjs",
     "dev:extension": "pnpm --filter pilo-extension run dev --",
     "dev:server": "pnpm --filter pilo-server run dev"
   },

packages/cli/src/commands/run.ts

@@ -205,37 +205,64 @@ async function executeRunCommand(task: string, options: any): Promise<void> {
 
     // Create browser instance with navigation retry config
     // CLI options take precedence over config values
-    const browser = new PlaywrightBrowser({
-      browser: browserOption,
-      bypassCSP: options.bypassCsp ?? cfg.bypass_csp,
-      channel: options.channel ?? cfg.channel,
-      executablePath: options.executablePath ?? cfg.executable_path,
-      blockAds: options.blockAds ?? cfg.block_ads,
-      blockResources,
-      headless: options.headless ?? cfg.headless,
-      proxyServer: options.proxy ?? cfg.proxy,
-      proxyUsername: options.proxyUsername ?? cfg.proxy_username,
-      proxyPassword: options.proxyPassword ?? cfg.proxy_password,
-      pwEndpoint: options.pwEndpoint ?? cfg.pw_endpoint,
-      pwCdpEndpoint: options.pwCdpEndpoint ?? cfg.pw_cdp_endpoint,
-      pwCdpEndpoints:
-        (options.pwCdpEndpoints as string[] | undefined) ??
-        cfg.pw_cdp_endpoints ??
-        (cfg.pw_cdp_endpoint ? [cfg.pw_cdp_endpoint] : undefined),
-      actionTimeoutMs: options.actionTimeoutMs ?? cfg.action_timeout_ms,
-      navigationRetry: {
-        baseTimeoutMs: options.navigationTimeoutMs ?? cfg.navigation_timeout_ms,
-        maxTimeoutMs: options.navigationMaxTimeoutMs ?? cfg.navigation_max_timeout_ms,
-        maxAttempts: options.navigationMaxAttempts ?? cfg.navigation_max_attempts,
-        timeoutMultiplier: options.navigationTimeoutMultiplier ?? cfg.navigation_timeout_multiplier,
-        onRetry: (attempt: number, error: Error, nextTimeout: number) => {
-          console.log(
-            chalk.yellow(`⚠️ Navigation retry ${attempt}: ${error.message}`),
-            chalk.gray(`(next timeout: ${Math.round(nextTimeout / 1000)}s)`),
-          );
+    let browser;
+    if (browserOption === "bidi") {
+      const { BiDiBrowser } = await import("pilo-core");
+      const bidiUrl = options.bidiUrl ?? cfg.bidi_url;
+      if (!bidiUrl) {
+        throw new Error("--bidi-url or PILO_BIDI_URL is required when using --browser bidi");
+      }
+      browser = new BiDiBrowser({
+        bidiUrl,
+        actionTimeoutMs: options.actionTimeoutMs ?? cfg.action_timeout_ms,
+      });
+    } else if (browserOption === "foxcloud") {
+      const { FoxcloudBrowser } = await import("pilo-core");
+      const foxcloudUrl = options.foxcloudUrl ?? cfg.foxcloud_url;
+      if (!foxcloudUrl) {
+        throw new Error(
+          "--foxcloud-url or PILO_FOXCLOUD_URL is required when using --browser foxcloud",
+        );
+      }
+      browser = new FoxcloudBrowser({
+        brokerUrl: foxcloudUrl,
+        proxyUrl: options.foxcloudProxyUrl ?? cfg.foxcloud_proxy_url,
+        actionTimeoutMs: options.actionTimeoutMs ?? cfg.action_timeout_ms,
+      });
+    } else {
+      browser = new PlaywrightBrowser({
+        browser: browserOption,
+        bypassCSP: options.bypassCsp ?? cfg.bypass_csp,
+        channel: options.channel ?? cfg.channel,
+        executablePath: options.executablePath ?? cfg.executable_path,
+        blockAds: options.blockAds ?? cfg.block_ads,
+        blockResources,
+        headless: options.headless ?? cfg.headless,
+        proxyServer: options.proxy ?? cfg.proxy,
+        proxyUsername: options.proxyUsername ?? cfg.proxy_username,
+        proxyPassword: options.proxyPassword ?? cfg.proxy_password,
+        pwEndpoint: options.pwEndpoint ?? cfg.pw_endpoint,
+        pwCdpEndpoint: options.pwCdpEndpoint ?? cfg.pw_cdp_endpoint,
+        pwCdpEndpoints:
+          (options.pwCdpEndpoints as string[] | undefined) ??
+          cfg.pw_cdp_endpoints ??
+          (cfg.pw_cdp_endpoint ? [cfg.pw_cdp_endpoint] : undefined),
+        actionTimeoutMs: options.actionTimeoutMs ?? cfg.action_timeout_ms,
+        navigationRetry: {
+          baseTimeoutMs: options.navigationTimeoutMs ?? cfg.navigation_timeout_ms,
+          maxTimeoutMs: options.navigationMaxTimeoutMs ?? cfg.navigation_max_timeout_ms,
+          maxAttempts: options.navigationMaxAttempts ?? cfg.navigation_max_attempts,
+          timeoutMultiplier:
+            options.navigationTimeoutMultiplier ?? cfg.navigation_timeout_multiplier,
+          onRetry: (attempt: number, error: Error, nextTimeout: number) => {
+            console.log(
+              chalk.yellow(`⚠️ Navigation retry ${attempt}: ${error.message}`),
+              chalk.gray(`(next timeout: ${Math.round(nextTimeout / 1000)}s)`),
+            );
+          },
         },
-      },
-    });
+      });
+    }
 
     // Create AI provider with CLI overrides (only pass if explicitly set on CLI)
     // Unlike other options, we use explicit undefined checks here because

packages/cli/src/utils.ts

@@ -1,7 +1,7 @@
 import { existsSync, readFileSync } from "fs";
 import { dirname, join } from "path";
 import { fileURLToPath } from "url";
-import { FIELDS, type PiloConfig } from "pilo-core";
+import { BROWSERS, FIELDS, type PiloConfig } from "pilo-core";
 
 /**
  * CLI-specific utilities and helpers
@@ -48,15 +48,14 @@ export function getPackageInfo(): { version: string; name: string; description:
  * Validate browser option
  */
 export function validateBrowser(browser: string): boolean {
-  const validBrowsers = ["firefox", "chrome", "chromium", "safari", "webkit", "edge"];
-  return validBrowsers.includes(browser);
+  return (BROWSERS as readonly string[]).includes(browser);
 }
 
 /**
  * Get list of valid browsers
  */
 export function getValidBrowsers(): string[] {
-  return ["firefox", "chrome", "chromium", "safari", "webkit", "edge"];
+  return [...BROWSERS];
 }
 
 /**

packages/cli/test/utils.test.ts

@@ -28,8 +28,17 @@ describe("CLI Utils", () => {
   describe("getValidBrowsers", () => {
     it("should return array of valid browsers", () => {
       const browsers = getValidBrowsers();
-      expect(browsers).toEqual(["firefox", "chrome", "chromium", "safari", "webkit", "edge"]);
-      expect(browsers.length).toBe(6);
+      expect(browsers).toEqual([
+        "firefox",
+        "chrome",
+        "chromium",
+        "safari",
+        "webkit",
+        "edge",
+        "bidi",
+        "foxcloud",
+      ]);
+      expect(browsers.length).toBe(8);
     });
   });
 

packages/core/package.json

@@ -48,6 +48,7 @@
     "ollama-ai-provider-v2": "^3.5.0",
     "playwright": "1.60.0",
     "turndown": "^7.2.2",
+    "ws": "^8.20.0",
     "zod": "^4.3.6"
   },
   "peerDependencies": {
@@ -63,6 +64,7 @@
     "@types/jsdom": "^28.0.1",
     "@types/node": "^25.7.0",
     "@types/turndown": "^5.0.6",
+    "@types/ws": "^8.18.1",
     "@vitest/coverage-v8": "^4.1.6",
     "esbuild": "^0.28.0",
     "jsdom": "^29.0.2",