Skip to content

Commit 84fe4ed

Browse files
lmorchardclaude
andcommitted
feat(search): add Exa Search API as a search provider
Add exa-api alongside the existing parallel-api/google/bing/duckduckgo providers. ExaSearchProvider POSTs to api.exa.ai/search, opting into contents.highlights so results include snippets (Exa returns metadata only by default), and maps url/title/highlights through the shared markdown + security-wrapper path identical to the Parallel provider. Wiring: - config: add "exa-api" to SEARCH_PROVIDERS and an exa_api_key field (env EXA_API_KEY, --exa-api-key) - factory + webAgent: exa-api case and key-required validation guard - run.ts / taskRunner.ts: select the API key by provider, since the agent exposes a single provider-agnostic searchApiKey Debug logging for both API providers (Exa and Parallel), gated on --debug via the [X:debug] console.warn convention: - request: the exact outbound body (query + options, API key omitted) - response: result count plus an abbreviated sample of the first result so all returned fields are visible (long strings truncated by the shared abbreviateForDebug helper) The debug flag threads through CreateSearchProviderOptions, which SearchService.create already forwards. Tests cover the factory, markdown formatting, empty/missing-title results, API error, the highlights opt-in, and debug request/response logging on and off for both API providers. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent d63d87f commit 84fe4ed

10 files changed

Lines changed: 460 additions & 13 deletions

File tree

packages/cli/src/commands/run.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -342,7 +342,10 @@ async function executeRunCommand(task: string, options: any): Promise<void> {
342342
maxConsecutiveErrors: options.maxConsecutiveErrors ?? cfg.max_consecutive_errors,
343343
maxTotalErrors: options.maxTotalErrors ?? cfg.max_total_errors,
344344
searchProvider: options.searchProvider ?? cfg.search_provider,
345-
searchApiKey: cfg.parallel_api_key,
345+
searchApiKey:
346+
(options.searchProvider ?? cfg.search_provider) === "exa-api"
347+
? cfg.exa_api_key
348+
: cfg.parallel_api_key,
346349
tabstackApiKey: options.tabstackApiKey ?? cfg.tabstack_api_key,
347350
tabstackApiUrl: options.tabstackApiUrl ?? cfg.tabstack_api_url,
348351
trustedHostnames: options.trustedHostnames ?? cfg.trusted_hostnames,

packages/core/src/config/defaults.ts

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,14 @@ export type ReasoningLevel = (typeof REASONING_LEVELS)[number];
4242
export const LOGGERS = ["console", "json"] as const;
4343
export type LoggerType = (typeof LOGGERS)[number];
4444

45-
export const SEARCH_PROVIDERS = ["none", "duckduckgo", "google", "bing", "parallel-api"] as const;
45+
export const SEARCH_PROVIDERS = [
46+
"none",
47+
"duckduckgo",
48+
"google",
49+
"bing",
50+
"parallel-api",
51+
"exa-api",
52+
] as const;
4653
export type SearchProviderName = (typeof SEARCH_PROVIDERS)[number];
4754

4855
export type ConfigFieldType = "string" | "string[]" | "number" | "boolean" | "enum";
@@ -136,6 +143,7 @@ export interface PiloConfig {
136143
// Search Configuration
137144
search_provider?: SearchProviderName;
138145
parallel_api_key?: string;
146+
exa_api_key?: string;
139147

140148
// Tabstack Configuration
141149
tabstack_api_key?: string;
@@ -215,6 +223,7 @@ export interface PiloConfigResolved {
215223
// Search Configuration
216224
search_provider: SearchProviderName;
217225
parallel_api_key?: string;
226+
exa_api_key?: string;
218227

219228
// Tabstack Configuration
220229
tabstack_api_key?: string;
@@ -733,6 +742,14 @@ export const FIELDS: Record<ConfigKey, FieldDef> = {
733742
description: "Parallel API key for search",
734743
category: "search",
735744
},
745+
exa_api_key: {
746+
type: "string",
747+
cli: "--exa-api-key",
748+
placeholder: "key",
749+
env: ["EXA_API_KEY"],
750+
description: "Exa API key for search",
751+
category: "search",
752+
},
736753

737754
// Tabstack Configuration
738755
tabstack_api_key: {
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
/**
2+
* Debug helpers for search providers.
3+
*/
4+
5+
const MAX_STRING_LEN = 120;
6+
7+
/**
8+
* Deep-clone a value for debug logging, truncating any long string so the
9+
* "flavor" of a response (text, summaries, snippets, etc.) is visible without
10+
* dumping the full payload. Non-string values pass through unchanged.
11+
*/
12+
export function abbreviateForDebug(value: unknown): unknown {
13+
const json = JSON.stringify(value, (_key, v) =>
14+
typeof v === "string" && v.length > MAX_STRING_LEN ? `${v.slice(0, MAX_STRING_LEN)}…` : v,
15+
);
16+
return json === undefined ? value : JSON.parse(json);
17+
}
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
/**
2+
* Exa API Search Provider
3+
*
4+
* API-based search provider that uses the Exa API for search.
5+
* Returns results formatted as markdown for consistency with browser providers.
6+
*/
7+
8+
import type { AriaBrowser } from "../../browser/ariaBrowser.js";
9+
import type { SearchProvider } from "../searchProvider.js";
10+
import {
11+
wrapExternalContentWithWarning,
12+
ExternalContentLabel,
13+
} from "../../utils/promptSecurity.js";
14+
import { abbreviateForDebug } from "../debugPreview.js";
15+
16+
interface ExaSearchResult {
17+
url: string;
18+
title?: string;
19+
highlights?: string[];
20+
}
21+
22+
interface ExaApiResponse {
23+
results?: ExaSearchResult[];
24+
}
25+
26+
export class ExaSearchProvider implements SearchProvider {
27+
readonly name = "exa-api";
28+
readonly requiresBrowser = false;
29+
30+
constructor(
31+
private apiKey: string,
32+
private debug = false,
33+
) {}
34+
35+
async search(query: string, _browser?: AriaBrowser): Promise<string> {
36+
const url = "https://api.exa.ai/search";
37+
const body = JSON.stringify({
38+
query,
39+
// Opt into highlights, or Exa returns metadata only (no snippets).
40+
contents: { highlights: { maxCharacters: 1500 } },
41+
});
42+
43+
if (this.debug) {
44+
// Log the exact outbound request body (sans API key) so the query and
45+
// contents options are observable. Matches the [X:debug] console.warn convention.
46+
console.warn(`[ExaSearch:debug] POST ${url}`, body);
47+
}
48+
49+
const response = await fetch(url, {
50+
method: "POST",
51+
headers: {
52+
"Content-Type": "application/json",
53+
"x-api-key": this.apiKey,
54+
},
55+
body,
56+
});
57+
58+
if (!response.ok) {
59+
const errorText = await response.text().catch(() => "Unknown error");
60+
throw new Error(`Exa API error (${response.status}): ${errorText}`);
61+
}
62+
63+
const data = (await response.json()) as ExaApiResponse;
64+
65+
if (this.debug) {
66+
// Log the count plus an abbreviated sample of the first result so all
67+
// returned fields (including ones we don't map, like summary/score/
68+
// publishedDate) are visible, with long strings truncated.
69+
const results = data.results ?? [];
70+
console.warn(
71+
`[ExaSearch:debug] response: ${results.length} result(s), sample:`,
72+
abbreviateForDebug(results[0]),
73+
);
74+
}
75+
76+
return this.formatAsMarkdown(query, data);
77+
}
78+
79+
private formatAsMarkdown(query: string, data: ExaApiResponse): string {
80+
const header = `# Search Results for "${query}" (via ${this.name})`;
81+
82+
let wrapped: string;
83+
if (!data.results || data.results.length === 0) {
84+
wrapped = wrapExternalContentWithWarning(
85+
`${header}\n\nNo results found.`,
86+
ExternalContentLabel.SearchResults,
87+
);
88+
} else {
89+
const lines: string[] = [];
90+
91+
data.results.forEach((result, index) => {
92+
const title = result.title || result.url;
93+
lines.push(`${index + 1}. [${title}](${result.url})`);
94+
if (result.highlights?.length) {
95+
lines.push(result.highlights.join("\n"));
96+
}
97+
lines.push("");
98+
});
99+
100+
wrapped = wrapExternalContentWithWarning(
101+
`${header}\n\n${lines.join("\n").trim()}`,
102+
ExternalContentLabel.SearchResults,
103+
);
104+
}
105+
106+
return wrapped;
107+
}
108+
}

packages/core/src/search/providers/parallelSearch.ts

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import {
1111
wrapExternalContentWithWarning,
1212
ExternalContentLabel,
1313
} from "../../utils/promptSecurity.js";
14+
import { abbreviateForDebug } from "../debugPreview.js";
1415

1516
interface ParallelSearchResult {
1617
url: string;
@@ -27,21 +28,33 @@ export class ParallelSearchProvider implements SearchProvider {
2728
readonly name = "parallel-api";
2829
readonly requiresBrowser = false;
2930

30-
constructor(private apiKey: string) {}
31+
constructor(
32+
private apiKey: string,
33+
private debug = false,
34+
) {}
3135

3236
async search(query: string, _browser?: AriaBrowser): Promise<string> {
33-
const response = await fetch("https://api.parallel.ai/v1beta/search", {
37+
const url = "https://api.parallel.ai/v1beta/search";
38+
const body = JSON.stringify({
39+
objective: query,
40+
search_queries: [query],
41+
excerpts: { max_chars_per_result: 1500 },
42+
});
43+
44+
if (this.debug) {
45+
// Log the exact outbound request body (sans API key) so the query and
46+
// options are observable. Matches the [X:debug] console.warn convention.
47+
console.warn(`[ParallelSearch:debug] POST ${url}`, body);
48+
}
49+
50+
const response = await fetch(url, {
3451
method: "POST",
3552
headers: {
3653
"Content-Type": "application/json",
3754
"x-api-key": this.apiKey,
3855
"parallel-beta": "search-extract-2025-10-10",
3956
},
40-
body: JSON.stringify({
41-
objective: query,
42-
search_queries: [query],
43-
excerpts: { max_chars_per_result: 1500 },
44-
}),
57+
body,
4558
});
4659

4760
if (!response.ok) {
@@ -55,6 +68,16 @@ export class ParallelSearchProvider implements SearchProvider {
5568
throw new Error(`Parallel API error: ${data.error}`);
5669
}
5770

71+
if (this.debug) {
72+
// Log the count plus an abbreviated sample of the first result so all
73+
// returned fields are visible, with long strings truncated.
74+
const results = data.results ?? [];
75+
console.warn(
76+
`[ParallelSearch:debug] response: ${results.length} result(s), sample:`,
77+
abbreviateForDebug(results[0]),
78+
);
79+
}
80+
5881
return this.formatAsMarkdown(query, data);
5982
}
6083

packages/core/src/search/searchProvider.ts

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ export interface SearchProvider {
2121
export interface CreateSearchProviderOptions {
2222
/** API key for providers that require authentication (e.g., Parallel) */
2323
apiKey?: string;
24+
/** When true, API providers log their outbound request at debug level */
25+
debug?: boolean;
2426
}
2527

2628
/**
@@ -49,7 +51,14 @@ export async function createSearchProvider(
4951
throw new Error("Parallel API key is required for parallel-api search provider");
5052
}
5153
const { ParallelSearchProvider } = await import("./providers/parallelSearch.js");
52-
return new ParallelSearchProvider(options.apiKey);
54+
return new ParallelSearchProvider(options.apiKey, options.debug);
55+
}
56+
case "exa-api": {
57+
if (!options.apiKey) {
58+
throw new Error("Exa API key is required for exa-api search provider");
59+
}
60+
const { ExaSearchProvider } = await import("./providers/exaSearch.js");
61+
return new ExaSearchProvider(options.apiKey, options.debug);
5362
}
5463
default:
5564
throw new Error(`Unknown search provider: ${providerName}`);

packages/core/src/webAgent.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,10 @@ export class WebAgent {
357357
throw new Error("parallel_api_key is required when search_provider is 'parallel-api'");
358358
}
359359

360+
if (this.searchProvider === "exa-api" && !this.searchApiKey) {
361+
throw new Error("exa_api_key is required when search_provider is 'exa-api'");
362+
}
363+
360364
// Initialize services
361365
this.compressor = new SnapshotCompressor();
362366
this.eventEmitter = options.eventEmitter ?? new WebAgentEventEmitter();
@@ -420,6 +424,7 @@ export class WebAgent {
420424
if (this.searchProvider !== "none") {
421425
this.searchService = await SearchService.create(this.searchProvider, this.browser, {
422426
apiKey: this.searchApiKey,
427+
debug: this.debug,
423428
});
424429
}
425430

packages/core/test/config.test.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ describe("ConfigManager", () => {
198198
"unsafe_mode",
199199
"search_provider",
200200
"parallel_api_key",
201+
"exa_api_key",
201202
"tabstack_api_key",
202203
"tabstack_api_url",
203204
"upload_allowed_paths",

0 commit comments

Comments
 (0)