feat(search): add Firecrawl search provider + standalone web_fetch tool

.changeset/firecrawl-search-provider.md

@@ -0,0 +1,12 @@
+---
+"@agent-infra/search": minor
+"@agent-infra/shared": minor
+---
+
+Add Firecrawl as a first-class search provider.
+
+`@agent-infra/shared` gains `SearchProvider.Firecrawl`, and `@agent-infra/search`
+adds a `firecrawl` provider (wrapping the `firecrawl` SDK) to the unified
+`SearchClient`, alongside the existing browser/Bing/Tavily/DuckDuckGo/SearXNG
+providers. Firecrawl's `/search` returns clean, LLM-ready markdown for every
+result in a single call; pass `scrapeOptions` to retrieve full-page content.

multimodal/agent-tars/cli/src/index.ts

@@ -85,10 +85,18 @@ export class AgentTARSCLI extends AgentCLI {
         .option('--search <search>', 'Search config')
         .option(
           '--search.provider [provider]',
-          'Search provider (browser_search, tavily, bing_search)',
+          'Search provider (browser_search, tavily, bing_search, firecrawl)',
         )
         .option('--search.count [count]', 'Search result count', { default: 10 })
         .option('--search.apiKey [apiKey]', 'Search API key')
+
+        // Fetch configuration (standalone `web_fetch` tool, independent of search)
+        .option('--fetch <fetch>', 'Fetch config')
+        .option(
+          '--fetch.apiKey [apiKey]',
+          'Firecrawl API key for web_fetch (optional; keyless tier available)',
+        )
+        .option('--fetch.baseUrl [baseUrl]', 'Firecrawl base URL override (self-hosted)')
     );
   }
 

multimodal/agent-tars/core/src/environments/local/fetch/fetch-tool.ts

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { ConsoleLogger, Tool, z } from '@tarko/mcp-agent';
+import { firecrawl } from '@agent-infra/search';
+
+/**
+ * Configuration for the fetch tool. Reuses the search provider's credentials
+ * (`apiKey` is optional — Firecrawl has a keyless free tier; `baseUrl` targets
+ * a self-hosted instance).
+ */
+export interface FetchToolConfig {
+  apiKey?: string;
+  baseUrl?: string;
+}
+
+/**
+ * FetchToolProvider — a `web_fetch` tool backed by Firecrawl's scrape API.
+ *
+ * Reads the full, LLM-ready content of a single URL without driving the
+ * browser. Distinct from search (`web_search`, discovery) and from the
+ * headless browser's `browser_get_markdown` (which only reads the tab the
+ * browser is already on). Kept in its own provider so the search tool stays
+ * search-only.
+ */
+export class FetchToolProvider {
+  private logger: ConsoleLogger;
+  private config: FetchToolConfig;
+
+  constructor(logger: ConsoleLogger, config: FetchToolConfig) {
+    this.logger = logger.spawn('FetchToolProvider');
+    this.config = config;
+  }
+
+  /**
+   * Create a `web_fetch` tool definition for agent registration.
+   */
+  createFetchTool(): Tool {
+    const client = firecrawl({
+      apiKey: this.config.apiKey,
+      apiUrl: this.config.baseUrl,
+    });
+
+    return new Tool({
+      id: 'web_fetch',
+      description:
+        'Fetch the full content of a specific web page as clean, LLM-ready ' +
+        'markdown — without opening it in the browser. Use this when you ' +
+        'already have a URL (e.g. from web_search results) and need its full ' +
+        'text, not just a snippet. Handles JavaScript-rendered pages and PDFs.',
+      parameters: z.object({
+        url: z
+          .string()
+          .describe('The full URL to fetch (must start with http or https).'),
+        formats: z
+          .array(z.enum(['markdown', 'html', 'links']))
+          .optional()
+          .describe('Output formats to return. Defaults to ["markdown"].'),
+      }),
+      function: async ({ url, formats }) => {
+        if (!url || url.trim() === '') {
+          return { error: 'A url is required' };
+        }
+
+        // Enforce the http(s) contract stated in the description, and reject
+        // other schemes (file:, javascript:, ...) before spending a request.
+        let parsed: URL;
+        try {
+          parsed = new URL(url);
+        } catch {
+          return { error: 'A valid URL is required' };
+        }
+        if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+          return { error: 'URL must start with http:// or https://' };
+        }
+
+        // Log only origin + path — query strings may carry tokens / signed-link
+        // credentials that should not land in logs.
+        const safeUrl = `${parsed.origin}${parsed.pathname}`;
+
+        try {
+          this.logger.info(`Fetching: "${safeUrl}"`);
+
+          const doc = await client.scrape(url, {
+            formats: formats?.length ? formats : ['markdown'],
+            onlyMainContent: true,
+          });
+
+          return {
+            url,
+            title: doc.metadata?.title,
+            markdown: doc.markdown,
+            html: doc.html,
+            links: doc.links,
+            metadata: doc.metadata,
+          };
+        } catch (error) {
+          this.logger.error(`Fetch error for "${safeUrl}": ${error}`);
+          return {
+            error: `Fetch failed: ${error instanceof Error ? error.message : String(error)}`,
+          };
+        }
+      },
+    });
+  }
+}

multimodal/agent-tars/core/src/environments/local/fetch/index.ts

@@ -0,0 +1,6 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './fetch-tool';

multimodal/agent-tars/core/src/environments/local/index.ts

@@ -16,6 +16,7 @@ import { ResourceCleaner } from '../../utils';
 import { AgentTARSOptions, BuiltInMCPServers, BuiltInMCPServerName } from '../../types';
 import { BrowserGUIAgent, BrowserManager, BrowserToolsManager } from './browser';
 import { SearchToolProvider } from './search';
+import { FetchToolProvider } from './fetch';
 import { FilesystemToolsManager } from './filesystem';
 import { WorkspacePathResolver } from '../../shared/workspace-path-resolver';
 import { AgentTARSBaseEnvironment } from '../base';
@@ -42,6 +43,7 @@ export class AgentTARSLocalEnvironment extends AgentTARSBaseEnvironment {
   private browserToolsManager?: BrowserToolsManager;
   private filesystemToolsManager?: FilesystemToolsManager;
   private searchToolProvider?: SearchToolProvider;
+  private fetchToolProvider?: FetchToolProvider;
   private browserGUIAgent?: BrowserGUIAgent;
   private mcpServers: BuiltInMCPServers = {};
   private mcpClients: Partial<Record<BuiltInMCPServerName, Client>> = {};
@@ -88,6 +90,11 @@ export class AgentTARSLocalEnvironment extends AgentTARSBaseEnvironment {
       await this.initializeSearchTools(registerToolFn);
     }
 
+    // Initialize fetch tool (independent of search)
+    if (this.options.fetch) {
+      await this.initializeFetchTools(registerToolFn);
+    }
+
     // Initialize MCP servers if using in-memory implementation
     if (this.options.mcpImpl === 'in-memory') {
       await this.initializeInMemoryMCP(registerToolFn);
@@ -137,6 +144,25 @@ export class AgentTARSLocalEnvironment extends AgentTARSBaseEnvironment {
     this.logger.info('✅ Search tools initialized successfully');
   }
 
+  /**
+   * Initialize the fetch tool (`web_fetch`).
+   *
+   * Standalone capability — reads any URL to clean markdown via Firecrawl's
+   * scrape API. Independent of the search provider; configured via
+   * `options.fetch`.
+   */
+  private async initializeFetchTools(registerToolFn: (tool: Tool) => void): Promise<void> {
+    this.logger.info('📄 Initializing fetch tool');
+
+    this.fetchToolProvider = new FetchToolProvider(this.logger, {
+      apiKey: this.options.fetch!.apiKey,
+      baseUrl: this.options.fetch!.baseUrl,
+    });
+    registerToolFn(this.fetchToolProvider.createFetchTool());
+
+    this.logger.info('✅ Fetch tool (web_fetch) initialized successfully');
+  }
+
   /**
    * Initialize in-memory MCP servers and clients
    */

multimodal/agent-tars/core/src/environments/local/search/search-tool.ts

@@ -80,6 +80,7 @@ export class SearchToolProvider {
       tavily: SearchProvider.Tavily,
       searxng: SearchProvider.SearXNG,
       duckduckgo: SearchProvider.DuckduckgoSearch,
+      firecrawl: SearchProvider.Firecrawl,
     };
 
     const resolvedProvider = providerMap[provider] || SearchProvider.BrowserSearch;

multimodal/agent-tars/interface/src/cli.ts

@@ -17,7 +17,7 @@ import { AgentTARSAppConfig } from './config';
  */
 export type AgentTARSCLIArguments = Pick<
   AgentTARSAppConfig,
-  'workspace' | 'browser' | 'planner' | 'search' | 'agio'
+  'workspace' | 'browser' | 'planner' | 'search' | 'fetch' | 'agio'
 > &
   AgentCLIArguments & {
     // Deprecated shortcut options for backward compatibility

multimodal/agent-tars/interface/src/core.ts

@@ -62,19 +62,20 @@ export interface AgentTARSSearchOptions {
    *
    * @defaultValue 'browser_search'
    */
-  provider: 'browser_search' | 'tavily' | 'bing_search';
+  provider: 'browser_search' | 'tavily' | 'bing_search' | 'firecrawl';
   /**
    * Search result count
    *
    * @defaultValue `10`
    */
   count?: number;
   /**
-   * Optional api key, required for tavily and bing_search.
+   * Optional api key, required for tavily and bing_search. Optional for
+   * firecrawl (keyless free tier available; provide `fc-...` for higher limits).
    */
   apiKey?: string;
   /**
-   * Optional api key, required for tavily and bing_search.
+   * Optional base url override (e.g. a self-hosted provider instance).
    */
   baseUrl?: string;
   /**
@@ -94,6 +95,28 @@ export interface AgentTARSSearchOptions {
   };
 }
 
+/**
+ * Fetch options for Agent TARS.
+ *
+ * Configures the `web_fetch` tool, which reads the full, LLM-ready content of a
+ * single URL (markdown/html/links) without driving the browser. This is an
+ * independent capability from search — it can be enabled on its own, alongside
+ * any (or no) search provider.
+ *
+ * Backed by Firecrawl's scrape API. An api key is optional (keyless free tier
+ * available); provide `fc-...` for higher rate limits.
+ */
+export interface AgentTARSFetchOptions {
+  /**
+   * Optional Firecrawl api key (`fc-...`). Optional on the keyless free tier.
+   */
+  apiKey?: string;
+  /**
+   * Optional base url override (e.g. a self-hosted Firecrawl instance).
+   */
+  baseUrl?: string;
+}
+
 /**
  * Options for the planning system within Agent TARS
  */
@@ -137,6 +160,12 @@ export interface AgentTARSOptions extends MCPAgentOptions {
    */
   search?: AgentTARSSearchOptions;
 
+  /**
+   * Fetch settings. Enables the standalone `web_fetch` tool (read any URL to
+   * clean markdown). Independent of `search`.
+   */
+  fetch?: AgentTARSFetchOptions;
+
   /**
    * Browser options
    */

packages/agent-infra/mcp-servers/search/src/server.ts

@@ -42,6 +42,7 @@ export function setSearchConfig(config: Partial<SearchSettings>) {
 const API_KEY_ENV_MAP = {
   [SearchProvider.BingSearch]: process.env.BING_SEARCH_API_KEY,
   [SearchProvider.Tavily]: process.env.TAVILY_API_KEY,
+  [SearchProvider.Firecrawl]: process.env.FIRECRAWL_API_KEY,
   [SearchProvider.BrowserSearch]: undefined,
   [SearchProvider.SearXNG]: undefined,
   [SearchProvider.DuckduckgoSearch]: undefined,
@@ -50,6 +51,7 @@ const API_KEY_ENV_MAP = {
 const API_BASE_URL_ENV_MAP = {
   [SearchProvider.BingSearch]: process.env.BING_SEARCH_API_BASE_URL,
   [SearchProvider.Tavily]: undefined,
+  [SearchProvider.Firecrawl]: process.env.FIRECRAWL_API_BASE_URL,
   [SearchProvider.BrowserSearch]: undefined,
   [SearchProvider.SearXNG]: undefined,
   [SearchProvider.DuckduckgoSearch]: undefined,

packages/agent-infra/search/search/FIRECRAWL.md

@@ -0,0 +1,85 @@
+# Firecrawl provider
+
+[Firecrawl](https://firecrawl.dev) is a web-data API for AI agents: it searches
+the web and returns **clean, LLM-ready markdown** for every result in a single
+call. This package wires Firecrawl in as a first-class `SearchProvider`
+(alongside `browser_search`, `tavily`, `bing_search`, `duckduckgo`, `searxng`).
+Separately, Agent TARS exposes a standalone `web_fetch` tool (configured via the
+independent `fetch` option — not tied to the search provider).
+
+## Usage
+
+```ts
+import { SearchClient, SearchProvider } from '@agent-infra/search';
+
+const client = new SearchClient({
+  provider: SearchProvider.Firecrawl,
+  providerConfig: { apiKey: process.env.FIRECRAWL_API_KEY }, // optional: keyless free tier exists
+});
+
+// Plain search (snippets)
+await client.search({ query: 'UI-TARS', count: 5 });
+
+// Search + full-page content in one call (Firecrawl's differentiator)
+await client.search(
+  { query: 'UI-TARS', count: 5 },
+  { scrapeOptions: { formats: ['markdown'], onlyMainContent: true } },
+);
+```
+
+In Agent TARS, **search and fetch are independent capabilities** — enable
+either, both, or neither:
+
+```ts
+{
+  // web_search — Firecrawl as the search provider
+  search: {
+    provider: 'firecrawl',
+    apiKey: process.env.FIRECRAWL_API_KEY, // optional
+    count: 10,
+  },
+  // web_fetch — standalone, NOT tied to the search provider
+  fetch: {
+    apiKey: process.env.FIRECRAWL_API_KEY, // optional
+  },
+}
+```
+
+Because they're decoupled, you can mix freely — e.g. `browser_search` (or
+`tavily`) for search **and** Firecrawl `fetch` for reading URLs, or `fetch`
+alone with no search provider at all.
+
+- `web_search` (from `search`) — discovery, optionally with full-page content
+  per result.
+- `web_fetch` (from `fetch`) — fetch any single URL as clean markdown without
+  navigating the browser. Complements the existing `browser_get_markdown`
+  (which only reads the tab the browser is already on).
+
+## Why Firecrawl fits
+
+Agent TARS today splits *find* (Tavily `web_search`) from *read* (a separate
+`LinkReader` / `text_browser_view` MCP). Firecrawl's `/search` returns search
+results **with** scraped content, and its `/scrape` reads arbitrary URLs — so a
+single provider covers both motions, and `web_fetch` is a drop-in for the
+commented-out `tavily_extract` path in `omni-tars`.
+
+## Firecrawl endpoint → Agent TARS fit
+
+This PR ships **search + scrape**. The other endpoints were evaluated:
+
+| Endpoint | Fit | Notes |
+|---|---|---|
+| **search** | ✅ shipped | First-class `SearchProvider`. Returns snippets, or full markdown per result via `scrapeOptions`. |
+| **scrape** | ✅ shipped | `web_fetch` tool — read any URL (incl. JS-rendered pages and PDFs) to markdown without driving the browser. |
+| **map** | 🟡 good follow-up | `map(url)` returns all discoverable URLs on a site — a natural cheap "site recon" tool the agent can call before deciding what to read. Bounded, fast, low risk. Recommended next addition. |
+| **crawl** | 🟡 careful | `crawl()` is async and can return many large pages — risks long latency and blowing the context window inside a synchronous agent loop. Viable only as an opt-in tool with a hard `limit`; better suited to a batch/offline job than the interactive loop. |
+| **parse** | 🟡 niche | `parse()` converts an **uploaded local file** (pdf/docx/xlsx/html) to markdown. Pairs with Agent TARS's filesystem environment ("read this local PDF"), but URL-hosted PDFs are already handled by `scrape`. Low-priority, real. |
+| **monitor** | ❌ out of scope | Firecrawl Monitoring is a **scheduled change-tracking** product (watch a page/site over time, diff, notify). That's a cron/background capability, not an interactive agent tool — Agent TARS has no scheduled-task surface to host it today. |
+
+## Cost note
+
+Plain search is 2 credits / 10 results. Adding `scrapeOptions` applies scrape
+costs per result (1 credit/page basic). `web_fetch` calls Firecrawl's `/scrape`
+endpoint (1 credit/page). Keep `scrapeOptions` off `web_search` unless you need
+content from *every* result; otherwise search first, then `web_fetch` only the
+URLs you want.