feat(search): add Firecrawl search provider + standalone web_fetch tool

.changeset/firecrawl-search-provider.md

@@ -0,0 +1,12 @@
+---
+"@agent-infra/search": minor
+"@agent-infra/shared": minor
+---
+
+Add Firecrawl as a first-class search provider.
+
+`@agent-infra/shared` gains `SearchProvider.Firecrawl`, and `@agent-infra/search`
+adds a `firecrawl` provider (wrapping the `firecrawl` SDK) to the unified
+`SearchClient`, alongside the existing browser/Bing/Tavily/DuckDuckGo/SearXNG
+providers. Firecrawl's `/search` returns clean, LLM-ready markdown for every
+result in a single call; pass `scrapeOptions` to retrieve full-page content.

multimodal/agent-tars/cli/src/index.ts

@@ -85,10 +85,18 @@ export class AgentTARSCLI extends AgentCLI {
         .option('--search <search>', 'Search config')
         .option(
           '--search.provider [provider]',
-          'Search provider (browser_search, tavily, bing_search)',
+          'Search provider (browser_search, tavily, bing_search, firecrawl)',
         )
         .option('--search.count [count]', 'Search result count', { default: 10 })
         .option('--search.apiKey [apiKey]', 'Search API key')
+
+        // Fetch configuration (standalone `web_fetch` tool, independent of search)
+        .option('--fetch <fetch>', 'Fetch config')
+        .option(
+          '--fetch.apiKey [apiKey]',
+          'Firecrawl API key for web_fetch (optional; keyless tier available)',
+        )
+        .option('--fetch.baseUrl [baseUrl]', 'Firecrawl base URL override (self-hosted)')
     );
   }
 

multimodal/agent-tars/core/src/environments/local/fetch/fetch-tool.ts

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { ConsoleLogger, Tool, z } from '@tarko/mcp-agent';
+import { firecrawl } from '@agent-infra/search';
+
+/**
+ * Configuration for the fetch tool. Reuses the search provider's credentials
+ * (`apiKey` is optional — Firecrawl has a keyless free tier; `baseUrl` targets
+ * a self-hosted instance).
+ */
+export interface FetchToolConfig {
+  apiKey?: string;
+  baseUrl?: string;
+}
+
+/**
+ * FetchToolProvider — a `web_fetch` tool backed by Firecrawl's scrape API.
+ *
+ * Reads the full, LLM-ready content of a single URL without driving the
+ * browser. Distinct from search (`web_search`, discovery) and from the
+ * headless browser's `browser_get_markdown` (which only reads the tab the
+ * browser is already on). Kept in its own provider so the search tool stays
+ * search-only.
+ */
+export class FetchToolProvider {
+  private logger: ConsoleLogger;
+  private config: FetchToolConfig;
+
+  constructor(logger: ConsoleLogger, config: FetchToolConfig) {
+    this.logger = logger.spawn('FetchToolProvider');
+    this.config = config;
+  }
+
+  /**
+   * Create a `web_fetch` tool definition for agent registration.
+   */
+  createFetchTool(): Tool {
+    const client = firecrawl({
+      apiKey: this.config.apiKey,
+      apiUrl: this.config.baseUrl,
+    });
+
+    return new Tool({
+      id: 'web_fetch',
+      description:
+        'Fetch the full content of a specific web page as clean, LLM-ready ' +
+        'markdown — without opening it in the browser. Use this when you ' +
+        'already have a URL (e.g. from web_search results) and need its full ' +
+        'text, not just a snippet. Handles JavaScript-rendered pages and PDFs.',
+      parameters: z.object({
+        url: z
+          .string()
+          .describe('The full URL to fetch (must start with http or https).'),
+        formats: z
+          .array(z.enum(['markdown', 'html', 'links']))
+          .optional()
+          .describe('Output formats to return. Defaults to ["markdown"].'),
+      }),
+      function: async ({ url, formats }) => {
+        if (!url || url.trim() === '') {
+          return { error: 'A url is required' };
+        }
+
+        // Enforce the http(s) contract stated in the description, and reject
+        // other schemes (file:, javascript:, ...) before spending a request.
+        let parsed: URL;
+        try {
+          parsed = new URL(url);
+        } catch {
+          return { error: 'A valid URL is required' };
+        }
+        if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+          return { error: 'URL must start with http:// or https://' };
+        }
+
+        // Log only origin + path — query strings may carry tokens / signed-link
+        // credentials that should not land in logs.
+        const safeUrl = `${parsed.origin}${parsed.pathname}`;
+
+        try {
+          this.logger.info(`Fetching: "${safeUrl}"`);
+
+          const doc = await client.scrape(url, {
+            formats: formats?.length ? formats : ['markdown'],
+            onlyMainContent: true,
+          });
+
+          return {
+            url,
+            title: doc.metadata?.title,
+            markdown: doc.markdown,
+            html: doc.html,
+            links: doc.links,
+            metadata: doc.metadata,
+          };
+        } catch (error) {
+          this.logger.error(`Fetch error for "${safeUrl}": ${error}`);
+          return {
+            error: `Fetch failed: ${error instanceof Error ? error.message : String(error)}`,
+          };
+        }
+      },
+    });
+  }
+}

multimodal/agent-tars/core/src/environments/local/fetch/index.ts

@@ -0,0 +1,6 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './fetch-tool';

multimodal/agent-tars/core/src/environments/local/index.ts

@@ -16,6 +16,7 @@ import { ResourceCleaner } from '../../utils';
 import { AgentTARSOptions, BuiltInMCPServers, BuiltInMCPServerName } from '../../types';
 import { BrowserGUIAgent, BrowserManager, BrowserToolsManager } from './browser';
 import { SearchToolProvider } from './search';
+import { FetchToolProvider } from './fetch';
 import { FilesystemToolsManager } from './filesystem';
 import { WorkspacePathResolver } from '../../shared/workspace-path-resolver';
 import { AgentTARSBaseEnvironment } from '../base';
@@ -42,6 +43,7 @@ export class AgentTARSLocalEnvironment extends AgentTARSBaseEnvironment {
   private browserToolsManager?: BrowserToolsManager;
   private filesystemToolsManager?: FilesystemToolsManager;
   private searchToolProvider?: SearchToolProvider;
+  private fetchToolProvider?: FetchToolProvider;
   private browserGUIAgent?: BrowserGUIAgent;
   private mcpServers: BuiltInMCPServers = {};
   private mcpClients: Partial<Record<BuiltInMCPServerName, Client>> = {};
@@ -88,6 +90,11 @@ export class AgentTARSLocalEnvironment extends AgentTARSBaseEnvironment {
       await this.initializeSearchTools(registerToolFn);
     }
 
+    // Initialize fetch tool (independent of search)
+    if (this.options.fetch) {
+      await this.initializeFetchTools(registerToolFn);
+    }
+
     // Initialize MCP servers if using in-memory implementation
     if (this.options.mcpImpl === 'in-memory') {
       await this.initializeInMemoryMCP(registerToolFn);
@@ -137,6 +144,25 @@ export class AgentTARSLocalEnvironment extends AgentTARSBaseEnvironment {
     this.logger.info('✅ Search tools initialized successfully');
   }
 
+  /**
+   * Initialize the fetch tool (`web_fetch`).
+   *
+   * Standalone capability — reads any URL to clean markdown via Firecrawl's
+   * scrape API. Independent of the search provider; configured via
+   * `options.fetch`.
+   */
+  private async initializeFetchTools(registerToolFn: (tool: Tool) => void): Promise<void> {
+    this.logger.info('📄 Initializing fetch tool');
+
+    this.fetchToolProvider = new FetchToolProvider(this.logger, {
+      apiKey: this.options.fetch!.apiKey,
+      baseUrl: this.options.fetch!.baseUrl,
+    });
+    registerToolFn(this.fetchToolProvider.createFetchTool());
+
+    this.logger.info('✅ Fetch tool (web_fetch) initialized successfully');
+  }
+
   /**
    * Initialize in-memory MCP servers and clients
    */

multimodal/agent-tars/core/src/environments/local/search/search-tool.ts

@@ -80,6 +80,7 @@ export class SearchToolProvider {
       tavily: SearchProvider.Tavily,
       searxng: SearchProvider.SearXNG,
       duckduckgo: SearchProvider.DuckduckgoSearch,
+      firecrawl: SearchProvider.Firecrawl,
     };
 
     const resolvedProvider = providerMap[provider] || SearchProvider.BrowserSearch;

multimodal/agent-tars/interface/src/cli.ts

@@ -17,7 +17,7 @@ import { AgentTARSAppConfig } from './config';
  */
 export type AgentTARSCLIArguments = Pick<
   AgentTARSAppConfig,
-  'workspace' | 'browser' | 'planner' | 'search' | 'agio'
+  'workspace' | 'browser' | 'planner' | 'search' | 'fetch' | 'agio'
 > &
   AgentCLIArguments & {
     // Deprecated shortcut options for backward compatibility

multimodal/agent-tars/interface/src/core.ts

@@ -62,19 +62,20 @@ export interface AgentTARSSearchOptions {
    *
    * @defaultValue 'browser_search'
    */
-  provider: 'browser_search' | 'tavily' | 'bing_search';
+  provider: 'browser_search' | 'tavily' | 'bing_search' | 'firecrawl';
   /**
    * Search result count
    *
    * @defaultValue `10`
    */
   count?: number;
   /**
-   * Optional api key, required for tavily and bing_search.
+   * Optional api key, required for tavily and bing_search. Optional for
+   * firecrawl (keyless free tier available; provide `fc-...` for higher limits).
    */
   apiKey?: string;
   /**
-   * Optional api key, required for tavily and bing_search.
+   * Optional base url override (e.g. a self-hosted provider instance).
    */
   baseUrl?: string;
   /**
@@ -94,6 +95,28 @@ export interface AgentTARSSearchOptions {
   };
 }
 
+/**
+ * Fetch options for Agent TARS.
+ *
+ * Configures the `web_fetch` tool, which reads the full, LLM-ready content of a
+ * single URL (markdown/html/links) without driving the browser. This is an
+ * independent capability from search — it can be enabled on its own, alongside
+ * any (or no) search provider.
+ *
+ * Backed by Firecrawl's scrape API. An api key is optional (keyless free tier
+ * available); provide `fc-...` for higher rate limits.
+ */
+export interface AgentTARSFetchOptions {
+  /**
+   * Optional Firecrawl api key (`fc-...`). Optional on the keyless free tier.
+   */
+  apiKey?: string;
+  /**
+   * Optional base url override (e.g. a self-hosted Firecrawl instance).
+   */
+  baseUrl?: string;
+}
+
 /**
  * Options for the planning system within Agent TARS
  */
@@ -137,6 +160,12 @@ export interface AgentTARSOptions extends MCPAgentOptions {
    */
   search?: AgentTARSSearchOptions;
 
+  /**
+   * Fetch settings. Enables the standalone `web_fetch` tool (read any URL to
+   * clean markdown). Independent of `search`.
+   */
+  fetch?: AgentTARSFetchOptions;
+
   /**
    * Browser options
    */

multimodal/websites/main/src/docs/source/docs/guide/quick-start.md

@@ -107,13 +107,14 @@ For search config, you can set the search provider:
 
 The comparison of search providers is as follows:
 
-| Search Provider                | Need API Key? | Speed       |
-| ------------------------------ | ------------- | ----------- |
-| Local Browser Search (Default) | NO            | Slow        |
-| Tavily                         | YES           | Fast        |
-| Bing Search                    | YES           | Fast        |
-| SearXNG Search                 | NO            | ❓Unknown   |
-| Duckduckgo Search              | NO            | ⚠️ Unstable |
+| Search Provider                | Need API Key?           | Speed       |
+| ------------------------------ | ----------------------- | ----------- |
+| Local Browser Search (Default) | NO                      | Slow        |
+| Tavily                         | YES                     | Fast        |
+| Bing Search                    | YES                     | Fast        |
+| Firecrawl                      | Optional (keyless tier) | Fast        |
+| SearXNG Search                 | NO                      | ❓Unknown   |
+| Duckduckgo Search              | NO                      | ⚠️ Unstable |
 
 ---
 
@@ -127,6 +128,19 @@ You can click **Test Search Service** button to check if current search setting
 
 ---
 
+---
+
+### Config Web Fetch
+
+Beyond search, Firecrawl also provides a standalone `web_fetch` tool that reads any URL as clean, LLM-ready markdown (including JavaScript-rendered pages and PDFs), without opening the browser. It is configured independently of search via the top-level `fetch` option, so you can pair it with any search provider or enable it on its own.
+
+| Option          | Description                                            |
+| --------------- | ------------------------------------------------------ |
+| `fetch.apiKey`  | Optional Firecrawl key (keyless free tier also works). |
+| `fetch.baseUrl` | Optional. Point at a self-hosted Firecrawl instance.   |
+
+---
+
 ## Start your first task
 
 Now you can start your first journey in Agent TARS! You can input your first question in the input box, and then press Enter to send your question.

packages/agent-infra/mcp-servers/search/src/server.ts

@@ -42,6 +42,7 @@ export function setSearchConfig(config: Partial<SearchSettings>) {
 const API_KEY_ENV_MAP = {
   [SearchProvider.BingSearch]: process.env.BING_SEARCH_API_KEY,
   [SearchProvider.Tavily]: process.env.TAVILY_API_KEY,
+  [SearchProvider.Firecrawl]: process.env.FIRECRAWL_API_KEY,
   [SearchProvider.BrowserSearch]: undefined,
   [SearchProvider.SearXNG]: undefined,
   [SearchProvider.DuckduckgoSearch]: undefined,
@@ -50,6 +51,7 @@ const API_KEY_ENV_MAP = {
 const API_BASE_URL_ENV_MAP = {
   [SearchProvider.BingSearch]: process.env.BING_SEARCH_API_BASE_URL,
   [SearchProvider.Tavily]: undefined,
+  [SearchProvider.Firecrawl]: process.env.FIRECRAWL_API_BASE_URL,
   [SearchProvider.BrowserSearch]: undefined,
   [SearchProvider.SearXNG]: undefined,
   [SearchProvider.DuckduckgoSearch]: undefined,