feat(fetch-url): support fetching images as multimodal content

packages/agent-core/src/tools/builtin/web/fetch-url.md

@@ -1,3 +1,3 @@
-Fetch content from a URL. The content is returned either as the main text extracted from the page, or as the full response body verbatim; a note at the top of the result states which of the two you received, so you can judge how complete it is. Use this when you need to read a specific web page.
+Fetch content from a URL. The content is returned either as the main text extracted from the page, as the full response body verbatim, or (for image URLs) as the image itself; a note at the top of the result states which of the two you received, so you can judge how complete it is. Use this when you need to read a specific web page or fetch an image from the internet.
 
 Only fully-formed public `http`/`https` URLs are supported; other schemes and private or loopback addresses are not fetched. Very large pages may be truncated or refused. The fetch carries no login or session for the target site, so pages behind authentication (private repositories, internal dashboards) return a login page or an error instead of the real content — if the text you get back looks like a generic landing or sign-in page, treat that as the login wall, not the answer, and reach the content through a credentialed route (an authenticated CLI or MCP tool) instead.

packages/agent-core/src/tools/builtin/web/fetch-url.ts

@@ -25,14 +25,18 @@ import DESCRIPTION from './fetch-url.md?raw';
  *   returned verbatim, in full.
  * - `extracted` — the body was an HTML page; only the main article text
  *   was extracted and returned.
+ * - `image` — the body was an image file; returned as a base64 data URL
+ *   for direct model consumption.
  */
-export type UrlFetchKind = 'passthrough' | 'extracted';
+export type UrlFetchKind = 'passthrough' | 'extracted' | 'image';
 
 export interface UrlFetchResult {
-  /** The text handed to the LLM. */
+  /** The text handed to the LLM (for passthrough / extracted). */
   content: string;
   /** Whether `content` is a verbatim passthrough or extracted main text. */
   kind: UrlFetchKind;
+  /** Present when `kind === 'image'` — the image as a base64 data URL. */
+  imageData?: { mimeType: string; base64: string };
 }
 
 export interface UrlFetcher {
@@ -89,7 +93,22 @@ export class FetchURLTool implements BuiltinTool<FetchURLInput> {
     }: ExecutableToolContext,
   ): Promise<ExecutableToolResult> {
     try {
-      const { content, kind } = await this.fetcher.fetch(args.url, { toolCallId });
+      const { content, kind, imageData } = await this.fetcher.fetch(args.url, { toolCallId });
+
+      // Image response: return as multimodal ContentPart[] so the model
+      // can see the image directly.
+      if (kind === 'image' && imageData !== undefined) {
+        return {
+          output: [
+            { type: 'text', text: `Fetched image from ${args.url}` },
+            {
+              type: 'image_url',
+              imageUrl: { url: `data:${imageData.mimeType};base64,${imageData.base64}` },
+            },
+          ],
+          isError: false,
+        };
+      }
 
       if (!content) {
         return {

packages/agent-core/src/tools/providers/local-fetch-url.ts

@@ -237,6 +237,26 @@ export class LocalFetchURLProvider implements UrlFetcher {
       );
     }
 
+    const contentType = (response.headers.get('content-type') ?? '').toLowerCase();
+
+    // Image responses: read binary body and return as base64 for direct
+    // model consumption.
+    if (contentType.startsWith('image/')) {
+      const arrayBuffer = await response.arrayBuffer();
+      const bytes = Buffer.from(arrayBuffer);
+      if (bytes.length > this.maxBytes) {
+        throw new Error(
+          `Response body too large: ${String(bytes.length)} bytes exceeds maxBytes (${String(this.maxBytes)}).`,
+        );
+      }
+      const base64 = bytes.toString('base64');
+      return {
+        content: `Image fetched (${contentType}, ${String(bytes.length)} bytes)`,
+        kind: 'image',
+        imageData: { mimeType: contentType.split(';')[0]!.trim(), base64 },
+      };
+    }
+
     // Reject oversized responses before buffering the full body.
     const contentLengthRaw = response.headers.get('content-length');
     if (contentLengthRaw !== null) {
@@ -263,7 +283,6 @@ export class LocalFetchURLProvider implements UrlFetcher {
       );
     }
 
-    const contentType = (response.headers.get('content-type') ?? '').toLowerCase();
     if (contentType.startsWith('text/plain') || contentType.startsWith('text/markdown')) {
       return { content: body, kind: 'passthrough' };
     }

packages/agent-core/test/tools/fetch-url.test.ts

@@ -105,6 +105,32 @@ describe('FetchURLTool', () => {
     expect(out).toContain('# Raw markdown');
   });
 
+  it('returns image data as multimodal ContentPart[] when fetcher returns an image', async () => {
+    const fetcher: UrlFetcher = {
+      fetch: vi.fn().mockResolvedValue({
+        content: 'Image fetched (image/png, 8 bytes)',
+        kind: 'image',
+        imageData: { mimeType: 'image/png', base64: 'dGVzdA==' },
+      }),
+    };
+    const tool = new FetchURLTool(fetcher);
+    const result = await executeTool(tool, {
+      turnId: 't1',
+      toolCallId: 'c-img',
+      args: { url: 'https://example.com/image.png' },
+      signal,
+    });
+    expect(result.isError).toBe(false);
+    expect(Array.isArray(result.output)).toBe(true);
+    const parts = result.output as Array<{ type: string }>;
+    expect(parts).toHaveLength(2);
+    expect(parts[0]).toMatchObject({ type: 'text' });
+    expect(parts[1]).toMatchObject({
+      type: 'image_url',
+      imageUrl: { url: 'data:image/png;base64,dGVzdA==' },
+    });
+  });
+
   it('returns empty message when fetcher returns empty string', async () => {
     const tool = new FetchURLTool(fakeFetcher(''));
     const result = await executeTool(tool, {

packages/agent-core/test/tools/providers/local-fetch-url.test.ts

@@ -88,6 +88,40 @@ describe('LocalFetchURLProvider content kind', () => {
     expect(result.kind).toBe('extracted');
     expect(result.content).toContain('quick brown fox');
   });
+
+  it('reports image/* bodies as image kind with base64 data', async () => {
+    const pngBytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+    const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(
+      new Response(pngBytes, {
+        status: 200,
+        headers: { 'content-type': 'image/png' },
+      }),
+    );
+    const provider = new LocalFetchURLProvider({ fetchImpl });
+
+    const result = await provider.fetch('https://example.com/image.png');
+
+    expect(result.kind).toBe('image');
+    expect(result.imageData).toBeDefined();
+    expect(result.imageData!.mimeType).toBe('image/png');
+    expect(result.imageData!.base64).toBe(pngBytes.toString('base64'));
+    expect(result.content).toContain('Image fetched');
+  });
+
+  it('rejects oversized image responses', async () => {
+    const bigBuffer = Buffer.alloc(11 * 1024 * 1024, 0xff);
+    const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(
+      new Response(bigBuffer, {
+        status: 200,
+        headers: { 'content-type': 'image/png' },
+      }),
+    );
+    const provider = new LocalFetchURLProvider({ fetchImpl });
+
+    await expect(provider.fetch('https://example.com/huge.png')).rejects.toThrow(
+      'Response body too large',
+    );
+  });
 });
 
 describe('LocalFetchURLProvider SSRF guard', () => {