Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 21 additions & 13 deletions packages/opencode/src/tool/webfetch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,24 @@ export const WebFetchTool = Tool.define(
const mime = contentType.split(";")[0]?.trim().toLowerCase() || ""
const title = `${params.url} (${contentType})`

// A non-200 success (202 Accepted, 204 No Content, ...) often carries a throttle
// or interstitial page instead of the requested content. Only `output` reaches the
// model, so the status has to be stated there for it to adapt.
const statusNotice =
response.status === 200
? ""
: `[HTTP ${response.status} — response may be a throttle or interstitial page, not the requested content]\n\n`

const result = (output: string) => ({
output: statusNotice + output,
title,
metadata: { status: response.status },
})

if (isImageAttachment(mime)) {
const base64Content = Buffer.from(arrayBuffer).toString("base64")
return {
title,
output: "Image fetched successfully",
metadata: {},
...result("Image fetched successfully"),
attachments: [
{
type: "file" as const,
Expand All @@ -130,25 +142,21 @@ export const WebFetchTool = Tool.define(
case "markdown":
if (contentType.includes("text/html")) {
const markdown = convertHTMLToMarkdown(content)
return {
output: markdown,
title,
metadata: {},
}
return result(markdown)
}
return { output: content, title, metadata: {} }
return result(content)

case "text":
if (contentType.includes("text/html")) {
return { output: extractTextFromHTML(content), title, metadata: {} }
return result(extractTextFromHTML(content))
}
return { output: content, title, metadata: {} }
return result(content)

case "html":
return { output: content, title, metadata: {} }
return result(content)

default:
return { output: content, title, metadata: {} }
return result(content)
}
}).pipe(Effect.orDie),
}
Expand Down
3 changes: 2 additions & 1 deletion packages/opencode/src/tool/webfetch.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
- Fetches content from a specified URL
- Takes a URL and optional format as input
- Fetches the URL content, converts to requested format (markdown by default)
- Returns the content in the specified format
- Returns the content in the specified format, preceded by a plain-text status notice if the response status is not 200
- Use this tool when you need to retrieve and analyze web content

Usage notes:
Expand All @@ -11,3 +11,4 @@ Usage notes:
- Format options: "markdown" (default), "text", or "html"
- This tool is read-only and does not modify any files
- Results may be summarized if the content is very large
- If the response status is not 200, a plain-text "[HTTP <status> ...]" line precedes the body regardless of the requested format; the body may be a throttle or interstitial page rather than the requested content
49 changes: 49 additions & 0 deletions packages/opencode/test/tool/webfetch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,55 @@ describe("tool.webfetch", () => {
),
)

it.instance("notes a non-200 status ahead of the body", () =>
withFetch(
() =>
new Response("<p>throttled</p>", {
status: 202,
headers: { "content-type": "text/html; charset=utf-8" },
}),
(url) =>
Effect.gen(function* () {
const result = yield* exec({ url: new URL("/page.html", url).toString(), format: "text" })
expect(result.output.startsWith("[HTTP 202")).toBe(true)
expect(result.output).toContain("throttled")
expect(result.metadata.status).toBe(202)
}),
),
)

it.instance("surfaces the notice alone when a throttled response has no body", () =>
withFetch(
() =>
new Response("", {
status: 202,
headers: { "content-type": "text/html; charset=utf-8" },
}),
(url) =>
Effect.gen(function* () {
const result = yield* exec({ url: new URL("/page.html", url).toString(), format: "text" })
expect(result.output.startsWith("[HTTP 202")).toBe(true)
expect(result.output.trim().endsWith("]")).toBe(true)
}),
),
)

it.instance("leaves 200 responses unannotated", () =>
withFetch(
() =>
new Response("<p>hello</p>", {
status: 200,
headers: { "content-type": "text/html; charset=utf-8" },
}),
(url) =>
Effect.gen(function* () {
const result = yield* exec({ url: new URL("/page.html", url).toString(), format: "text" })
expect(result.output).toBe("hello")
expect(result.metadata.status).toBe(200)
}),
),
)

it.instance("extracts text from html without scripts or styles", () =>
withFetch(
() =>
Expand Down
Loading