diff --git a/src/lib/responses-translation.ts b/src/lib/responses-translation.ts new file mode 100644 index 000000000..82d02d6d2 --- /dev/null +++ b/src/lib/responses-translation.ts @@ -0,0 +1,327 @@ +// Translates between the Chat Completions dialect this project speaks +// everywhere (both /chat/completions and the Anthropic-compatible +// /v1/messages route funnel through createChatCompletions) and Copilot's +// Responses API, which is the only endpoint some models accept requests on +// (see Model["supported_endpoints"] in services/copilot/get-models.ts). +// Keeping the translation here - rather than in the routes/handlers - means +// every existing caller of createChatCompletions keeps working unchanged. + +import type { + ChatCompletionChunk, + ChatCompletionResponse, + ChatCompletionsPayload, + Message, + TextPart, + ToolCall, +} from "~/services/copilot/create-chat-completions" +import type { + ResponseContentPart, + ResponseInputItem, + ResponseMessageItem, + ResponsesPayload, + ResponsesResult, + ResponseStreamEvent, +} from "~/services/copilot/create-responses" + +export function chatPayloadToResponsesPayload( + payload: ChatCompletionsPayload, +): ResponsesPayload { + const input: Array = [] + + for (const message of payload.messages) { + if (message.role === "tool") { + input.push({ + type: "function_call_output", + call_id: message.tool_call_id ?? "", + output: contentToText(message.content), + }) + continue + } + + if (message.tool_calls && message.tool_calls.length > 0) { + if (message.content) { + input.push(toMessageItem(message)) + } + for (const toolCall of message.tool_calls) { + input.push({ + type: "function_call", + call_id: toolCall.id, + name: toolCall.function.name, + arguments: toolCall.function.arguments, + }) + } + continue + } + + input.push(toMessageItem(message)) + } + + return { + model: payload.model, + input, + max_output_tokens: payload.max_tokens, + temperature: payload.temperature, + top_p: payload.top_p, + stream: payload.stream, + tools: payload.tools?.map((tool) => ({ + type: "function" as const, + name: tool.function.name, + description: tool.function.description, + parameters: tool.function.parameters, + })), + tool_choice: translateToolChoice(payload.tool_choice), + } +} + +function toMessageItem(message: Message): ResponseMessageItem { + return { + type: "message", + role: message.role as ResponseMessageItem["role"], + content: contentToParts(message.content, message.role), + } +} + +function contentToParts( + content: Message["content"], + role: Message["role"], +): Array { + const textType: "input_text" | "output_text" = + role === "assistant" ? "output_text" : "input_text" + + if (content === null) return [] + if (typeof content === "string") { + return content.length > 0 ? [{ type: textType, text: content }] : [] + } + + return content.map((part): ResponseContentPart => { + if (part.type === "image_url") { + return { type: "input_image", image_url: part.image_url.url } + } + return { type: textType, text: part.text } + }) +} + +function contentToText(content: Message["content"]): string { + if (content === null) return "" + if (typeof content === "string") return content + return content + .filter((part): part is TextPart => part.type === "text") + .map((part) => part.text) + .join("") +} + +function finalFinishReason( + hasToolCalls: boolean, + incomplete: boolean, +): "stop" | "length" | "tool_calls" { + if (hasToolCalls) return "tool_calls" + if (incomplete) return "length" + return "stop" +} + +function translateToolChoice( + toolChoice: ChatCompletionsPayload["tool_choice"], +): ResponsesPayload["tool_choice"] { + if (!toolChoice || typeof toolChoice === "string") return toolChoice + return { type: "function", name: toolChoice.function.name } +} + +export function responsesResultToChatCompletion( + result: ResponsesResult, +): ChatCompletionResponse { + let content: string | null = null + const toolCalls: Array = [] + + for (const item of result.output) { + if (item.type === "message") { + const text = item.content.map((part) => part.text).join("") + content = (content ?? "") + text + } else { + toolCalls.push({ + id: item.call_id, + type: "function", + function: { name: item.name, arguments: item.arguments }, + }) + } + } + + const finishReason = finalFinishReason( + toolCalls.length > 0, + result.status === "incomplete", + ) + + return { + id: result.id, + object: "chat.completion", + created: result.created_at, + model: result.model, + choices: [ + { + index: 0, + message: { + role: "assistant", + content, + ...(toolCalls.length > 0 && { tool_calls: toolCalls }), + }, + logprobs: null, + finish_reason: finishReason, + }, + ], + ...(result.usage && { + usage: { + prompt_tokens: result.usage.input_tokens, + completion_tokens: result.usage.output_tokens, + total_tokens: result.usage.total_tokens, + ...(result.usage.input_tokens_details?.cached_tokens !== undefined && { + prompt_tokens_details: { + cached_tokens: result.usage.input_tokens_details.cached_tokens, + }, + }), + }, + }), + } +} + +interface StreamAccumulator { + id: string + created: number + model: string + toolCallIndexByItemId: Map + nextToolIndex: number +} + +interface ChunkDelta { + role?: "assistant" + content?: string + tool_calls?: Array<{ + index: number + id?: string + type?: "function" + function?: { name?: string; arguments?: string } + }> +} + +interface ChunkOptions { + finishReason?: ChatCompletionChunk["choices"][number]["finish_reason"] + usage?: ChatCompletionChunk["usage"] +} + +function chatChunk( + acc: StreamAccumulator, + delta: ChunkDelta, + options: ChunkOptions = {}, +): ChatCompletionChunk { + const { finishReason = null, usage } = options + return { + id: acc.id, + object: "chat.completion.chunk", + created: acc.created, + model: acc.model, + choices: [{ index: 0, delta, finish_reason: finishReason, logprobs: null }], + ...(usage && { usage }), + } +} + +function translateResponseStreamEvent( + event: ResponseStreamEvent, + acc: StreamAccumulator, +): Array { + switch (event.type) { + case "response.created": + case "response.in_progress": { + const { response } = event + acc.id = response.id + acc.created = response.created_at + acc.model = response.model + return event.type === "response.created" ? + [chatChunk(acc, { role: "assistant", content: "" })] + : [] + } + case "response.output_text.delta": { + return [chatChunk(acc, { content: event.delta })] + } + case "response.output_item.added": { + const { item } = event + if (item.type !== "function_call") return [] + + const index = acc.nextToolIndex + acc.nextToolIndex += 1 + acc.toolCallIndexByItemId.set(item.id, index) + return [ + chatChunk(acc, { + tool_calls: [ + { + index, + id: item.call_id, + type: "function", + function: { name: item.name, arguments: "" }, + }, + ], + }), + ] + } + case "response.function_call_arguments.delta": { + const index = acc.toolCallIndexByItemId.get(event.item_id) + if (index === undefined) return [] + return [ + chatChunk(acc, { + tool_calls: [{ index, function: { arguments: event.delta } }], + }), + ] + } + case "response.completed": + case "response.incomplete": + case "response.failed": { + const { response } = event + const finishReason = finalFinishReason( + acc.toolCallIndexByItemId.size > 0, + event.type === "response.incomplete", + ) + const usage = response.usage && { + prompt_tokens: response.usage.input_tokens, + completion_tokens: response.usage.output_tokens, + total_tokens: response.usage.total_tokens, + } + return [chatChunk(acc, {}, { finishReason, usage })] + } + default: { + return [] + } + } +} + +const isTerminalResponseEvent = (type: string) => + type === "response.completed" + || type === "response.incomplete" + || type === "response.failed" + +export async function* translateResponsesStream( + source: AsyncIterable<{ data?: string; event?: string }>, +): AsyncGenerator<{ data: string }> { + const acc: StreamAccumulator = { + id: "resp-stream", + created: Math.floor(Date.now() / 1000), + model: "", + toolCallIndexByItemId: new Map(), + nextToolIndex: 0, + } + + for await (const rawEvent of source) { + if (!rawEvent.data || rawEvent.data === "[DONE]") continue + + let parsed: ResponseStreamEvent + try { + parsed = JSON.parse(rawEvent.data) as ResponseStreamEvent + } catch { + continue + } + + for (const chunk of translateResponseStreamEvent(parsed, acc)) { + yield { data: JSON.stringify(chunk) } + } + + if (isTerminalResponseEvent(parsed.type)) { + yield { data: "[DONE]" } + } + } +} diff --git a/src/services/copilot/create-chat-completions.ts b/src/services/copilot/create-chat-completions.ts index 8534151da..ca549d3bd 100644 --- a/src/services/copilot/create-chat-completions.ts +++ b/src/services/copilot/create-chat-completions.ts @@ -3,13 +3,57 @@ import { events } from "fetch-event-stream" import { copilotHeaders, copilotBaseUrl } from "~/lib/api-config" import { HTTPError } from "~/lib/error" +import { + chatPayloadToResponsesPayload, + responsesResultToChatCompletion, + translateResponsesStream, +} from "~/lib/responses-translation" import { state } from "~/lib/state" +import type { Model } from "./get-models" + +import { createResponses, type ResponsesResult } from "./create-responses" + +// Some models (the GPT reasoning family: gpt-5.x/6.x, gpt-6-astra, +// mai-code-1.1-flash, ...) are Responses-API-only on Copilot's side - +// /chat/completions rejects them with "not accessible via the +// /chat/completions endpoint", regardless of payload shape. Model. +// supported_endpoints (from GET /models) says so upfront; when it's +// present and excludes /chat/completions, reroute through /responses and +// translate the result back, so every existing caller of this function - +// both the /chat/completions route and the Anthropic-compatible +// /v1/messages route - keeps working unchanged. +const supportsChatCompletionsEndpoint = (model: Model | undefined) => + !model?.supported_endpoints + || model.supported_endpoints.includes("/chat/completions") + +const supportsResponsesEndpoint = (model: Model | undefined) => + model?.supported_endpoints?.includes("/responses") ?? false + +const isResponsesNonStreaming = ( + result: Awaited>, +): result is ResponsesResult => Object.hasOwn(result, "output") + export const createChatCompletions = async ( payload: ChatCompletionsPayload, ) => { if (!state.copilotToken) throw new Error("Copilot token not found") + const selectedModel = state.models?.data.find((m) => m.id === payload.model) + + if ( + !supportsChatCompletionsEndpoint(selectedModel) + && supportsResponsesEndpoint(selectedModel) + ) { + consola.debug( + `Model ${payload.model} is Responses-API-only, rerouting through /responses`, + ) + const result = await createResponses(chatPayloadToResponsesPayload(payload)) + return isResponsesNonStreaming(result) ? + responsesResultToChatCompletion(result) + : translateResponsesStream(result) + } + const enableVision = payload.messages.some( (x) => typeof x.content !== "string" diff --git a/src/services/copilot/create-responses.ts b/src/services/copilot/create-responses.ts new file mode 100644 index 000000000..d869b8a7c --- /dev/null +++ b/src/services/copilot/create-responses.ts @@ -0,0 +1,172 @@ +import consola from "consola" +import { events } from "fetch-event-stream" + +import { copilotHeaders, copilotBaseUrl } from "~/lib/api-config" +import { HTTPError } from "~/lib/error" +import { state } from "~/lib/state" + +// Some newer models (the GPT reasoning family: gpt-5.x/6.x, mai-code, ...) +// are only reachable through Copilot's Responses API - /chat/completions +// rejects them with "not accessible via the /chat/completions endpoint". +// See Model["supported_endpoints"] in get-models.ts, which is how a caller +// knows to reach for this instead of create-chat-completions. +export const createResponses = async (payload: ResponsesPayload) => { + if (!state.copilotToken) throw new Error("Copilot token not found") + + const isAgentCall = payload.input.some( + (item) => + item.type === "function_call" || item.type === "function_call_output", + ) + + const headers: Record = { + ...copilotHeaders(state), + "X-Initiator": isAgentCall ? "agent" : "user", + } + + const response = await fetch(`${copilotBaseUrl(state)}/responses`, { + method: "POST", + headers, + body: JSON.stringify(payload), + }) + + if (!response.ok) { + consola.error("Failed to create response", response) + throw new HTTPError("Failed to create response", response) + } + + if (payload.stream) { + return events(response) + } + + return (await response.json()) as ResponsesResult +} + +// Request types + +export interface ResponsesPayload { + model: string + input: Array + instructions?: string | null + max_output_tokens?: number | null + temperature?: number | null + top_p?: number | null + stream?: boolean | null + tools?: Array | null + tool_choice?: + "none" | "auto" | "required" | { type: "function"; name: string } | null + reasoning?: { effort: string } | null +} + +export type ResponseInputItem = + | ResponseMessageItem + | ResponseFunctionCallItem + | ResponseFunctionCallOutputItem + +export interface ResponseMessageItem { + type: "message" + role: "system" | "developer" | "user" | "assistant" + content: Array +} + +export interface ResponseFunctionCallItem { + type: "function_call" + call_id: string + name: string + arguments: string +} + +export interface ResponseFunctionCallOutputItem { + type: "function_call_output" + call_id: string + output: string +} + +export type ResponseContentPart = + | { type: "input_text"; text: string } + | { type: "output_text"; text: string } + | { type: "input_image"; image_url: string } + +export interface ResponseTool { + type: "function" + name: string + description?: string + parameters: Record +} + +// Result types (non-streaming) + +export interface ResponsesResult { + id: string + object: "response" + created_at: number + model: string + status: "completed" | "incomplete" | "failed" | "in_progress" + output: Array + usage?: ResponsesUsage +} + +export type ResponseOutputItem = + | { + type: "message" + id: string + role: "assistant" + status: string + content: Array<{ type: "output_text"; text: string }> + } + | { + type: "function_call" + id: string + call_id: string + name: string + arguments: string + status: string + } + +export interface ResponsesUsage { + input_tokens: number + output_tokens: number + total_tokens: number + input_tokens_details?: { cached_tokens?: number } +} + +// Streaming event types - only the ones we translate; unknown types are +// forwarded as no-ops by the translator. + +export type ResponseStreamEvent = + | { type: "response.created"; response: ResponsesResult } + | { type: "response.in_progress"; response: ResponsesResult } + | { + type: "response.output_item.added" + output_index: number + item: ResponseOutputItem + } + | { + type: "response.output_item.done" + output_index: number + item: ResponseOutputItem + } + | { + type: "response.output_text.delta" + item_id: string + output_index: number + delta: string + } + | { type: "response.output_text.done"; item_id: string } + | { + type: "response.function_call_arguments.delta" + item_id: string + output_index: number + delta: string + } + | { type: "response.function_call_arguments.done"; item_id: string } + | { type: "response.completed"; response: ResponsesResult } + | { type: "response.incomplete"; response: ResponsesResult } + | { type: "response.failed"; response: ResponsesResult } + +// The Responses API has many more event types than we translate above +// (response.content_part.*, response.output_text.done, ...). The stream +// translator's `default` switch branch is a no-op for anything that isn't +// one of the literal members of ResponseStreamEvent - the single `as +// ResponseStreamEvent` assertion at the JSON.parse call site is what lets +// an unrecognized `type` reach that default branch instead of failing to +// compile. diff --git a/src/services/copilot/get-models.ts b/src/services/copilot/get-models.ts index 3cfa30af0..cd79a409f 100644 --- a/src/services/copilot/get-models.ts +++ b/src/services/copilot/get-models.ts @@ -52,4 +52,11 @@ export interface Model { state: string terms: string } + // Present on newer models to say which upstream endpoint(s) accept them - + // e.g. the GPT reasoning family (gpt-5.x/6.x) only lists "/responses", + // NOT "/chat/completions". Absent on older models, which are implicitly + // /chat/completions-only. See create-chat-completions.ts, which reroutes + // through /responses (lib/responses-translation.ts) for models that + // don't list /chat/completions here. + supported_endpoints?: Array }