diff --git a/src/api/providers/fetchers/__tests__/litellm.spec.ts b/src/api/providers/fetchers/__tests__/litellm.spec.ts index c05cda883..900c4c0c4 100644 --- a/src/api/providers/fetchers/__tests__/litellm.spec.ts +++ b/src/api/providers/fetchers/__tests__/litellm.spec.ts @@ -697,4 +697,148 @@ describe("getLiteLLMModels", () => { description: "model-with-only-max-output-tokens via LiteLLM proxy", }) }) + + it("falls back to /v1/models when /v1/model/info returns 403 Forbidden", async () => { + const infoError = { + message: "Request failed with status code 403", + response: { status: 403, statusText: "Forbidden" }, + isAxiosError: true, + } + const mockFallbackResponse = { + data: { + data: [{ id: "fallback-model-1" }, { id: "fallback-model-2" }], + }, + } + + mockedAxios.get.mockRejectedValueOnce(infoError).mockResolvedValueOnce(mockFallbackResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(mockedAxios.get).toHaveBeenNthCalledWith(1, "http://localhost:4000/v1/model/info", expect.any(Object)) + expect(mockedAxios.get).toHaveBeenNthCalledWith(2, "http://localhost:4000/v1/models", expect.any(Object)) + + expect(result).toEqual({ + "fallback-model-1": { + maxTokens: 8192, + contextWindow: 128000, + supportsImages: false, + supportsPromptCache: false, + description: "fallback-model-1 via LiteLLM proxy", + }, + "fallback-model-2": { + maxTokens: 8192, + contextWindow: 128000, + supportsImages: false, + supportsPromptCache: false, + description: "fallback-model-2 via LiteLLM proxy", + }, + }) + }) + + it("throws original error when both /v1/model/info and fallback /v1/models fail", async () => { + const infoError = { + message: "Forbidden", + response: { status: 403, statusText: "Forbidden" }, + isAxiosError: true, + } + const fallbackError = new Error("Models endpoint 404") + + mockedAxios.isAxiosError.mockReturnValue(true) + mockedAxios.get.mockRejectedValueOnce(infoError).mockRejectedValueOnce(fallbackError) + + await expect(getLiteLLMModels("test-api-key", "http://localhost:4000")).rejects.toThrow( + "Failed to fetch LiteLLM models: 403 Forbidden. Check base URL and API key.", + ) + }) + + it("does not attempt fallback and throws immediately when /v1/model/info returns non-403 error", async () => { + const infoError = { + message: "Internal Server Error", + response: { status: 500, statusText: "Internal Server Error" }, + isAxiosError: true, + } + + mockedAxios.isAxiosError.mockReturnValue(true) + mockedAxios.get.mockRejectedValueOnce(infoError) + + await expect(getLiteLLMModels("test-api-key", "http://localhost:4000")).rejects.toThrow( + "Failed to fetch LiteLLM models: 500 Internal Server Error. Check base URL and API key.", + ) + + expect(mockedAxios.get).toHaveBeenCalledTimes(1) + expect(mockedAxios.get).toHaveBeenNthCalledWith(1, "http://localhost:4000/v1/model/info", expect.any(Object)) + }) + + it("falls back using error.status when error.response is undefined", async () => { + const infoError = { + message: "Forbidden status on error directly", + status: 403, + isAxiosError: true, + } + const mockFallbackResponse = { + data: { + data: [{ id: "fallback-model-status" }], + }, + } + + mockedAxios.get.mockRejectedValueOnce(infoError).mockResolvedValueOnce(mockFallbackResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(result).toEqual({ + "fallback-model-status": { + maxTokens: 8192, + contextWindow: 128000, + supportsImages: false, + supportsPromptCache: false, + description: "fallback-model-status via LiteLLM proxy", + }, + }) + }) + + it("skips fallback models that do not have an ID", async () => { + const infoError = { + message: "Forbidden", + response: { status: 403, statusText: "Forbidden" }, + isAxiosError: true, + } + const mockFallbackResponse = { + data: { + data: [{ id: "valid-fallback" }, { id: "" }, { name: "invalid" }], + }, + } + + mockedAxios.get.mockRejectedValueOnce(infoError).mockResolvedValueOnce(mockFallbackResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(result).toEqual({ + "valid-fallback": { + maxTokens: 8192, + contextWindow: 128000, + supportsImages: false, + supportsPromptCache: false, + description: "valid-fallback via LiteLLM proxy", + }, + }) + }) + + it("throws error when fallback response is not in expected format", async () => { + const infoError = { + message: "Forbidden", + response: { status: 403, statusText: "Forbidden" }, + isAxiosError: true, + } + const mockFallbackResponse = { + data: { + data: "not-an-array", + }, + } + + mockedAxios.get.mockRejectedValueOnce(infoError).mockResolvedValueOnce(mockFallbackResponse) + + await expect(getLiteLLMModels("test-api-key", "http://localhost:4000")).rejects.toThrow( + "Failed to fetch LiteLLM models: Failed to fetch LiteLLM models: Unexpected response format.", + ) + }) }) diff --git a/src/api/providers/fetchers/litellm.ts b/src/api/providers/fetchers/litellm.ts index de895d01f..063313bae 100644 --- a/src/api/providers/fetchers/litellm.ts +++ b/src/api/providers/fetchers/litellm.ts @@ -28,40 +28,88 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise urlObj.pathname = urlObj.pathname.replace(/\/+$/, "").replace(/\/+/g, "/") + "/v1/model/info" const url = urlObj.href // Added timeout to prevent indefinite hanging - const response = await axios.get(url, { headers, timeout: 5000 }) + let response + let isFallback = false + try { + response = await axios.get(url, { headers, timeout: 5000 }) + } catch (infoError: any) { + const status = infoError?.response?.status || infoError?.status + if (status !== 403) { + throw infoError + } + console.warn( + `[getLiteLLMModels] Failed to fetch /v1/model/info (status 403), attempting fallback to /v1/models:`, + infoError.message, + ) + const fallbackUrlObj = new URL(baseUrl) + fallbackUrlObj.pathname = fallbackUrlObj.pathname.replace(/\/+$/, "").replace(/\/+/g, "/") + "/v1/models" + const fallbackUrl = fallbackUrlObj.href + try { + response = await axios.get(fallbackUrl, { headers, timeout: 5000 }) + isFallback = true + } catch (fallbackError: any) { + console.warn(`[getLiteLLMModels] Fallback to /v1/models also failed:`, fallbackError.message) + throw infoError + } + } const models: ModelRecord = {} - // Process the model info from the response - if (response.data && response.data.data && Array.isArray(response.data.data)) { - for (const model of response.data.data) { - const modelName = model.model_name - const modelInfo = model.model_info - const litellmModelName = model?.litellm_params?.model as string | undefined - - if (!modelName || !modelInfo || !litellmModelName) continue + if (isFallback) { + if (response.data && response.data.data && Array.isArray(response.data.data)) { + for (const model of response.data.data) { + const modelName = model.id + if (!modelName) continue - models[modelName] = { - maxTokens: modelInfo.max_output_tokens || modelInfo.max_tokens || 8192, - contextWindow: modelInfo.max_input_tokens || 200000, - supportsImages: Boolean(modelInfo.supports_vision), - supportsPromptCache: Boolean(modelInfo.supports_prompt_caching), - inputPrice: modelInfo.input_cost_per_token ? modelInfo.input_cost_per_token * 1000000 : undefined, - outputPrice: modelInfo.output_cost_per_token - ? modelInfo.output_cost_per_token * 1000000 - : undefined, - cacheWritesPrice: modelInfo.cache_creation_input_token_cost - ? modelInfo.cache_creation_input_token_cost * 1000000 - : undefined, - cacheReadsPrice: modelInfo.cache_read_input_token_cost - ? modelInfo.cache_read_input_token_cost * 1000000 - : undefined, - description: `${modelName} via LiteLLM proxy`, + models[modelName] = { + maxTokens: 8192, + contextWindow: 128000, + supportsImages: false, + supportsPromptCache: false, + description: `${modelName} via LiteLLM proxy`, + } } + } else { + console.error( + "Error fetching LiteLLM models: Unexpected response format from /v1/models", + response.data, + ) + throw new Error("Failed to fetch LiteLLM models: Unexpected response format.") } } else { - // If response.data.data is not in the expected format, consider it an error. - console.error("Error fetching LiteLLM models: Unexpected response format", response.data) - throw new Error("Failed to fetch LiteLLM models: Unexpected response format.") + // Process the model info from the response + if (response.data && response.data.data && Array.isArray(response.data.data)) { + for (const model of response.data.data) { + const modelName = model.model_name + const modelInfo = model.model_info + const litellmModelName = model?.litellm_params?.model as string | undefined + + if (!modelName || !modelInfo || !litellmModelName) continue + + models[modelName] = { + maxTokens: modelInfo.max_output_tokens || modelInfo.max_tokens || 8192, + contextWindow: modelInfo.max_input_tokens || 200000, + supportsImages: Boolean(modelInfo.supports_vision), + supportsPromptCache: Boolean(modelInfo.supports_prompt_caching), + inputPrice: modelInfo.input_cost_per_token + ? modelInfo.input_cost_per_token * 1000000 + : undefined, + outputPrice: modelInfo.output_cost_per_token + ? modelInfo.output_cost_per_token * 1000000 + : undefined, + cacheWritesPrice: modelInfo.cache_creation_input_token_cost + ? modelInfo.cache_creation_input_token_cost * 1000000 + : undefined, + cacheReadsPrice: modelInfo.cache_read_input_token_cost + ? modelInfo.cache_read_input_token_cost * 1000000 + : undefined, + description: `${modelName} via LiteLLM proxy`, + } + } + } else { + // If response.data.data is not in the expected format, consider it an error. + console.error("Error fetching LiteLLM models: Unexpected response format", response.data) + throw new Error("Failed to fetch LiteLLM models: Unexpected response format.") + } } return models