diff --git a/conf/gateway.yaml b/conf/gateway.yaml index a8d51df..13fdaa2 100644 --- a/conf/gateway.yaml +++ b/conf/gateway.yaml @@ -139,9 +139,13 @@ models: - name: whisper-1 # audio stt family protocol: stt unit_price_micros: 100 # per audio second - - name: kling-video # video family, synchronous vendor shape + - name: kling-v1-6 # Kling text2video: {code, data.task_id} envelope, billed per second protocol: video provider: kling + unit_price_micros: 60000 + - name: vidu-video # generic synchronous vendor shape (the video fallback wire) + protocol: video + provider: vidu - name: grok-imagine-video # async video: request_id + poll, billed per second on done protocol: video provider: xai @@ -255,6 +259,10 @@ accounts: provider: kling priority: 1 protocols: ["video"] + - name: mock-vidu-1 + provider: vidu + priority: 1 + protocols: ["video"] - name: mock-xai-1 # e2e: the xAI host answers video with a request_id + poll provider: xai endpoint: mock://api.x.ai diff --git a/crates/engines/src/families.rs b/crates/engines/src/families.rs index 8894736..42b0bc7 100644 --- a/crates/engines/src/families.rs +++ b/crates/engines/src/families.rs @@ -614,6 +614,8 @@ enum VideoDialect { DashScope, /// MiniMax Hailuo `/v1/video_generation` + `/v1/query/video_generation`. Minimax, + /// Kling `/v1/videos/text2video` + GET by task id; `{code, data}` envelope. + Kling, } fn video_dialect(provider: &str) -> VideoDialect { @@ -622,6 +624,7 @@ fn video_dialect(provider: &str) -> VideoDialect { "siliconflow" => VideoDialect::SiliconFlow, "alibaba" | "dashscope" => VideoDialect::DashScope, "minimax" => VideoDialect::Minimax, + "kling" => VideoDialect::Kling, _ => VideoDialect::Generations, } } @@ -629,10 +632,11 @@ fn video_dialect(provider: &str) -> VideoDialect { /// The vendor's async handle in a submit reply, whichever dialect answered; /// `None` for a synchronous reply that already carries the video. pub fn video_handle(v: &Value) -> Option<&str> { - // task ids before `request_id`: DashScope's reply carries both, and its - // top-level request_id is the HTTP call's trace id, not the job + // task ids before `request_id`: DashScope and Kling replies carry both, and + // their top-level request_id is the HTTP call's trace id, not the job v["output"]["task_id"] .as_str() + .or_else(|| v["data"]["task_id"].as_str()) .or_else(|| v["requestId"].as_str()) .or_else(|| v["task_id"].as_str().filter(|id| !id.is_empty())) .or_else(|| v["request_id"].as_str()) @@ -669,7 +673,17 @@ impl ModelEngine for VideoEngine { let dialect = video_dialect(self.base.provider()); let model = self.base.model_name()?; let mut body = Map::new(); - body.insert("model".into(), model.into()); + // Kling names the field model_name and takes no inline image on this path + if dialect == VideoDialect::Kling { + if p.image.is_some() { + return Err(GatewayError::bad_request( + "kling image-to-video is not wired; send a text prompt", + )); + } + body.insert("model_name".into(), model.into()); + } else { + body.insert("model".into(), model.into()); + } let (path, fields) = match dialect { VideoDialect::Sora => ( "videos", @@ -723,21 +737,35 @@ impl ModelEngine for VideoEngine { ("first_frame_image", p.image), ], ), + VideoDialect::Kling => ( + "videos/text2video", + vec![ + ("duration", p.duration_seconds.map(|d| d.to_string().into())), + ("aspect_ratio", p.aspect_ratio.map(Value::from)), + // Kling has quality modes, not resolutions: 1080p rides mode=pro + ( + "mode", + p.resolution.map(|r| { + match r.as_str() { + "1080p" | "pro" => "pro", + _ => "std", + } + .into() + }), + ), + ], + ), VideoDialect::Generations => ( "videos/generations", vec![ ("duration", p.duration_seconds.map(Value::from)), ("resolution", p.resolution.map(Value::from)), + ("aspect_ratio", p.aspect_ratio.map(Value::from)), ("image", p.image), ], ), }; body.insert("prompt".into(), p.prompt.into()); - if dialect == VideoDialect::Generations - && let Some(ar) = p.aspect_ratio - { - body.insert("aspect_ratio".into(), ar.into()); - } for (k, v) in fields { if let Some(v) = v { body.insert(k.into(), v); @@ -747,13 +775,31 @@ impl ModelEngine for VideoEngine { .base .round_trip(&self.base.vendor_url(path), body.into()) .await?; - if dialect == VideoDialect::Minimax { - reject_minimax_error(&v)?; - } + reject_envelope_error(dialect, &v)?; Ok(video_outcome(model, v, status)) } } +/// Vendors whose business errors ride an HTTP 200: MiniMax's `base_resp`, +/// Kling's `{code, message}` envelope. +fn reject_envelope_error(dialect: VideoDialect, v: &Value) -> GResult<()> { + match dialect { + VideoDialect::Minimax => reject_minimax_error(v), + VideoDialect::Kling => { + let code = v["code"].as_i64().unwrap_or(0); + if code == 0 { + return Ok(()); + } + Err(GatewayError::new( + gw_consts::ErrCode::FED_RESP_STATUS_NOT_ZERO, + 502, + format!("kling code {code}: {}", v["message"]), + )) + } + _ => Ok(()), + } +} + fn video_outcome(model: &str, v: Value, status: u16) -> EngineOutcome { let message = v["video"]["url"] .as_str() @@ -792,6 +838,11 @@ pub async fn video_poll( versioned_url(base, &format!("query/video_generation?task_id={id}")), Vec::new(), ), + VideoDialect::Kling => ( + "GET", + versioned_url(base, &format!("videos/text2video/{id}")), + Vec::new(), + ), _ => ( "GET", versioned_url(base, &format!("videos/{id}")), @@ -799,9 +850,7 @@ pub async fn video_poll( ), }; let (status, body) = video_send(transport, account, method, url, req_body).await?; - if dialect == VideoDialect::Minimax { - reject_minimax_error(&body)?; - } + reject_envelope_error(dialect, &body)?; Ok(normalize_video_poll(dialect, status, body)) } @@ -828,6 +877,15 @@ fn normalize_video_poll(dialect: VideoDialect, status: u16, body: Value) -> Vide !body["file_id"].as_str().unwrap_or_default().is_empty() as i64, None, ), + VideoDialect::Kling => { + let videos = &body["data"]["task_result"]["videos"]; + ( + body["data"]["task_status"] == "succeed", + whole_seconds(&videos[0]["duration"]) + .unwrap_or_else(|| videos.as_array().map_or(0, Vec::len) as i64), + None, + ) + } VideoDialect::Generations => ( body["status"] == "done", whole_seconds(&body["video"]["duration"]).unwrap_or(0), diff --git a/crates/engines/src/transport.rs b/crates/engines/src/transport.rs index 3425d19..7a650d3 100644 --- a/crates/engines/src/transport.rs +++ b/crates/engines/src/transport.rs @@ -774,6 +774,27 @@ impl MockTransport { "seconds": body["seconds"], "size": body["size"] })); } + if req.url.contains("/videos/text2video/") { + let id = req.url.rsplit('/').next().unwrap_or_default(); + return Self::ok_json(if id.contains("pending") { + json!({"code": 0, "request_id": "kling-trace", + "data": {"task_id": id, "task_status": "processing"}}) + } else { + json!({"code": 0, "request_id": "kling-trace", + "data": {"task_id": id, "task_status": "succeed", + "task_result": {"videos": [{"id": "v1", "duration": "5", + "url": format!("mock://videos/{id}.mp4")}]}}}) + }); + } + if req.url.contains("/videos/text2video") { + let body = Self::parse(&req.body, "video")?; + if body["model_name"].is_null() { + return Self::ok_json(json!({"code": 1200, "message": "model_name is required"})); + } + return Self::ok_json(json!({"code": 0, "request_id": "kling-trace", + "data": {"task_id": format!("kl-{}", slug(&body)), + "task_status": "submitted"}})); + } if req.url.contains("/video/submit") { let body = Self::parse(&req.body, "video")?; return Self::ok_json(json!({"requestId": format!("sf-{}", slug(&body))})); diff --git a/crates/engines/tests/request_construction.rs b/crates/engines/tests/request_construction.rs index b80a977..51d536e 100644 --- a/crates/engines/tests/request_construction.rs +++ b/crates/engines/tests/request_construction.rs @@ -855,7 +855,7 @@ async fn video_request_shape() { ); let req = typed_req( Protocol::Video, - "kling-video", + "vidu-video", TypedParams::Video(VideoParams { prompt: "a dog surfing".into(), duration_seconds: Some(5), @@ -866,7 +866,7 @@ async fn video_request_shape() { ); let _ = VideoEngine::new(req, t.clone()).run().await.unwrap(); let b = t.body_json(); - assert_eq!(b["model"], "kling-video"); + assert_eq!(b["model"], "vidu-video"); assert_eq!(b["prompt"], "a dog surfing"); assert_eq!(b["duration"], 5); assert_eq!(b["resolution"], "1080p"); @@ -957,6 +957,57 @@ async fn video_dialects_shape_the_submit_by_provider() { "url: {}", t.url() ); + + let t = RecordingTransport::new( + r#"{"code":0,"request_id":"trace","data":{"task_id":"kl-1","task_status":"submitted"}}"#, + ); + let mut req = typed_req( + Protocol::Video, + "kling-v1-6", + TypedParams::Video(VideoParams { + prompt: "a dog surfing".into(), + duration_seconds: Some(5), + aspect_ratio: Some("16:9".into()), + ..Default::default() + }), + ); + req.account = Some(std::sync::Arc::new(Account { + name: "kling-1".into(), + provider: "kling".into(), + endpoint: "https://api-singapore.klingai.com".into(), + ..Default::default() + })); + let _ = VideoEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + assert_eq!( + b["model_name"], "kling-v1-6", + "kling field is model_name: {b}" + ); + assert!(b.get("model").is_none(), "{b}"); + assert_eq!(b["duration"], "5", "kling duration is a string: {b}"); + assert_eq!(b["aspect_ratio"], "16:9"); + assert!( + t.url().ends_with("/v1/videos/text2video"), + "url: {}", + t.url() + ); + + let mut req = typed_req( + Protocol::Video, + "kling-v1-6", + TypedParams::Video(VideoParams { + prompt: "a dog surfing".into(), + image: Some(serde_json::json!("https://img/dog.png")), + ..Default::default() + }), + ); + req.account = Some(std::sync::Arc::new(Account { + name: "kling-1".into(), + provider: "kling".into(), + ..Default::default() + })); + let err = VideoEngine::new(req, t).run().await.unwrap_err(); + assert_eq!(err.http_status, 400, "kling image-to-video is not wired"); } #[tokio::test] diff --git a/crates/server/tests/e2e.rs b/crates/server/tests/e2e.rs index 9191420..246f45f 100644 --- a/crates/server/tests/e2e.rs +++ b/crates/server/tests/e2e.rs @@ -1458,7 +1458,7 @@ async fn async_video_bills_once_on_the_first_done_poll() { .oneshot(post( "/v1/videos/generations", Some("ak-demo-123"), - r#"{"model":"kling-video","prompt":"a dog surfing"}"#, + r#"{"model":"vidu-video","prompt":"a dog surfing"}"#, )) .await .unwrap(); @@ -1611,6 +1611,13 @@ async fn dashscope_and_hailuo_dialects_bill_once_and_hailuo_serves_content() { 600_000, false, ), + ( + r#"{"model":"kling-v1-6","prompt":"a lantern floating upriver","duration":5}"#, + "succeed", + 5, + 300_000, + false, + ), ( r#"{"model":"MiniMax-Hailuo-02","prompt":"a lantern floating upriver","duration":6}"#, "Success", @@ -1626,11 +1633,7 @@ async fn dashscope_and_hailuo_dialects_bill_once_and_hailuo_serves_content() { .unwrap(); assert_eq!(resp.status(), StatusCode::OK); let j = body_json(resp).await; - let id = j["output"]["task_id"] - .as_str() - .or_else(|| j["task_id"].as_str()) - .unwrap() - .to_owned(); + let id = gw_engines::families::video_handle(&j).unwrap().to_owned(); assert!( id.contains("lantern"), "job keyed by the task id, not a trace id: {j}" @@ -1646,6 +1649,7 @@ async fn dashscope_and_hailuo_dialects_bill_once_and_hailuo_serves_content() { assert_eq!(status, StatusCode::OK, "{poll}"); let state = poll["output"]["task_status"] .as_str() + .or_else(|| poll["data"]["task_status"].as_str()) .or_else(|| poll["status"].as_str()) .unwrap(); assert_eq!(state, done, "{poll}"); diff --git a/docs/api.md b/docs/api.md index 6151bf4..e9ffbe4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -155,9 +155,10 @@ the serving account's provider name: `openai`/`azure` speak Sora's `/v1/videos` `GET /v1/videos/{id}/content`), `siliconflow` Wan's `video/submit` + `video/status`, `alibaba`/`dashscope` the DashScope task API (async header, `output.task_id`, poll `/api/v1/tasks/{id}`), `minimax` Hailuo's -`video_generation` + `query/video_generation` + file content, and anything else the -xAI/Kling-style `videos/generations` shape. Each dialect forwards only the -fields its vendor takes (`aspect_ratio` only on the generic shape; Wan takes no +`video_generation` + `query/video_generation` + file content, `kling` Kling's +`videos/text2video` (`model_name`, string durations, a `{code, data}` envelope), +and anything else the generic `videos/generations` shape. Each dialect forwards +only the fields its vendor takes (`resolution` never reaches Kling; Wan takes no duration). When the reply is an async handle, the gateway remembers which key, model and diff --git a/docs/providers.md b/docs/providers.md index f05a1b0..9ff1369 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -81,15 +81,15 @@ usage); the rest are marked non-streaming below and always answer buffered: | `minimax-v1` | MiniMax legacy v1 (`abab*`) | `https://api.minimax.chat` | `Bearer` (non-streaming); kept for existing accounts — the vendor has retired it for new ones; new integrations should use MiniMax's OpenAI-/Anthropic-compatible endpoints | The factory also dispatches `video`, `search`, generic `audio`, and -`passthrough` protocols (kling-video, grok-imagine-video, sora-2 and +`passthrough` protocols (kling-v1-6, grok-imagine-video, sora-2 and brave-search ship example accounts in the default config). `protocol: video` picks its wire from the account's provider name — `openai`/`azure` → Sora, `siliconflow` → Wan submit/status, `alibaba`/`dashscope` → the DashScope task API (the account endpoint is the bare `https://dashscope-intl.aliyuncs.com`; some Wan models reject submit `parameters`, so the gateway forwards only what -the caller sets), `minimax` → Hailuo, anything else the xAI/Kling-style -`videos/generations` shape. Live-verified: sora-2 (4 s billed from the -`seconds` string, content download through the gateway), Wan2.2-T2V-A14B on +the caller sets), `minimax` → Hailuo, `kling` → Kling text2video, and +anything else the generic `videos/generations` shape. Live-verified: sora-2 +(4 s billed from the `seconds` string, content download through the gateway), Wan2.2-T2V-A14B on SiliconFlow and MiniMax-Hailuo-02 (one unit per delivered video, file content proxied through the gateway), and wan2.2-t2v-plus on DashScope intl (5 s from `usage.video_duration`). diff --git a/scripts/live-matrix/live.yaml b/scripts/live-matrix/live.yaml index e4a2dd4..9646b19 100644 --- a/scripts/live-matrix/live.yaml +++ b/scripts/live-matrix/live.yaml @@ -81,6 +81,7 @@ models: - {name: Wan-AI/Wan2.2-T2V-A14B, provider: siliconflow, protocol: video, unit_price_micros: 500000} - {name: wan2.2-t2v-plus, provider: dashscope, protocol: video, unit_price_micros: 120000} - {name: MiniMax-Hailuo-02, provider: minimax, protocol: video, unit_price_micros: 300000} + - {name: kling-v1-6, provider: kling, protocol: video, unit_price_micros: 60000} # realtime with real vendor usage (audio output at its own weight) - {name: gpt-realtime-mini, provider: openai, protocol: realtime, input_price_per_1k_micros: 600, output_price_per_1k_micros: 2400, token_rate: {audio_prompt: 16.0, audio_completion: 8.0}} # local ollama through the generic openai-compatible path @@ -97,6 +98,11 @@ models: - {name: "us.meta.llama3-1-8b-instruct-v1:0", protocol: aws-llama, input_price_per_1k_micros: 220, output_price_per_1k_micros: 220} accounts: + - name: kling + provider: kling + endpoint: https://api-singapore.klingai.com + api_key_env: KLING_API_KEY + protocols: ["video"] - {name: cohere, provider: cohere, endpoint: https://api.cohere.com, api_key_env: COHERE_API_KEY, protocols: ["rerank"], cost_unit_price_micros: 1000} - {name: jina, provider: jina, endpoint: https://api.jina.ai, api_key_env: JINA_API_KEY, protocols: ["rerank"]} - {name: bedrock, provider: aws, endpoint: https://bedrock-runtime.us-east-1.amazonaws.com, api_key_env: AWS_BEARER_TOKEN_BEDROCK, protocols: ["aws-anthropic", "aws-converse", "aws-llama"], timeout_seconds: 120} diff --git a/scripts/live-matrix/live_matrix.py b/scripts/live-matrix/live_matrix.py index ee8ac6d..a10c160 100644 --- a/scripts/live-matrix/live_matrix.py +++ b/scripts/live-matrix/live_matrix.py @@ -372,6 +372,7 @@ def video_handle(j: dict[str, Any]) -> str | None: """The async handle, whichever dialect answered (mirrors the gateway's extraction).""" return ( (j.get("output") or {}).get("task_id") + or (j.get("data") or {}).get("task_id") or j.get("requestId") or j.get("task_id") or j.get("request_id") @@ -380,7 +381,20 @@ def video_handle(j: dict[str, Any]) -> str | None: def video_state(j: dict[str, Any]) -> str: - return str(j.get("status") or (j.get("output") or {}).get("task_status") or "") + return str( + j.get("status") + or (j.get("output") or {}).get("task_status") + or (j.get("data") or {}).get("task_status") + or "" + ) + + +def _video_duration(j: dict[str, Any]) -> float: + videos = ((j.get("data") or {}).get("task_result") or {}).get("videos") or [] + duration = (j.get("video") or {}).get("duration") + if duration is None and videos: + duration = videos[0].get("duration") or len(videos) + return float(duration or 0) def case_video( @@ -420,7 +434,7 @@ def case_video( gw.call(f"/v1/videos/{rid}") count, row = gw.ledger() ticks = (j.get("usage") or {}).get("cost_in_usd_ticks") - expected_units = units if units is not None else math.ceil((j.get("video") or {}).get("duration", 0)) + expected_units = units if units is not None else math.ceil(_video_duration(j)) expected_vendor = ticks // 10_000 if ticks else row["vendor_cost_micros"] ok = ( count == before + 2 @@ -956,6 +970,7 @@ def run_group(gw: Gateway, group: str) -> None: # parameters (size/duration) silently kill the task into UNKNOWN on intl — submit bare case_video(gw, "wan2.2-t2v-plus", {}, done="SUCCEEDED", units=5) case_video(gw, "MiniMax-Hailuo-02", {"duration": 6}, done="Success", units=1, content=True) + case_video(gw, "kling-v1-6", {"duration": 5, "aspect_ratio": "16:9"}, done="succeed") elif group == "openai-rt": case_realtime(gw, "gpt-realtime-mini") elif group == "ollama":