From 128faf145fbbccf1f8b95c5262829a69f1ee3ebb Mon Sep 17 00:00:00 2001 From: Kaiyi Date: Tue, 21 Jul 2026 14:28:41 +0800 Subject: [PATCH 01/45] feat: send prompt-attached videos directly with the prompt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Videos attached to a prompt (pasted in the TUI, uploaded in the web UI) previously reached the model only after it opened the file with ReadMediaFile — an extra tool round trip that could leave the video unseen when the model never made that call. They are now uploaded through the model provider's file channel and embedded directly in the user message as the provider-issued reference; ReadMediaFile stays as the fallback and as the model's own way to open video files. - agent-core: new uploadVideo agent RPC and Session.uploadVideo in the SDK; the TUI uploads pasted videos at submit time, falling back to the file-tag form on failure - kap-server: inline file-source video prompt parts at the REST edge and map provider file ids back to local uploads behind GET /files/llm/{llm_id} - kimi-web: play provider-referenced prompt videos after reload/resume --- .changeset/prompt-video-inline.md | 5 + .changeset/sdk-session-upload-video.md | 5 + .changeset/web-video-playback-after-reload.md | 5 + apps/kimi-code/src/tui/kimi-tui.ts | 64 +++++-- .../src/tui/utils/image-placeholder.ts | 153 ++++++++++++++++- .../test/tui/input/image-placeholder.test.ts | 109 ++++++++++++ .../test/tui/kimi-tui-message-flow.test.ts | 65 ++++++- apps/kimi-web/src/api/daemon/client.ts | 4 + apps/kimi-web/src/api/types.ts | 3 + .../src/composables/client/useSideChat.ts | 2 + .../src/composables/messagesToTurns.ts | 43 ++++- .../src/composables/useKimiWebClient.ts | 1 + apps/kimi-web/test/turn-logic.test.ts | 81 +++++++++ .../agent/contextMemory/protocolMessage.ts | 8 +- packages/agent-core/src/agent/index.ts | 46 ++++- packages/agent-core/src/agent/tool/index.ts | 10 ++ packages/agent-core/src/rpc/core-api.ts | 13 +- packages/agent-core/src/rpc/core-impl.ts | 5 + packages/agent-core/src/session/rpc.ts | 5 + .../src/tools/builtin/file/read-media.ts | 2 +- packages/agent-core/test/agent/basic.test.ts | 120 ++++++++++++- packages/kap-server/src/lib/llmVideoRefs.ts | 35 ++++ packages/kap-server/src/protocol/rest-file.ts | 6 + packages/kap-server/src/routes/files.ts | 37 ++++ packages/kap-server/src/routes/prompts.ts | 101 ++++++++++- .../apiSurface.snapshot.test.ts.snap | 4 + packages/kap-server/test/files.test.ts | 12 ++ packages/kap-server/test/prompts.test.ts | 162 +++++++++++++++++- packages/node-sdk/src/rpc.ts | 10 ++ packages/node-sdk/src/session.ts | 13 ++ packages/node-sdk/src/types.ts | 2 +- .../test/session-prompt-input.test.ts | 17 ++ 32 files changed, 1112 insertions(+), 36 deletions(-) create mode 100644 .changeset/prompt-video-inline.md create mode 100644 .changeset/sdk-session-upload-video.md create mode 100644 .changeset/web-video-playback-after-reload.md create mode 100644 packages/kap-server/src/lib/llmVideoRefs.ts diff --git a/.changeset/prompt-video-inline.md b/.changeset/prompt-video-inline.md new file mode 100644 index 0000000000..a24770dd6d --- /dev/null +++ b/.changeset/prompt-video-inline.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": minor +--- + +Send pasted videos to the model directly with the prompt instead of routing them through a ReadMediaFile tool call, so video input is faster and no longer depends on an extra tool round trip. diff --git a/.changeset/sdk-session-upload-video.md b/.changeset/sdk-session-upload-video.md new file mode 100644 index 0000000000..aafd048f2e --- /dev/null +++ b/.changeset/sdk-session-upload-video.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code-sdk": patch +--- + +Add Session.uploadVideo to upload a local video through the session's current model and embed the returned reference as a prompt part. diff --git a/.changeset/web-video-playback-after-reload.md b/.changeset/web-video-playback-after-reload.md new file mode 100644 index 0000000000..23bd3115d8 --- /dev/null +++ b/.changeset/web-video-playback-after-reload.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +web: Keep uploaded videos playable in the chat after reloading or resuming a session. diff --git a/apps/kimi-code/src/tui/kimi-tui.ts b/apps/kimi-code/src/tui/kimi-tui.ts index 9d03f9395b..62f7427992 100644 --- a/apps/kimi-code/src/tui/kimi-tui.ts +++ b/apps/kimi-code/src/tui/kimi-tui.ts @@ -133,7 +133,7 @@ import { isDeadTerminalError } from './utils/dead-terminal'; import { formatErrorMessage } from './utils/event-payload'; import { pickForegroundTasks } from './utils/foreground-task'; import { ImageAttachmentStore, type ImageAttachment } from './utils/image-attachment-store'; -import { extractMediaAttachments, rewriteMediaPlaceholders } from './utils/image-placeholder'; +import { extractMediaSegments, materializeMediaSegments, rewriteMediaPlaceholders, segmentsToPromptParts } from './utils/image-placeholder'; import { hasPatchChanges } from './utils/object-patch'; import { sessionRowsForPicker } from './utils/session-picker-rows'; import { formatBashOutputForDisplay } from './utils/shell-output'; @@ -1121,30 +1121,70 @@ export class KimiTUI { this.updateQueueDisplay(); } + // Submits chained behind an in-flight video upload (see sendNormalUserInput). + private pendingSubmits = 0; + private inputSubmitChain: Promise = Promise.resolve(); + sendNormalUserInput(text: string): void { if (this.btwPanelController.sendUserInput(text)) return; if (this.state.appState.model.trim().length === 0) { this.showError(LLM_NOT_SET_MESSAGE); return; } - const extraction = extractMediaAttachments(text, this.imageStore); + const extraction = extractMediaSegments(text, this.imageStore); if (!this.validateMediaCapabilities(extraction)) return; const session = this.session; if (session === undefined) { this.showError(LLM_NOT_SET_MESSAGE); return; } - if (extraction.hasMedia) { - this.sendMessage(session, text, { - hasMedia: true, - parts: extraction.parts, - imageAttachmentIds: extraction.imageAttachmentIds, - }); - } else { - this.sendMessage(session, text); + const hasVideo = extraction.videoAttachmentIds.length > 0; + if (!hasVideo && this.pendingSubmits === 0) { + // Fully synchronous path: nothing to upload, nothing queued behind an + // upload, so ordering is trivially safe. + if (extraction.hasMedia) { + this.sendMessage(session, text, { + hasMedia: true, + parts: segmentsToPromptParts(extraction.segments), + imageAttachmentIds: extraction.imageAttachmentIds, + }); + } else { + this.sendMessage(session, text); + } + this.updateQueueDisplay(); + this.state.ui.requestRender(); + return; } - this.updateQueueDisplay(); - this.state.ui.requestRender(); + // Video attachments upload asynchronously before the prompt goes out. + // Chain this submit (and any submitted while an upload is in flight) so + // a slow upload cannot be overtaken by a later message. + this.pendingSubmits += 1; + this.inputSubmitChain = this.inputSubmitChain.then(async () => { + try { + if (extraction.hasMedia) { + const parts = hasVideo + ? await materializeMediaSegments(extraction.segments, (attachment) => + session.uploadVideo(attachment.sourcePath), + ) + : segmentsToPromptParts(extraction.segments); + this.sendMessage(session, text, { + hasMedia: true, + parts, + imageAttachmentIds: extraction.imageAttachmentIds, + }); + } else { + this.sendMessage(session, text); + } + } catch (error) { + // The tag fallback itself could not be prepared (unwritable cache + // dir, vanished video source…); nothing was dispatched. + this.showError(`Failed to prepare media attachment: ${formatErrorMessage(error)}`); + } finally { + this.pendingSubmits -= 1; + } + this.updateQueueDisplay(); + this.state.ui.requestRender(); + }); } validateMediaCapabilities(extraction: { diff --git a/apps/kimi-code/src/tui/utils/image-placeholder.ts b/apps/kimi-code/src/tui/utils/image-placeholder.ts index 56edcee88b..7c15201f57 100644 --- a/apps/kimi-code/src/tui/utils/image-placeholder.ts +++ b/apps/kimi-code/src/tui/utils/image-placeholder.ts @@ -1,17 +1,30 @@ /** - * Scan submitted text for media placeholders and produce - * the `PromptPart[]` we'll send to the SDK prompt endpoint. + * Scan submitted text for media placeholders and produce the prompt content + * we'll send to the SDK prompt endpoint. * - * Rules: + * Two extraction paths share the same placeholder scan: + * + * - `extractMediaAttachments` (sync): image placeholders expand to image + * content parts; video placeholders are copied into the shared cache + * (`getCacheDir()`) and expand to `