From ff321da139a0bc81f81573492b38d973fc9e7a36 Mon Sep 17 00:00:00 2001 From: AIdoesmyjob <130925557+AIdoesmyjob@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:18:46 +0000 Subject: [PATCH] fix(chat): persist assistant image attachments --- .../Layers/ProviderRuntimeIngestion.test.ts | 92 ++++++++++++- .../Layers/ProviderRuntimeIngestion.ts | 123 ++++++++++++++++++ apps/server/src/orchestration/decider.ts | 1 + .../components/chat/MessagesTimeline.test.tsx | 40 ++++++ .../src/components/chat/MessagesTimeline.tsx | 87 ++++++++----- packages/contracts/src/orchestration.ts | 1 + 6 files changed, 310 insertions(+), 34 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts index 74ece50cd31..f3d71bc4327 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts @@ -220,6 +220,8 @@ describe("ProviderRuntimeIngestion", () => { async function createHarness(options?: { serverSettings?: Partial }) { const workspaceRoot = makeTempDir("t3-provider-project-"); + const serverBaseDir = makeTempDir("t3-provider-server-"); + const attachmentsDir = NodePath.join(serverBaseDir, "userdata", "attachments"); NodeFS.mkdirSync(NodePath.join(workspaceRoot, ".git")); const provider = createProviderServiceHarness(); const orchestrationLayer = OrchestrationEngineLive.pipe( @@ -240,7 +242,7 @@ describe("ProviderRuntimeIngestion", () => { Layer.provideMerge(SqlitePersistenceMemory), Layer.provideMerge(Layer.succeed(ProviderService, provider.service)), Layer.provideMerge(makeTestServerSettingsLayer(options?.serverSettings)), - Layer.provideMerge(ServerConfig.layerTest(process.cwd(), process.cwd())), + Layer.provideMerge(ServerConfig.layerTest(process.cwd(), serverBaseDir)), Layer.provideMerge(NodeServices.layer), ); runtime = ManagedRuntime.make(layer); @@ -316,6 +318,7 @@ describe("ProviderRuntimeIngestion", () => { emit: provider.emit, setProviderSession: provider.setSession, drain, + attachmentsDir, }; } @@ -361,6 +364,93 @@ describe("ProviderRuntimeIngestion", () => { expect(thread.session?.lastError).toBe("turn failed"); }); + it("persists completed image views as assistant message attachments", async () => { + const harness = await createHarness({ serverSettings: { enableAssistantStreaming: true } }); + const sourceDir = makeTempDir("t3-provider-assistant-image-"); + const sourcePath = NodePath.join(sourceDir, "filter-manager.png"); + NodeFS.writeFileSync(sourcePath, Buffer.from([0x89, 0x50, 0x4e, 0x47])); + const turnId = asTurnId("turn-assistant-image"); + + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-turn-started-assistant-image"), + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + createdAt: "2026-01-01T00:00:00.000Z", + turnId, + }); + harness.emit({ + type: "item.completed", + eventId: asEventId("evt-image-completed-assistant-image"), + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + createdAt: "2026-01-01T00:00:01.000Z", + turnId, + itemId: asItemId("item-image-assistant-image"), + payload: { + itemType: "image_view", + status: "completed", + detail: sourcePath, + data: { + item: { + type: "imageView", + path: sourcePath, + }, + }, + }, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-message-delta-assistant-image"), + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + createdAt: "2026-01-01T00:00:02.000Z", + turnId, + itemId: asItemId("item-message-assistant-image"), + payload: { + streamKind: "assistant_text", + delta: "Screenshot attached.", + }, + }); + harness.emit({ + type: "item.completed", + eventId: asEventId("evt-message-completed-assistant-image"), + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + createdAt: "2026-01-01T00:00:03.000Z", + turnId, + itemId: asItemId("item-message-assistant-image"), + payload: { + itemType: "assistant_message", + status: "completed", + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.messages.some( + (message) => + message.role === "assistant" && + message.text === "Screenshot attached." && + message.attachments?.length === 1 && + message.attachments[0]?.name === "filter-manager.png", + ), + ); + const message = thread.messages.find( + (entry) => entry.role === "assistant" && entry.text === "Screenshot attached.", + ); + const attachment = message?.attachments?.[0]; + expect(attachment).toMatchObject({ + type: "image", + name: "filter-manager.png", + mimeType: "image/png", + sizeBytes: 4, + }); + expect(attachment).toBeDefined(); + expect( + NodeFS.readFileSync(NodePath.join(harness.attachmentsDir, `${attachment!.id}.png`)), + ).toEqual(Buffer.from([0x89, 0x50, 0x4e, 0x47])); + }); + it("applies provider session.state.changed transitions directly", async () => { const harness = await createHarness(); const waitingAt = "2026-01-01T00:00:00.000Z"; diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index a8a51b30260..ab49a62bfc0 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -1,5 +1,6 @@ import { ApprovalRequestId, + type ChatAttachment, type AssistantDeliveryMode, CommandId, MessageId, @@ -16,17 +17,23 @@ import { type OrchestrationThread, type OrchestrationThreadActivity, type ProviderRuntimeEvent, + PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, } from "@t3tools/contracts"; import * as Cache from "effect/Cache"; import * as Cause from "effect/Cause"; import * as Crypto from "effect/Crypto"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Stream from "effect/Stream"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; +import { createAttachmentId, resolveAttachmentPath } from "../../attachmentStore.ts"; +import { ServerConfig } from "../../config.ts"; +import { IMAGE_EXTENSION_BY_MIME_TYPE, SAFE_IMAGE_FILE_EXTENSIONS } from "../../imageMime.ts"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; import { ProjectionTurnRepository } from "../../persistence/Services/ProjectionTurns.ts"; import { ProjectionTurnRepositoryLive } from "../../persistence/Layers/ProjectionTurns.ts"; @@ -124,6 +131,32 @@ function sameId(left: string | null | undefined, right: string | null | undefine return left === right; } +const IMAGE_MIME_TYPE_BY_EXTENSION = new Map(); +for (const [mimeType, extension] of Object.entries(IMAGE_EXTENSION_BY_MIME_TYPE)) { + if (!IMAGE_MIME_TYPE_BY_EXTENSION.has(extension)) { + IMAGE_MIME_TYPE_BY_EXTENSION.set(extension, mimeType); + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function resolveImageViewPath(event: ProviderRuntimeEvent): string | undefined { + if (event.type !== "item.completed" || event.payload.itemType !== "image_view") { + return undefined; + } + const data = isRecord(event.payload.data) ? event.payload.data : undefined; + const item = data && isRecord(data.item) ? data.item : undefined; + const candidates = [item?.path, data?.path, event.payload.detail]; + for (const candidate of candidates) { + if (typeof candidate !== "string") continue; + const trimmed = candidate.trim(); + if (trimmed.length > 0) return trimmed; + } + return undefined; +} + function hasAssistantMessageForTurn( messages: ReadonlyArray, turnId: TurnId, @@ -688,10 +721,13 @@ export function runtimeEventToActivities( const make = Effect.gen(function* () { const crypto = yield* Crypto.Crypto; + const fileSystem = yield* FileSystem.FileSystem; const orchestrationEngine = yield* OrchestrationEngineService; + const path = yield* Path.Path; const projectionSnapshotQuery = yield* ProjectionSnapshotQuery; const providerService = yield* ProviderService; const projectionTurnRepository = yield* ProjectionTurnRepository; + const serverConfig = yield* ServerConfig; const serverSettingsService = yield* ServerSettingsService; const providerCommandId = (event: ProviderRuntimeEvent, tag: string) => crypto.randomUUIDv4.pipe( @@ -938,6 +974,57 @@ const make = Effect.gen(function* () { const clearAssistantMessageState = (messageId: MessageId) => clearBufferedAssistantText(messageId); + const persistAssistantImageAttachment = Effect.fn("persistAssistantImageAttachment")(function* ( + event: ProviderRuntimeEvent, + threadId: ThreadId, + ) { + const sourcePath = resolveImageViewPath(event); + if (!sourcePath || !path.isAbsolute(sourcePath)) { + return undefined; + } + + const extension = path.extname(sourcePath).toLowerCase(); + if (!SAFE_IMAGE_FILE_EXTENSIONS.has(extension)) { + return undefined; + } + + const fileInfo = yield* fileSystem.stat(sourcePath).pipe(Effect.orElseSucceed(() => null)); + const sizeBytes = fileInfo ? Number(fileInfo.size) : 0; + if ( + !fileInfo || + fileInfo.type !== "File" || + !Number.isSafeInteger(sizeBytes) || + sizeBytes <= 0 || + sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + ) { + return undefined; + } + + const mimeType = IMAGE_MIME_TYPE_BY_EXTENSION.get(extension); + const attachmentId = createAttachmentId(threadId); + if (!mimeType || !attachmentId) { + return undefined; + } + + const attachment = { + type: "image" as const, + id: attachmentId, + name: path.basename(sourcePath), + mimeType, + sizeBytes, + } satisfies ChatAttachment; + const destinationPath = resolveAttachmentPath({ + attachmentsDir: serverConfig.attachmentsDir, + attachment, + }); + if (!destinationPath) { + return undefined; + } + + yield* fileSystem.copyFile(sourcePath, destinationPath); + return attachment; + }); + const flushBufferedAssistantMessage = (input: { event: ProviderRuntimeEvent; threadId: ThreadId; @@ -1501,6 +1588,42 @@ const make = Effect.gen(function* () { } } + if (event.type === "item.completed" && event.payload.itemType === "image_view") { + const imageAttachment = yield* persistAssistantImageAttachment(event, thread.id).pipe( + Effect.catchCause((cause) => + Effect.logWarning("provider runtime ingestion failed to persist assistant image", { + eventId: event.eventId, + threadId: thread.id, + cause: Cause.pretty(cause), + }).pipe(Effect.as(undefined)), + ), + ); + if (imageAttachment) { + const turnId = toTurnId(event.turnId); + const assistantMessageId = yield* getOrCreateAssistantMessageId({ + threadId: thread.id, + event, + ...(turnId ? { turnId } : {}), + }); + if (turnId) { + yield* rememberAssistantMessageId(thread.id, turnId, assistantMessageId); + } + const detailedThread = yield* getLoadedThreadDetail(); + const existingAttachments = + findMessageById(detailedThread?.messages ?? [], assistantMessageId)?.attachments ?? []; + yield* orchestrationEngine.dispatch({ + type: "thread.message.assistant.delta", + commandId: yield* providerCommandId(event, "assistant-image-attachment"), + threadId: thread.id, + messageId: assistantMessageId, + delta: "", + attachments: [...existingAttachments, imageAttachment], + ...(turnId ? { turnId } : {}), + createdAt: now, + }); + } + } + const pauseForUserTurnId = event.type === "request.opened" || event.type === "user-input.requested" ? toTurnId(event.turnId) diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index 100369ae6e3..06f3dc31d66 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -1010,6 +1010,7 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" messageId: command.messageId, role: "assistant", text: command.delta, + ...(command.attachments !== undefined ? { attachments: command.attachments } : {}), turnId: command.turnId ?? null, streaming: true, createdAt: command.createdAt, diff --git a/apps/web/src/components/chat/MessagesTimeline.test.tsx b/apps/web/src/components/chat/MessagesTimeline.test.tsx index 83ca7d3e952..40f66f458cf 100644 --- a/apps/web/src/components/chat/MessagesTimeline.test.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.test.tsx @@ -296,6 +296,46 @@ describe("MessagesTimeline", () => { expect(markup).toContain("1 changed file"); }); + it("renders assistant image attachments inline", () => { + const turnId = TurnId.make("turn-with-image"); + const markup = renderToStaticMarkup( + , + ); + + expect(markup).toContain('aria-label="Preview filter-manager.png"'); + expect(markup).toContain('alt="filter-manager.png"'); + expect(markup).toContain("https://t3.example.test/api/assets/filter-manager.png"); + expect(markup).toContain("Screenshot attached."); + }); + it("uses LegendList isNearEnd when deciding whether the live edge is visible", async () => { const { resolveTimelineIsAtEnd, diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index a429b54deaf..b700dd0cb1d 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -866,6 +866,49 @@ const TimelineRowContent = memo(function TimelineRowContent({ row }: { row: Time ); }); +function MessageImageGrid({ + images, + className, + imageClassName, +}: { + images: ReadonlyArray[number]>; + className: string; + imageClassName: string; +}) { + const ctx = use(TimelineRowCtx); + if (images.length === 0) return null; + + return ( +
+ {images.map((image) => ( +
+ {image.previewUrl ? ( + + ) : ( +
+ {image.name} +
+ )} +
+ ))} +
+ ); +} + function UserTimelineRow({ row }: { row: Extract }) { const ctx = use(TimelineRowCtx); const userImages = row.message.attachments ?? []; @@ -891,39 +934,11 @@ function UserTimelineRow({ row }: { row: Extract
- {regularImages.length > 0 && ( -
- {regularImages.map((image: NonNullable[number]) => ( -
- {image.previewUrl ? ( - - ) : ( -
- {image.name} -
- )} -
- ))} -
- )} + {previewAnnotations.map((annotation, index) => ( }) { const ctx = use(TimelineRowCtx); const messageText = row.message.text || (row.message.streaming ? "" : "(empty response)"); + const assistantImages = row.message.attachments ?? []; return ( <>
+