diff --git a/.env.example b/.env.example index 445933a67..a7e741cb1 100644 --- a/.env.example +++ b/.env.example @@ -16,3 +16,11 @@ # `OBSERVABILITY_ENABLED=1` uses the same unified switch as the backend. When # enabled by the LexVoice runtime, browser-side probes publish LiveKit data # packets for the local observability report. + +# AgentWidget host delivery is authenticated server-to-server. Use the same +# high-entropy channel id in the host tool and the browser's +# `?agentwidgetChannel=...` query; never expose this token to the browser. +# AGENTWIDGET_HOST_TOKEN= +# AGENTWIDGET_COMPOSER_API_KEY= +# AGENTWIDGET_COMPOSER_BASE_URL= +# AGENTWIDGET_COMPOSER_MODEL_ID= diff --git a/README.md b/README.md index 54ebe2f90..b8bb7a835 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,11 @@ Also available for: ### Features: - Real-time voice interaction with LiveKit Agents +- AgentWidget surfaces delivered from Codex, MCP, or another host agent - Camera video streaming support - Screen sharing capabilities - Audio visualization and level monitoring - Virtual avatar integration -- Light/dark theme switching with system preference detection - Customizable branding, colors, and UI text via configuration This template is built with Next.js and is free for you to use or modify as you see fit. @@ -91,6 +91,29 @@ And open http://localhost:3000 in your browser. You'll also need a LiveKit server and an agent worker. In integrated workspaces, those are normally provided by the LexVoice project. +### AgentWidget AI frontdesk + +The main page composes AgentWidget presentation over the existing +`SessionProvider` and its one LiveKit room. It does not create a second LexVoice +connection. A host tool publishes an established result to: + +```text +POST /api/agentwidget/spawn +Authorization: Bearer $AGENTWIDGET_HOST_TOKEN +x-agentwidget-channel-id: +``` + +Open the UI with the same channel as +`?agentwidgetChannel=`. The browser subscribes to the +surface stream and renders the SDK Catalog; it never receives the host token. +Set `AGENTWIDGET_HOST_TOKEN` on the Next.js server. Unknown result shapes also +require the server-side `AGENTWIDGET_COMPOSER_*` variables documented by the +AgentWidget SDK. + +Surface delivery is currently process-local, like the session lifecycle API. +Deploy `/api/agentwidget/*` on one Next.js instance or with sticky routing until +the channel hub is replaced by shared pub/sub. + ## Configuration This starter is designed to be flexible so you can adapt it to your specific agent use case. You can easily configure it to work with different types of inputs and outputs: diff --git a/app-config.ts b/app-config.ts index 79d6b54c8..9a0f1e2df 100644 --- a/app-config.ts +++ b/app-config.ts @@ -135,7 +135,7 @@ export function resolveInputDeviceConfig({ usesServerRoomInput, supportsScreenShare: usesBrowserRawVideoInput ? false : APP_CONFIG_DEFAULTS.supportsScreenShare, showDefaultCameraPreview: usesBrowserRawVideoInput - ? false + ? !usesServerRoomInput : (APP_CONFIG_DEFAULTS.showDefaultCameraPreview ?? true), }; } @@ -193,8 +193,8 @@ export function buildDefaultVideoTracks( ]; } -export function getDefaultVideoTrack(): string { - return ROOM_INPUT_VIDEO_TRACK_NAME; +export function getDefaultVideoTrack(isBrowserInput = false): string { + return isBrowserInput ? BROWSER_VIDEO_TRACK_NAME : ROOM_INPUT_VIDEO_TRACK_NAME; } export const APP_CONFIG_DEFAULTS: AppConfig = { diff --git a/app/(app)/layout.tsx b/app/(app)/layout.tsx index aaeb0376d..bbe08fbfb 100644 --- a/app/(app)/layout.tsx +++ b/app/(app)/layout.tsx @@ -1,46 +1,7 @@ -import { headers } from 'next/headers'; -import { getAppConfig } from '@/lib/utils'; - interface LayoutProps { children: React.ReactNode; } -export default async function Layout({ children }: LayoutProps) { - const hdrs = await headers(); - const { companyName, logo, logoDark } = await getAppConfig(hdrs); - - return ( - <> -
- - {/* eslint-disable-next-line @next/next/no-img-element */} - {`${companyName} - {/* eslint-disable-next-line @next/next/no-img-element */} - {`${companyName} - - - Built with{' '} - - Lexmount Agent Studio - - -
- - {children} - - ); +export default function Layout({ children }: LayoutProps) { + return children; } diff --git a/app/api/agentwidget/spawn/route.js b/app/api/agentwidget/spawn/route.js new file mode 100644 index 000000000..599b86431 --- /dev/null +++ b/app/api/agentwidget/spawn/route.js @@ -0,0 +1,72 @@ +import { timingSafeEqual } from 'node:crypto'; +import { + AgentWidgetComposerError, + createOpenAICompatibleWidgetComposerFromEnv, + createSpawnWidgetResult, +} from '@lexmount/agentwidget-sdk/host'; +import { + getAgentWidgetSurfaceChannel, + parseAgentWidgetChannelId, +} from '@/lib/agentwidget/surface-channel'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +const CHANNEL_HEADER = 'x-agentwidget-channel-id'; +const MAX_BODY_BYTES = 16 * 1024; +let composer; + +function json(payload, status = 200) { + return Response.json(payload, { + status, + headers: { 'cache-control': 'no-store' }, + }); +} + +function hasValidBearer(request, expectedToken) { + if (!expectedToken) return true; + const supplied = request.headers.get('authorization')?.replace(/^Bearer\s+/i, '') ?? ''; + const expected = Buffer.from(expectedToken); + const actual = Buffer.from(supplied); + return expected.length === actual.length && timingSafeEqual(expected, actual); +} + +export async function POST(request) { + const channelHeader = request.headers.get(CHANNEL_HEADER); + const token = process.env.AGENTWIDGET_HOST_TOKEN; + if (channelHeader && !token) return json({ error: 'HOST_TOKEN_NOT_CONFIGURED' }, 503); + if (!hasValidBearer(request, token)) return json({ error: 'UNAUTHORIZED' }, 401); + + try { + if (request.headers.get('content-type')?.split(';', 1)[0] !== 'application/json') { + return json({ error: 'UNSUPPORTED_MEDIA_TYPE' }, 415); + } + const text = await request.text(); + if (!text || Buffer.byteLength(text, 'utf8') > MAX_BODY_BYTES) { + return json({ error: 'INVALID_REQUEST' }, 400); + } + const input = JSON.parse(text); + const result = await createSpawnWidgetResult(input, { + getComposer: () => { + composer ??= createOpenAICompatibleWidgetComposerFromEnv(process.env); + return composer; + }, + }); + if (channelHeader) { + getAgentWidgetSurfaceChannel().publish( + parseAgentWidgetChannelId(channelHeader), + result.structuredContent + ); + } + return json({ structuredContent: result.structuredContent }); + } catch (error) { + if (error instanceof AgentWidgetComposerError) { + return json({ error: error.code }, error.code === 'MODEL_NOT_CONFIGURED' ? 503 : 502); + } + if (error instanceof SyntaxError || error instanceof TypeError || error?.name === 'ZodError') { + return json({ error: 'INVALID_REQUEST' }, 400); + } + console.error('[agentwidget] spawn failed', error); + return json({ error: 'COMPOSER_FAILED' }, 500); + } +} diff --git a/app/api/agentwidget/surfaces/route.js b/app/api/agentwidget/surfaces/route.js new file mode 100644 index 000000000..454dc9a1c --- /dev/null +++ b/app/api/agentwidget/surfaces/route.js @@ -0,0 +1,53 @@ +import { + getAgentWidgetSurfaceChannel, + parseAgentWidgetChannelId, +} from '@/lib/agentwidget/surface-channel'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +const encoder = new TextEncoder(); + +export function GET(request) { + let channelId; + try { + channelId = parseAgentWidgetChannelId(new URL(request.url).searchParams.get('channel')); + } catch { + return Response.json({ error: 'INVALID_CHANNEL' }, { status: 400 }); + } + + let close = () => undefined; + const stream = new ReadableStream({ + start(controller) { + let closed = false; + const send = (value) => { + if (!closed) controller.enqueue(encoder.encode(value)); + }; + send('retry: 1000\n\n'); + const unsubscribe = getAgentWidgetSurfaceChannel().subscribe(channelId, (envelope) => { + send(`event: surface\ndata: ${JSON.stringify(envelope)}\n\n`); + }); + const heartbeat = setInterval(() => send(': keep-alive\n\n'), 15_000); + close = () => { + if (closed) return; + closed = true; + clearInterval(heartbeat); + unsubscribe(); + controller.close(); + }; + request.signal.addEventListener('abort', close, { once: true }); + }, + cancel() { + close(); + }, + }); + + return new Response(stream, { + headers: { + 'cache-control': 'no-cache, no-transform', + connection: 'keep-alive', + 'content-type': 'text/event-stream; charset=utf-8', + 'x-accel-buffering': 'no', + }, + }); +} diff --git a/app/layout.tsx b/app/layout.tsx index 6c1935ae5..0b4c38ada 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -1,7 +1,9 @@ import type { Metadata } from 'next'; import { headers } from 'next/headers'; -import { ApplyThemeScript, ThemeToggle } from '@/components/app/theme-toggle'; +import '@lexmount/agentwidget-sdk/styles.css'; +import { ApplyThemeScript } from '@/components/app/theme-toggle'; import { cn, getAppConfig, getStyles } from '@/lib/utils'; +import '@/styles/agentwidget-frontdesk.css'; import '@/styles/globals.css'; const metadataBaseUrl = @@ -30,12 +32,7 @@ export default async function RootLayout({ children }: RootLayoutProps) { - - {children} -
- -
- + {children} ); } diff --git a/components/agentwidget/ai-frontdesk.jsx b/components/agentwidget/ai-frontdesk.jsx new file mode 100644 index 000000000..867c8083b --- /dev/null +++ b/components/agentwidget/ai-frontdesk.jsx @@ -0,0 +1,314 @@ +'use client'; + +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { RoomEvent } from 'livekit-client'; +import { AnimatePresence, motion } from 'motion/react'; +import { createLexVoiceAdapter } from '@lexmount/agentwidget-sdk/adapter-lexvoice'; +import { createAgentWidgetResourceRegistry } from '@lexmount/agentwidget-sdk/core'; +import { + AgentWidgetAmbient, + AgentWidgetDock, + CompactSurface, + DOCK_APPS, + RECIPE_BY_ID, + createCanvasState, + createDefaultCatalogModels, + deriveCanvasScene, + recipeForWidgetType, + removeCanvasSurface, + upsertCanvasSurface, +} from '@lexmount/agentwidget-sdk/react'; +import { + createAgentWidgetSurfaceChannelClient, + hasConfiguredAgentWidgetSurfaceChannel, +} from '@lexmount/agentwidget-sdk/surface-client'; +import { useChat, useRoomContext, useVoiceAssistant } from '@livekit/components-react'; +import { useSession } from '@/components/app/session-provider'; +import { TileLayout } from '@/components/app/tile-layout'; +import { useChatMessages } from '@/hooks/useChatMessages'; +import { + getActiveAgentSession, + registerAgentSessionLocalCleanup, + requestAgentSessionStop, +} from '@/lib/session-stop-client'; + +function connectionLabel(connection) { + return ( + { + idle: '点击呼吸球开始', + connecting: '正在连接', + reconnecting: '正在重连', + connected: '已连接', + }[connection] ?? 'AI 前台' + ); +} + +export function AiFrontdesk({ onStartCall, startDisabled = false, startPending = false }) { + const room = useRoomContext(); + const voiceAssistant = useVoiceAssistant(); + const { send } = useChat(); + const { appConfig, isSessionActive, endSession, getCurrentSessionId, browserSourceClient } = + useSession(); + const messages = useChatMessages({ + enableSmartParticipantMatching: appConfig.enableSmartParticipantMatching, + enableTranscriptionDebug: appConfig.enableTranscriptionDebug, + userTranscriptionIdentities: appConfig.userTranscriptionIdentities, + }); + const [models, setModels] = useState(createDefaultCatalogModels); + const [canvas, setCanvas] = useState(() => createCanvasState([])); + const [lastRecipeByDock, setLastRecipeByDock] = useState({}); + const [microphoneEnabled, setMicrophoneEnabled] = useState(true); + const [cameraEnabled, setCameraEnabled] = useState(true); + const resources = useMemo(() => createAgentWidgetResourceRegistry(), []); + const runtimeRef = useRef({}); + + const connection = isSessionActive + ? room.state === 'disconnected' + ? 'connecting' + : room.state + : 'idle'; + const agentState = isSessionActive ? (voiceAssistant.state ?? 'idle') : 'idle'; + const launchPhase = isSessionActive || canvas.surfaceIds.length > 0 ? 'active' : 'intro'; + runtimeRef.current = { + agentState, + cameraEnabled, + connection, + microphoneEnabled, + onStartCall, + room, + send, + }; + + const setMicrophone = useCallback( + async (enabled) => { + if (browserSourceClient.enabled && appConfig.usesBrowserRawAudioInput) { + await browserSourceClient.setAudioEnabled(enabled); + } else { + await room.localParticipant.setMicrophoneEnabled(enabled); + } + setMicrophoneEnabled(enabled); + }, + [appConfig.usesBrowserRawAudioInput, browserSourceClient, room] + ); + + const setCamera = useCallback( + async (enabled) => { + if (browserSourceClient.enabled && appConfig.usesBrowserRawVideoInput) { + await browserSourceClient.setVideoEnabled(enabled); + } else { + await room.localParticipant.setCameraEnabled(enabled); + } + setCameraEnabled(enabled); + }, + [appConfig.usesBrowserRawVideoInput, browserSourceClient, room] + ); + + runtimeRef.current.setCamera = setCamera; + runtimeRef.current.setMicrophone = setMicrophone; + + const adapterClient = useMemo(() => { + const listeners = new Set(); + const snapshot = () => ({ + agentState: runtimeRef.current.agentState, + cameraEnabled: runtimeRef.current.cameraEnabled, + connection: runtimeRef.current.connection, + microphoneEnabled: runtimeRef.current.microphoneEnabled, + roomName: room.name || null, + }); + const emit = (type) => { + const next = snapshot(); + for (const listener of listeners) listener({ type }, next); + }; + const onConnection = () => emit('connection'); + room.on(RoomEvent.ConnectionStateChanged, onConnection); + return { + room, + resourceRegistry: resources, + getSnapshot: snapshot, + subscribe(listener) { + listeners.add(listener); + return () => listeners.delete(listener); + }, + startAudio: () => room.startAudio(), + start: () => Promise.resolve(runtimeRef.current.onStartCall?.()), + sendText: (text) => runtimeRef.current.send(text), + stop: () => endSession(), + setMicrophoneEnabled: (enabled) => runtimeRef.current.setMicrophone(enabled), + setCameraEnabled: (enabled) => runtimeRef.current.setCamera(enabled), + dispose: async () => { + room.off(RoomEvent.ConnectionStateChanged, onConnection); + listeners.clear(); + resources.clear(); + }, + }; + }, [endSession, resources, room]); + + const adapter = useMemo(() => createLexVoiceAdapter({ client: adapterClient }), [adapterClient]); + useEffect(() => () => void adapterClient.dispose(), [adapterClient]); + + useEffect(() => { + if (!hasConfiguredAgentWidgetSurfaceChannel()) return undefined; + const client = createAgentWidgetSurfaceChannelClient(); + const unsubscribe = client.subscribe((event) => { + if (event.type !== 'surface') return; + const widget = event.envelope?.surface; + const recipe = recipeForWidgetType(widget?.widgetType); + const payload = widget?.payload ?? widget?.data; + if (!recipe || !payload || typeof payload !== 'object') return; + setModels((current) => ({ + ...current, + [recipe.id]: { + widgetType: recipe.id, + title: widget.title || current[recipe.id].title, + payload, + }, + })); + setLastRecipeByDock((current) => ({ ...current, [recipe.dockId]: recipe.id })); + setCanvas((current) => upsertCanvasSurface(current, recipe.id)); + }); + return () => { + unsubscribe(); + client.dispose(); + }; + }, []); + + const stopSession = useCallback(() => { + const sessionId = getCurrentSessionId() ?? getActiveAgentSession()?.sessionId; + const localCleanup = Promise.resolve().then(() => adapter.stop()); + registerAgentSessionLocalCleanup(localCleanup); + void requestAgentSessionStop(sessionId); + }, [adapter, getCurrentSessionId]); + + const selectDock = useCallback( + (dockId) => { + const recipeId = lastRecipeByDock[dockId]; + if (!recipeId) return; + setCanvas((current) => upsertCanvasSurface(current, recipeId)); + }, + [lastRecipeByDock] + ); + + const openDockIds = useMemo( + () => [ + ...new Set( + canvas.surfaceIds.map((recipeId) => RECIPE_BY_ID.get(recipeId)?.dockId).filter(Boolean) + ), + ], + [canvas.surfaceIds] + ); + const focusedRecipe = RECIPE_BY_ID.get(canvas.focusedSurfaceId); + const focusedDockId = focusedRecipe?.dockId ?? null; + const latestAgentMessage = [...messages] + .reverse() + .find((message) => message.from?.isLocal === false || !message.from); + const agentCaption = latestAgentMessage + ? { + id: latestAgentMessage.id, + source: 'agent', + text: latestAgentMessage.message, + } + : null; + + return ( +
+ +
+ ); +} diff --git a/components/app/view-controller.tsx b/components/app/view-controller.tsx index a89274fb7..dc1ef4ce9 100644 --- a/components/app/view-controller.tsx +++ b/components/app/view-controller.tsx @@ -1,38 +1,14 @@ 'use client'; -import { useRef, useState, useSyncExternalStore } from 'react'; -import { AnimatePresence, motion } from 'motion/react'; +import { useState, useSyncExternalStore } from 'react'; import { useRoomContext } from '@livekit/components-react'; +import { AiFrontdesk } from '@/components/agentwidget/ai-frontdesk'; import { useSession } from '@/components/app/session-provider'; -import { SessionView } from '@/components/app/session-view'; -import { WelcomeView } from '@/components/app/welcome-view'; import { getAgentSessionStopPending, subscribeAgentSessionStop } from '@/lib/session-stop-client'; -const MotionWelcomeView = motion.create(WelcomeView); -const MotionSessionView = motion.create(SessionView); - -const VIEW_MOTION_PROPS = { - variants: { - visible: { - opacity: 1, - }, - hidden: { - opacity: 0, - }, - }, - initial: 'hidden', - animate: 'visible', - exit: 'hidden', - transition: { - duration: 0.5, - ease: 'linear', - }, -}; - export function ViewController() { const room = useRoomContext(); - const isSessionActiveRef = useRef(false); - const { appConfig, isSessionActive, startSession } = useSession(); + const { isSessionActive, startSession } = useSession(); const [startPending, setStartPending] = useState(false); const stopPending = useSyncExternalStore( subscribeAgentSessionStop, @@ -41,21 +17,17 @@ export function ViewController() { ); const isStartDisabled = isSessionActive || stopPending || startPending; - // animation handler holds a reference to stale isSessionActive value - isSessionActiveRef.current = isSessionActive; - - // disconnect room after animation completes - const handleAnimationComplete = () => { - if (!isSessionActiveRef.current && room.state !== 'disconnected') { - room.disconnect(); - } - }; - const handleStartCall = () => { if (isStartDisabled) { return; } + // Preserve browser autoplay authority while the call still runs inside + // the user's click gesture. Connecting the room first can lose it. + void room.startAudio().catch((error: unknown) => { + console.warn('Unable to unlock room audio from the start-call gesture', error); + }); + void (async () => { setStartPending(true); try { @@ -67,28 +39,10 @@ export function ViewController() { }; return ( - - {/* Welcome screen */} - {!isSessionActive && ( - - )} - {/* Session view */} - {isSessionActive && ( - - )} - + ); } diff --git a/hooks/useRoom.ts b/hooks/useRoom.ts index d44922f1d..40365854d 100644 --- a/hooks/useRoom.ts +++ b/hooks/useRoom.ts @@ -36,13 +36,7 @@ function requiresRoomVideoInputReady(appConfig: AppConfig) { export function useRoom(appConfig: AppConfig) { const aborted = useRef(false); const sessionIdRef = useRef(null); - const room = useMemo( - () => - new Room({ - reconnectPolicy: { nextRetryDelayInMs: () => null }, - }), - [] - ); + const room = useMemo(() => new Room(), []); const [isSessionActive, setIsSessionActive] = useState(false); const handleBrowserVideoError = useCallback((error: Error) => { toastAlert({ @@ -215,6 +209,7 @@ export function useRoom(appConfig: AppConfig) { setIsSessionActive(true); beginFrontendObservabilitySession(room); + let hasDispatchedAgentSession = false; const dispatchAgentSession = async () => { recordFrontendObservability(FRONTEND_EVENTS.DISPATCH_STARTED); dispatchSessionId = sessionId; @@ -225,6 +220,7 @@ export function useRoom(appConfig: AppConfig) { }); registerAgentSessionDispatch(room.name, sessionId, dispatchPromise); await dispatchPromise; + hasDispatchedAgentSession = true; recordFrontendObservability(FRONTEND_EVENTS.DISPATCH_FINISHED); await flushFrontendObservabilityEvents({ enabled: !!appConfig.observabilityEnabled, @@ -245,7 +241,6 @@ export function useRoom(appConfig: AppConfig) { } if (appConfig.usesServerRoomInput) { - await room.localParticipant.setMicrophoneEnabled(false); return; } @@ -292,6 +287,9 @@ export function useRoom(appConfig: AppConfig) { throw dispatchResult.reason; } } else { + if (appConfig.usesServerRoomInput) { + await dispatchAgentSession(); + } await startLocalInput(); } } else { @@ -307,7 +305,7 @@ export function useRoom(appConfig: AppConfig) { ]); } - if (!usesSandboxConcurrentStartup) { + if (!usesSandboxConcurrentStartup && !hasDispatchedAgentSession) { await dispatchAgentSession(); } } catch (error) { diff --git a/lib/agentwidget/surface-channel.js b/lib/agentwidget/surface-channel.js new file mode 100644 index 000000000..0e958b181 --- /dev/null +++ b/lib/agentwidget/surface-channel.js @@ -0,0 +1,46 @@ +const CHANNEL_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/; +const GLOBAL_KEY = Symbol.for('lexmount.agentwidget.surface-channel'); + +export function parseAgentWidgetChannelId(value) { + if (typeof value !== 'string' || !CHANNEL_PATTERN.test(value)) { + throw new TypeError('Invalid AgentWidget channel id'); + } + return value; +} + +function createHub() { + const channels = new Map(); + + const entryFor = (channelId) => { + let entry = channels.get(channelId); + if (!entry) { + entry = { latest: null, listeners: new Set() }; + channels.set(channelId, entry); + } + return entry; + }; + + return { + publish(channelId, envelope) { + const entry = entryFor(parseAgentWidgetChannelId(channelId)); + entry.latest = envelope; + for (const listener of entry.listeners) listener(envelope); + }, + + subscribe(channelId, listener) { + const resolved = parseAgentWidgetChannelId(channelId); + const entry = entryFor(resolved); + entry.listeners.add(listener); + if (entry.latest) listener(entry.latest); + return () => { + entry.listeners.delete(listener); + if (!entry.latest && entry.listeners.size === 0) channels.delete(resolved); + }; + }, + }; +} + +export function getAgentWidgetSurfaceChannel() { + globalThis[GLOBAL_KEY] ??= createHub(); + return globalThis[GLOBAL_KEY]; +} diff --git a/lib/utils.ts b/lib/utils.ts index beb2f368b..f01818475 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -187,7 +187,7 @@ export function getClientConfigFromEnv(): AppConfig { inputDeviceConfig.usesBrowserRawVideoInput, inputDeviceConfig.usesServerRoomInput ), - defaultVideoTrack: getDefaultVideoTrack(), + defaultVideoTrack: getDefaultVideoTrack(inputDeviceConfig.usesBrowserRawVideoInput), browserMediaStreamName: readEnv( 'BROWSER_MEDIA_STREAM_NAME', diff --git a/next.config.ts b/next.config.ts index cf939c7b9..aede3a517 100644 --- a/next.config.ts +++ b/next.config.ts @@ -3,6 +3,7 @@ import type { NextConfig } from 'next'; const nextConfig: NextConfig = { assetPrefix: '.', allowedDevOrigins: ['liveavatar.local.lexmount.net'], + transpilePackages: ['@lexmount/agentwidget-sdk'], }; export default nextConfig; diff --git a/package.json b/package.json index 92b4fdb67..810595748 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "debug:status": "node scripts/toggle-debug.js" }, "dependencies": { + "@lexmount/agentwidget-sdk": "git+https://github.com/lexmount/agent-uilib.git#996976ea8c4a2de515943a6d5138e385c6f6cd50", "@livekit/components-react": "^2.9.15", "@livekit/protocol": "^1.40.0", "@phosphor-icons/react": "^2.1.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e9818bec2..e7274f6c4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@lexmount/agentwidget-sdk': + specifier: git+https://github.com/lexmount/agent-uilib.git#996976ea8c4a2de515943a6d5138e385c6f6cd50 + version: git+https://github.com/lexmount/agent-uilib.git#996976ea8c4a2de515943a6d5138e385c6f6cd50(@types/dom-mediacapture-record@1.0.22)(react-dom@19.1.1(react@19.1.1))(react@19.1.1) '@livekit/components-react': specifier: ^2.9.15 version: 2.9.15(@livekit/krisp-noise-filter@0.2.16(livekit-client@2.15.8(@types/dom-mediacapture-record@1.0.22)))(livekit-client@2.15.8(@types/dom-mediacapture-record@1.0.22))(react-dom@19.1.1(react@19.1.1))(react@19.1.1)(tslib@2.8.1) @@ -562,6 +565,13 @@ packages: '@jridgewell/trace-mapping@0.3.30': resolution: {integrity: sha512-GQ7Nw5G2lTu/BtHTKfXhKHok2WGetd4XYcVKGx00SjAk8GMwgJM3zr6zORiPGuOE+/vkc90KtTosSSvaCjKb2Q==} + '@lexmount/agentwidget-sdk@git+https://github.com/lexmount/agent-uilib.git#996976ea8c4a2de515943a6d5138e385c6f6cd50': + resolution: {commit: 996976ea8c4a2de515943a6d5138e385c6f6cd50, repo: https://github.com/lexmount/agent-uilib.git, type: git} + version: 0.1.0 + peerDependencies: + react: '>=19' + react-dom: '>=19' + '@livekit/components-core@0.12.10': resolution: {integrity: sha512-lSGci8c8IB/qCi42g1tzNtDGpnBWH1XSSk/OA9Lzk7vqOG0LlkwD3zXfBeKfO2eWFmYRfrZ2GD59GaH2NtTgag==} engines: {node: '>=18'} @@ -596,6 +606,9 @@ packages: '@livekit/protocol@1.42.0': resolution: {integrity: sha512-42sYSCay2PZrn5yHHt+O3RQpTElcTrA7bqg7iYbflUApeerA5tUCJDr8Z4abHsYHVKjqVUbkBq/TPmT3X6aYOQ==} + '@livekit/protocol@1.50.4': + resolution: {integrity: sha512-L1uggNQAqyY21smQY8AllyOYbcv9Me9TaxwuLytL1R8ck9nbYPmQLNwEDi3pOFGAMa5F8I2nUi2Jc59W5awxlA==} + '@napi-rs/wasm-runtime@0.2.10': resolution: {integrity: sha512-bCsCyeZEwVErsGmyPNSzwfwFn4OdxBj0mmv6hOFucB/k81Ojdu68RbZdxYsRQUPc9l6SU5F/cG+bXgWs3oUgsQ==} @@ -1737,6 +1750,17 @@ packages: react-dom: optional: true + framer-motion@13.1.0: + resolution: {integrity: sha512-QSZrF0Id3QGuHJ+OL+9PSY9pk86C8ERFalwAGSchzTm65+ZoGH/RM26lmEARLljcHj2lqhv0jZOOks+EI3COOw==} + peerDependencies: + react: ^18.0.0 || ^19.0.0 + react-dom: ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + react: + optional: true + react-dom: + optional: true + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -2092,6 +2116,11 @@ packages: peerDependencies: '@types/dom-mediacapture-record': ^1 + livekit-client@2.22.0: + resolution: {integrity: sha512-GLtYQfRh/RsvXaOX1x609bFZ17yyKmKWDqu9JmkMKn9vIFLi2GsapRv9gT8OJO/1R2dsitrkbGmloyUfxWQsaA==} + peerDependencies: + '@types/dom-mediacapture-record': ^1 + livekit-server-sdk@2.13.3: resolution: {integrity: sha512-ItSQ2gE1oz/Ev9mfBRdAw+P05rt/BaYRkldggKz0+3rh/Yt0ag0BLID3VrgCVFVRAQ2YEJKcJJyj5p4epIJ8QA==} engines: {node: '>=18'} @@ -2174,9 +2203,15 @@ packages: motion-dom@12.16.0: resolution: {integrity: sha512-Z2nGwWrrdH4egLEtgYMCEN4V2qQt1qxlKy/uV7w691ztyA41Q5Rbn0KNGbsNVDZr9E8PD2IOQ3hSccRnB6xWzw==} + motion-dom@13.0.0: + resolution: {integrity: sha512-Xk+SJas70uMAUIApg+m3lZDShxI3LBFHq7mFGbBKoRXc2PVPDyAKmzN64Bbzt4CZdP/CItTiJxWtn4TA0v53Ng==} + motion-utils@12.12.1: resolution: {integrity: sha512-f9qiqUHm7hWSLlNW8gS9pisnsN7CRFRD58vNjptKdsqFLpkVnX00TNeD6Q0d27V9KzT7ySFyK1TZ/DShfVOv6w==} + motion-utils@13.0.0: + resolution: {integrity: sha512-7DnN7TmbLcYXcG4RVadXIihWlyuM9afoUww8Y5Agg431kGKiuL2/OMyP4mJ5wLz+pvN3t5ySClLOaVXJ+wekRQ==} + motion@12.16.0: resolution: {integrity: sha512-P3HA83fnPMEGBLfKdD5vDdjH1Aa3wM3jT3+HX3fCVpy/4/lJiqvABajLgZenBu+rzkFzmeaPkvT7ouf9Tq5tVQ==} peerDependencies: @@ -2191,6 +2226,17 @@ packages: react-dom: optional: true + motion@13.1.0: + resolution: {integrity: sha512-qtvscq59uCPdWnNW4SdSkrxR+BS/QYsa923bx7ocA+4p+ZGNbbVQwkSnG4aukB81QWjtl3AxX36plxNyZLmHCA==} + peerDependencies: + react: ^18.0.0 || ^19.0.0 + react-dom: ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + react: + optional: true + react-dom: + optional: true + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -2658,6 +2704,9 @@ packages: resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==} engines: {node: '>=18'} + three@0.134.0: + resolution: {integrity: sha512-LbBerg7GaSPjYtTOnu41AMp7tV6efUNR3p4Wk5NzkSsNTBuA5mDGOfwwZL1jhhVMLx9V20HolIUo0+U3AXehbg==} + tinyglobby@0.2.13: resolution: {integrity: sha512-mEwzpUgrLySlveBwEVDMKk5B57bhLPYovRfPAXD5gA/98Opn0rCDj3GtLwFvCvH5RK9uPCExUROW5NjDwvqkxw==} engines: {node: '>=12.0.0'} @@ -2760,10 +2809,17 @@ packages: peerDependencies: react: ^16.8.0 || ^17 || ^18 || ^19 || ^19.0.0-rc + vanta@0.5.24: + resolution: {integrity: sha512-fvieEbHy1ZS23zrcX+topzqAgA4Uct1enngOEWLFBgs9TtOf6RDFOYatH7KSVdrABzQDMCQ5myQy+nTSZZwLzg==} + webrtc-adapter@9.0.3: resolution: {integrity: sha512-5fALBcroIl31OeXAdd1YUntxiZl1eHlZZWzNg3U4Fn+J9/cGL3eT80YlrsWGvj2ojuz1rZr2OXkgCzIxAZ7vRQ==} engines: {node: '>=6.0.0', npm: '>=3.10.0'} + webrtc-adapter@9.0.6: + resolution: {integrity: sha512-CHbl2ZQbxx164IgWRgzJno4hWtM4tFbRam1QfI3Yxhs3w/DvqluVxVWeXs3oL5/fbGkSNLKo0Ty5MgUWceNhog==} + engines: {node: '>=6.0.0', npm: '>=3.10.0'} + which-boxed-primitive@1.1.1: resolution: {integrity: sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA==} engines: {node: '>= 0.4'} @@ -2797,6 +2853,9 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + zod@4.4.3: + resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} + snapshots: '@alloc/quick-lru@5.2.0': {} @@ -3143,6 +3202,20 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@lexmount/agentwidget-sdk@git+https://github.com/lexmount/agent-uilib.git#996976ea8c4a2de515943a6d5138e385c6f6cd50(@types/dom-mediacapture-record@1.0.22)(react-dom@19.1.1(react@19.1.1))(react@19.1.1)': + dependencies: + '@livekit/protocol': 1.50.4 + '@phosphor-icons/react': 2.1.10(react-dom@19.1.1(react@19.1.1))(react@19.1.1) + livekit-client: 2.22.0(@types/dom-mediacapture-record@1.0.22) + motion: 13.1.0(react-dom@19.1.1(react@19.1.1))(react@19.1.1) + react: 19.1.1 + react-dom: 19.1.1(react@19.1.1) + three: 0.134.0 + vanta: 0.5.24 + zod: 4.4.3 + transitivePeerDependencies: + - '@types/dom-mediacapture-record' + '@livekit/components-core@0.12.10(livekit-client@2.15.8(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)': dependencies: '@floating-ui/dom': 1.6.13 @@ -3178,6 +3251,10 @@ snapshots: dependencies: '@bufbuild/protobuf': 1.10.1 + '@livekit/protocol@1.50.4': + dependencies: + '@bufbuild/protobuf': 1.10.1 + '@napi-rs/wasm-runtime@0.2.10': dependencies: '@emnapi/core': 1.4.3 @@ -4418,6 +4495,15 @@ snapshots: react: 19.1.1 react-dom: 19.1.1(react@19.1.1) + framer-motion@13.1.0(react-dom@19.1.1(react@19.1.1))(react@19.1.1): + dependencies: + motion-dom: 13.0.0 + motion-utils: 13.0.0 + tslib: 2.8.1 + optionalDependencies: + react: 19.1.1 + react-dom: 19.1.1(react@19.1.1) + fsevents@2.3.3: optional: true @@ -4762,6 +4848,19 @@ snapshots: typed-emitter: 2.1.0 webrtc-adapter: 9.0.3 + livekit-client@2.22.0(@types/dom-mediacapture-record@1.0.22): + dependencies: + '@livekit/mutex': 1.1.1 + '@livekit/protocol': 1.50.4 + '@types/dom-mediacapture-record': 1.0.22 + events: 3.3.0 + jose: 6.1.0 + loglevel: 1.9.2 + sdp-transform: 2.15.0 + tslib: 2.8.1 + typed-emitter: 2.1.0 + webrtc-adapter: 9.0.6 + livekit-server-sdk@2.13.3: dependencies: '@bufbuild/protobuf': 1.10.1 @@ -4828,8 +4927,14 @@ snapshots: dependencies: motion-utils: 12.12.1 + motion-dom@13.0.0: + dependencies: + motion-utils: 13.0.0 + motion-utils@12.12.1: {} + motion-utils@13.0.0: {} + motion@12.16.0(react-dom@19.1.1(react@19.1.1))(react@19.1.1): dependencies: framer-motion: 12.16.0(react-dom@19.1.1(react@19.1.1))(react@19.1.1) @@ -4838,6 +4943,14 @@ snapshots: react: 19.1.1 react-dom: 19.1.1(react@19.1.1) + motion@13.1.0(react-dom@19.1.1(react@19.1.1))(react@19.1.1): + dependencies: + framer-motion: 13.1.0(react-dom@19.1.1(react@19.1.1))(react@19.1.1) + tslib: 2.8.1 + optionalDependencies: + react: 19.1.1 + react-dom: 19.1.1(react@19.1.1) + ms@2.1.3: {} nanoid@3.3.11: {} @@ -5322,6 +5435,8 @@ snapshots: mkdirp: 3.0.1 yallist: 5.0.0 + three@0.134.0: {} + tinyglobby@0.2.13: dependencies: fdir: 6.4.4(picomatch@4.0.2) @@ -5454,10 +5569,16 @@ snapshots: lodash.debounce: 4.0.8 react: 19.1.1 + vanta@0.5.24: {} + webrtc-adapter@9.0.3: dependencies: sdp: 3.2.1 + webrtc-adapter@9.0.6: + dependencies: + sdp: 3.2.1 + which-boxed-primitive@1.1.1: dependencies: is-bigint: 1.1.0 @@ -5508,3 +5629,5 @@ snapshots: yallist@5.0.0: {} yocto-queue@0.1.0: {} + + zod@4.4.3: {} diff --git a/styles/agentwidget-frontdesk.css b/styles/agentwidget-frontdesk.css new file mode 100644 index 000000000..5236e675a --- /dev/null +++ b/styles/agentwidget-frontdesk.css @@ -0,0 +1,171 @@ +.ai-frontdesk { + position: fixed; + inset: 0; + min-width: 320px; + overflow: hidden; + background: #fff; + color: #151922; +} + +.ai-frontdesk-curtain { + position: fixed; + z-index: 2; + inset: 0; + pointer-events: none; + background: radial-gradient(circle at 50% 52%, transparent 0 14%, #fff 56%); + opacity: 0; + transition: opacity 500ms ease; +} + +.ai-frontdesk[data-launch-phase='intro'] .ai-frontdesk-curtain { + opacity: 1; +} + +.ai-frontdesk-header { + position: fixed; + z-index: 30; + top: 0; + right: 0; + left: 0; + display: flex; + height: 58px; + align-items: center; + justify-content: space-between; + padding: 0 24px; + pointer-events: none; +} + +.ai-frontdesk-brand { + font-size: 15px; + font-weight: 720; + letter-spacing: -0.02em; +} + +.ai-frontdesk-status { + border: 1px solid rgb(15 23 42 / 7%); + border-radius: 999px; + background: rgb(255 255 255 / 74%); + padding: 7px 11px; + color: #667085; + font-size: 11px; + backdrop-filter: blur(18px); +} + +.ai-frontdesk-intro { + position: fixed; + z-index: 24; + top: calc(50% - 148px); + left: 50%; + width: min(620px, calc(100vw - 40px)); + transform: translateX(-50%); + color: #344054; + font-size: clamp(16px, 2.4vw, 22px); + font-weight: 540; + letter-spacing: -0.025em; + text-align: center; +} + +.ai-frontdesk-video { + position: fixed; + z-index: 1; + inset: 0; + opacity: 0.36; + filter: saturate(0.8); + transition: opacity 360ms ease; +} + +.ai-frontdesk-video[aria-hidden='true'] { + opacity: 0.1; +} + +.ai-frontdesk-stage { + position: relative; + z-index: 10; + display: flex; + min-height: 100svh; + align-items: center; + justify-content: center; + padding: 78px 28px 150px; +} + +.ai-frontdesk[data-launch-phase='intro'] .ai-frontdesk-stage { + opacity: 0; + pointer-events: none; +} + +.ai-frontdesk-canvas { + display: grid; + width: 100%; + max-width: 1000px; + max-height: calc(100svh - 230px); + align-items: center; + justify-content: center; + gap: 22px; +} + +.ai-frontdesk-canvas[data-canvas-scene='empty'], +.ai-frontdesk-canvas[data-canvas-scene='spotlight'] { + grid-template-columns: minmax(0, 620px); +} + +.ai-frontdesk-canvas:not([data-canvas-scene='empty']):not([data-canvas-scene='spotlight']) { + grid-template-columns: 344px minmax(440px, 620px); +} + +.ai-frontdesk-surface { + display: flex; + min-width: 0; + align-items: center; + justify-content: center; +} + +.ai-frontdesk-surface[data-focused='true'] { + grid-column: 2; + grid-row: 1 / span 5; +} + +.ai-frontdesk-canvas[data-canvas-scene='spotlight'] .ai-frontdesk-surface[data-focused='true'] { + grid-column: 1; +} + +.ai-frontdesk-surface[data-focused='false'] { + grid-column: 1; +} + +.ai-frontdesk-surface[data-focused='true'] > article { + width: 100%; + max-width: 620px; + max-height: calc(100svh - 230px); +} + +@media (max-width: 860px) { + .ai-frontdesk-header { + padding-inline: 16px; + } + + .ai-frontdesk-status { + display: none; + } + + .ai-frontdesk-stage { + padding: 72px 0 142px; + } + + .ai-frontdesk-canvas, + .ai-frontdesk-canvas:not([data-canvas-scene='empty']):not([data-canvas-scene='spotlight']) { + display: flex; + max-width: none; + justify-content: flex-start; + overflow-x: auto; + gap: 14px; + padding: 16px max(22px, calc((100vw - 440px) / 2)); + scroll-snap-type: x mandatory; + scrollbar-width: none; + } + + .ai-frontdesk-surface { + width: min(440px, calc(100vw - 44px)); + flex: 0 0 min(440px, calc(100vw - 44px)); + scroll-snap-align: center; + } +} diff --git a/tests/agentwidget-frontdesk.test.mjs b/tests/agentwidget-frontdesk.test.mjs new file mode 100644 index 000000000..c4a0a9da0 --- /dev/null +++ b/tests/agentwidget-frontdesk.test.mjs @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import test from 'node:test'; + +const root = new URL('../', import.meta.url); + +const readSource = (path) => readFile(new URL(path, root), 'utf8'); + +test('AI frontdesk composes AgentWidget over the existing LiveKit room', async () => { + const [controller, frontdesk] = await Promise.all([ + readSource('components/app/view-controller.tsx'), + readSource('components/agentwidget/ai-frontdesk.jsx'), + ]); + + assert.match(controller, / 0 \? 'active' : 'intro'/); + assert.match(frontdesk, /!isSessionActive && canvas\.surfaceIds\.length === 0/); +}); + +test('AgentWidget host route publishes canonical SDK output to an isolated channel', async () => { + const [spawnRoute, surfaceRoute] = await Promise.all([ + readSource('app/api/agentwidget/spawn/route.js'), + readSource('app/api/agentwidget/surfaces/route.js'), + ]); + + assert.match(spawnRoute, /@lexmount\/agentwidget-sdk\/host/); + assert.match(spawnRoute, /createSpawnWidgetResult\(input/); + assert.match(spawnRoute, /x-agentwidget-channel-id/); + assert.match(spawnRoute, /\.publish\(/); + assert.match(surfaceRoute, /text\/event-stream/); + assert.match(surfaceRoute, /searchParams\.get\('channel'\)/); +}); + +test('surface channel isolates sessions and replays only each channel latest surface', async () => { + const { getAgentWidgetSurfaceChannel } = await import( + `../lib/agentwidget/surface-channel.js?test=${Date.now()}` + ); + const hub = getAgentWidgetSurfaceChannel(); + const firstChannel = 'agentwidget-test-channel-a'; + const secondChannel = 'agentwidget-test-channel-b'; + const firstSurface = { protocol: 'agentwidget/1.0', surface: { id: 'a' } }; + const secondSurface = { protocol: 'agentwidget/1.0', surface: { id: 'b' } }; + const firstEvents = []; + const secondEvents = []; + + hub.publish(firstChannel, firstSurface); + const unsubscribeFirst = hub.subscribe(firstChannel, (event) => firstEvents.push(event)); + const unsubscribeSecond = hub.subscribe(secondChannel, (event) => secondEvents.push(event)); + hub.publish(secondChannel, secondSurface); + + assert.deepEqual(firstEvents, [firstSurface]); + assert.deepEqual(secondEvents, [secondSurface]); + unsubscribeFirst(); + unsubscribeSecond(); +}); + +test('AgentWidget dependency is pinned to the reviewed UI library commit', async () => { + const manifest = JSON.parse(await readSource('package.json')); + assert.equal( + manifest.dependencies['@lexmount/agentwidget-sdk'], + 'git+https://github.com/lexmount/agent-uilib.git#996976ea8c4a2de515943a6d5138e385c6f6cd50' + ); +});