diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index cd206abc1c..87c3539a71 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -116,6 +116,7 @@ jest.mock("./components/Layout/MainLayout", () => { }); jest.mock("./components/Chat/ChatWindow", () => { + const { useLocation } = jest.requireActual("react-router") as typeof import("react-router"); const MockChatWindow = ({ onNewAttack, activeTarget, @@ -126,6 +127,7 @@ jest.mock("./components/Chat/ChatWindow", () => { onConversationCreated, onSelectConversation, labels, + scenarioResultId, }: { onNewAttack: () => void; activeTarget: unknown; @@ -136,7 +138,9 @@ jest.mock("./components/Chat/ChatWindow", () => { onConversationCreated: (attackResultId: string, conversationId: string) => void; onSelectConversation: (convId: string) => void; labels: Record; + scenarioResultId?: string | null; }) => { + const location = useLocation(); return (
{attackResultId ?? "none"} @@ -146,6 +150,8 @@ jest.mock("./components/Chat/ChatWindow", () => { {attackTarget?.identifier_hash ?? "none"} {labels.operator ?? ""} {JSON.stringify(labels)} + {scenarioResultId ?? "none"} + {`${location.pathname}${location.search}`} @@ -333,12 +339,12 @@ jest.mock("./components/Scenarios/ScenarioDetail", () => { }; }); -jest.mock("./components/Scenarios/ScenarioRunStarted", () => { - const MockScenarioRunStarted = () =>
; - MockScenarioRunStarted.displayName = "MockScenarioRunStarted"; +jest.mock("./components/Scenarios/ScenarioRunPage", () => { + const MockScenarioRunPage = () =>
; + MockScenarioRunPage.displayName = "MockScenarioRunPage"; return { __esModule: true, - default: MockScenarioRunStarted, + default: MockScenarioRunPage, }; }); @@ -417,14 +423,14 @@ describe("App", () => { expect(screen.getByTestId("scenario-detail")).toBeInTheDocument(); }); - it("renders the scenario run-started shell and marks the sidebar current when deep-linked to /scenario-history/:id", () => { + it("renders the scenario run dashboard and marks the sidebar current when deep-linked to /scenario-history/:id", () => { renderApp("/scenario-history/sr-123"); expect(screen.getByTestId("main-layout")).toHaveAttribute( "data-current-view", "scenarios" ); - expect(screen.getByTestId("scenario-run-started")).toBeInTheDocument(); + expect(screen.getByTestId("scenario-run-page")).toBeInTheDocument(); }); it("switches to the scenarios view via the sidebar", () => { @@ -812,6 +818,70 @@ describe("App", () => { expect(screen.getByTestId("conversation-id")).toHaveTextContent("conv-main") ); expect(screen.getByTestId("active-conversation-id")).toHaveTextContent("conv-main"); + expect(screen.getByTestId("scenario-result-id")).toHaveTextContent("none"); + }); + + it("hydrates validated scenario provenance on a direct attack reload", async () => { + const scenarioResultId = "123e4567-e89b-12d3-a456-426614174000"; + mockGetAttack.mockResolvedValue({ + attack_result_id: "ar-1", + conversation_id: "conv-main", + labels: {}, + related_conversation_ids: [], + }); + + renderApp(`/attacks/ar-1?scenarioResultId=${scenarioResultId}`); + + await waitFor(() => + expect(screen.getByTestId("scenario-result-id")).toHaveTextContent(scenarioResultId) + ); + expect(screen.getByTestId("route-location")).toHaveTextContent( + `/attacks/ar-1?scenarioResultId=${scenarioResultId}` + ); + }); + + it.each([ + "/attacks/ar-1?scenarioResultId=run-1", + "/attacks/ar-1?scenarioResultId=https%3A%2F%2Fevil.example", + "/attacks/ar-1?scenarioResultId=123e4567-e89b-12d3-a456-426614174000&scenarioResultId=123e4567-e89b-12d3-a456-426614174000", + ])("ignores unsafe or ambiguous scenario provenance on %s", async (path: string) => { + mockGetAttack.mockResolvedValue({ + attack_result_id: "ar-1", + conversation_id: "conv-main", + labels: {}, + related_conversation_ids: [], + }); + + renderApp(path); + + await waitFor(() => + expect(screen.getByTestId("conversation-id")).toHaveTextContent("conv-main") + ); + expect(screen.getByTestId("scenario-result-id")).toHaveTextContent("none"); + }); + + it("preserves validated provenance within an attack and clears it for a new attack", async () => { + const scenarioResultId = "123e4567-e89b-12d3-a456-426614174000"; + mockGetAttack.mockResolvedValue({ + attack_result_id: "ar-1", + conversation_id: "conv-main", + labels: {}, + related_conversation_ids: ["conv-456"], + }); + renderApp(`/attacks/ar-1?scenarioResultId=${scenarioResultId}`); + await waitFor(() => + expect(screen.getByTestId("conversation-id")).toHaveTextContent("conv-main") + ); + + fireEvent.click(screen.getByTestId("select-conversation")); + expect(screen.getByTestId("route-location")).toHaveTextContent( + `/attacks/ar-1/conversations/conv-456?scenarioResultId=${scenarioResultId}` + ); + expect(screen.getByTestId("scenario-result-id")).toHaveTextContent(scenarioResultId); + + fireEvent.click(screen.getByTestId("new-attack")); + expect(screen.getByTestId("route-location")).toHaveTextContent("/chat"); + expect(screen.getByTestId("scenario-result-id")).toHaveTextContent("none"); }); it("uses the conversation from a deep link when it belongs to the attack", async () => { @@ -843,6 +913,24 @@ describe("App", () => { ); }); + it("retains validated provenance while canonicalizing an unknown conversation route", async () => { + const scenarioResultId = "123e4567-e89b-12d3-a456-426614174000"; + mockGetAttack.mockResolvedValue({ + attack_result_id: "ar-1", + conversation_id: "conv-main", + labels: {}, + related_conversation_ids: [], + }); + renderApp(`/attacks/ar-1/conversations/bogus?scenarioResultId=${scenarioResultId}`); + + await waitFor(() => + expect(screen.getByTestId("route-location")).toHaveTextContent( + `/attacks/ar-1?scenarioResultId=${scenarioResultId}` + ) + ); + expect(screen.getByTestId("scenario-result-id")).toHaveTextContent(scenarioResultId); + }); + it("hydrates history filters from the URL query string", () => { renderApp("/history?outcome=success&attackType=PromptSendingAttack"); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 0dfeaa5640..a8d5446fb8 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -12,7 +12,7 @@ import Initializers from './components/Initializers/Initializers' import AttackHistory from './components/History/AttackHistory' import ScenarioCatalog from './components/Scenarios/ScenarioCatalog' import ScenarioDetail from './components/Scenarios/ScenarioDetail' -import ScenarioRunStarted from './components/Scenarios/ScenarioRunStarted' +import ScenarioRunPage from './components/Scenarios/ScenarioRunPage' import FeedbackDialog from './components/Feedback/FeedbackDialog' import type { HistoryFilters } from './components/History/historyFilters' import { ConnectionBanner } from './components/ConnectionBanner' @@ -31,6 +31,11 @@ import { import { attacksApi, versionApi } from './services/api' import { toApiError } from './services/errors' import { useTour } from './hooks/useTour' +import { + attackConversationRoutePath, + attackRoutePath, + scenarioRunProvenance, +} from './utils/routeParams' const AUTO_DISMISS_MS = 5_000 @@ -73,10 +78,6 @@ interface LoadedAttack { status: AttackLoadStatus } -const attackPath = (attackId: string) => `/attacks/${attackId}` -const conversationPath = (attackId: string, conversationId: string) => - `/attacks/${attackId}/conversations/${conversationId}` - function ConnectionBannerContainer() { const { status, reconnectCount } = useConnectionHealth() // Track how many reconnects the user has already had the banner dismissed for. @@ -122,6 +123,10 @@ function App() { // the History nav button can restore filters after visiting another view. const [searchParams, setSearchParams] = useSearchParams() const historyFilters = useMemo(() => filtersFromSearchParams(searchParams), [searchParams]) + const scenarioResultId = useMemo( + () => scenarioRunProvenance(searchParams), + [searchParams], + ) const lastHistorySearch = useRef('') useEffect(() => { if (location.pathname === VIEW_PATHS.history) { @@ -277,10 +282,10 @@ function App() { routeConversationId === readyAttack.mainConversationId || readyAttack.relatedConversationIds.includes(routeConversationId) if (!isKnown) { - navigate(attackPath(readyAttack.id), { replace: true }) + navigate(attackRoutePath(readyAttack.id, scenarioResultId), { replace: true }) } } - }, [readyAttack, routeConversationId, navigate]) + }, [readyAttack, routeConversationId, navigate, scenarioResultId]) const handleNavigate = useCallback((view: ViewName) => { // Re-attach the last filter query so returning to history restores filters. @@ -318,16 +323,16 @@ function App() { }) // Replace when promoting an empty /chat to its attack url (first message); // push when branching from an existing attack so Back returns to the source. - navigate(attackPath(arId), { replace: routeAttackId === null }) + navigate(attackRoutePath(arId), { replace: routeAttackId === null }) }, [activeTarget, routeAttackId, navigate]) const handleSelectConversation = useCallback((convId: string) => { if (!routeAttackId) return - navigate(conversationPath(routeAttackId, convId)) - }, [routeAttackId, navigate]) + navigate(attackConversationRoutePath(routeAttackId, convId, scenarioResultId)) + }, [routeAttackId, navigate, scenarioResultId]) const handleOpenAttack = useCallback((openAttackResultId: string) => { - navigate(attackPath(openAttackResultId)) + navigate(attackRoutePath(openAttackResultId)) }, [navigate]) const chatElement = isAttackNotFound || isAttackError ? ( @@ -353,6 +358,7 @@ function App() { attackTarget={readyAttack ? readyAttack.target : null} isLoadingAttack={isLoadingAttack} relatedConversationCount={readyAttack ? readyAttack.relatedConversationIds.length : 0} + scenarioResultId={readyAttack ? scenarioResultId : null} /> ) @@ -423,7 +429,7 @@ function App() { /> } /> - } /> + } /> = ({ children, -}) => {children}; +}) => ( + + {children} + +); function mockMatchMedia(matchesNarrowScreen: boolean): void { (window.matchMedia as jest.Mock).mockImplementation((query: string) => ({ @@ -318,6 +323,57 @@ describe("ChatWindow Integration", () => { expect(screen.getByRole("textbox")).toBeInTheDocument(); }); + it("shows a safe scenario-run breadcrumb only when provenance is present", () => { + const scenarioResultId = "123e4567-e89b-12d3-a456-426614174000"; + const { rerender } = render( + + + + ); + + expect(screen.getByRole("navigation", { name: "Attack provenance" })).toBeInTheDocument(); + expect(screen.getByRole("link", { + name: `Return to scenario run ${scenarioResultId}`, + })).toHaveAttribute("href", `/scenario-history/${scenarioResultId}`); + + rerender( + + + + ); + expect(screen.queryByRole("navigation", { name: "Attack provenance" })).not.toBeInTheDocument(); + }); + + it("returns to the originating scenario run from the breadcrumb", async () => { + const user = userEvent.setup(); + const scenarioResultId = "123e4567-e89b-12d3-a456-426614174000"; + render( + + + + } + /> + Originating scenario run} + /> + + + + ); + + await user.click(screen.getByRole("link", { + name: `Return to scenario run ${scenarioResultId}`, + })); + + expect(screen.getByRole("heading", { + level: 1, + name: "Originating scenario run", + })).toBeInTheDocument(); + }); + it("defaults to raw mode when no Markdown preference is stored", () => { render( diff --git a/frontend/src/components/Chat/ChatWindow.tsx b/frontend/src/components/Chat/ChatWindow.tsx index 7dc10d34d7..839ebf8929 100644 --- a/frontend/src/components/Chat/ChatWindow.tsx +++ b/frontend/src/components/Chat/ChatWindow.tsx @@ -2,6 +2,9 @@ import { useState, useRef, useEffect, useCallback, useMemo } from 'react' import type { ChangeEvent } from 'react' import { Button, + Breadcrumb, + BreadcrumbDivider, + BreadcrumbItem, Drawer, Menu, MenuItem, @@ -17,6 +20,7 @@ import { } from '@fluentui/react-components' import type { SwitchOnChangeData } from '@fluentui/react-components' import { AddRegular, ArrowDownloadRegular, PanelRightRegular } from '@fluentui/react-icons' +import { Link } from 'react-router' import MessageList from './MessageList' import SystemPromptBanner from './SystemPromptBanner' import ChatInputArea from './ChatInputArea' @@ -34,6 +38,7 @@ import { exportConversation } from '../../utils/conversationExport' import type { ExportFormat } from '../../utils/conversationExport' import type { Message, MessageAttachment, TargetInstance, TargetInfo } from '../../types' import { targetInfoMatchesTarget } from '../../utils/targetIdentity' +import { scenarioRunRoutePath } from '../../utils/routeParams' import type { ViewName } from '../Sidebar/Navigation' import { useChatWindowStyles } from './ChatWindow.styles' @@ -84,6 +89,8 @@ interface ChatWindowProps { isLoadingAttack?: boolean /** Number of related (non-main) conversations in the loaded attack. */ relatedConversationCount?: number + /** Validated scenario-run provenance for attacks opened from a run dashboard. */ + scenarioResultId?: string | null } export default function ChatWindow({ @@ -101,6 +108,7 @@ export default function ChatWindow({ attackTarget, isLoadingAttack, relatedConversationCount, + scenarioResultId, }: ChatWindowProps) { const styles = useChatWindowStyles() const restoreFocusTargetAttributes = useRestoreFocusTarget() @@ -684,6 +692,25 @@ export default function ChatWindow({ /> )}
+ {scenarioResultId && ( +
+ + + Scenario History + + + + + Scenario run {scenarioResultId.slice(0, 8)} + + + +
+ )}
{activeTarget ? ( diff --git a/frontend/src/components/Scenarios/ScenarioFlow.test.tsx b/frontend/src/components/Scenarios/ScenarioFlow.test.tsx new file mode 100644 index 0000000000..a4fec85ae8 --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioFlow.test.tsx @@ -0,0 +1,211 @@ +import { render, screen, waitFor, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { MemoryRouter, Route, Routes, useLocation } from 'react-router' + +import { useScenarioRunProgress } from '@/hooks/useScenarioRunProgress' +import { scenariosApi, targetsApi } from '@/services/api' +import type { + RegisteredScenario, + ScenarioDefaultRunSizeEstimate, + TargetInstance, +} from '@/types' +import type { ScenarioRunProgressState } from '@/utils/scenarioRunProgress' + +import ScenarioCatalog from './ScenarioCatalog' +import ScenarioDetail from './ScenarioDetail' +import ScenarioRunPage from './ScenarioRunPage' + +jest.mock('@/hooks/useScenarioRunProgress', () => ({ + useScenarioRunProgress: jest.fn(), +})) + +jest.mock('@/services/api', () => ({ + scenariosApi: { + cancelRun: jest.fn(), + estimateRun: jest.fn(), + getScenario: jest.fn(), + listCatalog: jest.fn(), + startRun: jest.fn(), + }, + targetsApi: { + listTargets: jest.fn(), + }, +})) + +const mockUseScenarioRunProgress = useScenarioRunProgress as jest.Mock +const mockEstimateRun = scenariosApi.estimateRun as jest.Mock +const mockGetScenario = scenariosApi.getScenario as jest.Mock +const mockListCatalog = scenariosApi.listCatalog as jest.Mock +const mockStartRun = scenariosApi.startRun as jest.Mock +const mockListTargets = targetsApi.listTargets as jest.Mock + +const SCENARIO_NAME = 'foundry.red_team_agent' +const RUN_ID = '123e4567-e89b-12d3-a456-426614174000' + +const SCENARIO: RegisteredScenario = { + scenario_name: SCENARIO_NAME, + scenario_type: 'RedTeamAgentScenario', + scenario_version: 1, + description: 'Red teams a configured target.', + description_markdown: 'Red teams a configured target.', + default_technique: 'default_technique', + default_techniques: ['crescendo'], + aggregate_techniques: ['default_technique'], + aggregate_technique_expansions: { + default_technique: ['crescendo'], + }, + all_techniques: ['crescendo'], + default_datasets: ['harmbench'], + default_dataset_summaries: [], + baseline_policy: 'enabled', + include_baseline_by_default: true, + supported_parameters: [], + default_run_size: { + version: 1, + status: 'exact', + total_attack_count: 2, + components: [], + datasets: [], + note: null, + retries_included: false, + }, +} + +const TARGET: TargetInstance = { + target_registry_name: 'target-a', + identifier: { + class_name: 'OpenAIChatTarget', + hash: 'target-a-hash', + }, +} + +const ESTIMATE: ScenarioDefaultRunSizeEstimate = { + version: 1, + status: 'exact', + total_attack_count: 2, + components: [{ + label: 'Configured attacks', + count: 2, + factors: [], + is_baseline: false, + note: null, + }], + datasets: [], + note: null, + retries_included: false, +} + +const RUN_STATE: ScenarioRunProgressState = { + loadStatus: 'ready', + run: { + scenario_result_id: RUN_ID, + scenario_name: 'RedTeamAgentScenario', + scenario_registry_name: SCENARIO_NAME, + scenario_version: 1, + status: 'IN_PROGRESS', + created_at: '2026-08-07T18:00:00Z', + }, + plan: { + version: 1, + scenario_registry_name: SCENARIO_NAME, + atomic_groups: [], + seed_groups: [], + }, + planComplete: true, + activeAtomicGroupIds: [], + results: [], + cursor: 'cursor-0', + hasMore: false, + error: null, + stale: false, +} + +function LocationProbe() { + const location = useLocation() + return {`${location.pathname}${location.search}`} +} + +function renderFlow(): void { + render( + + + + + } /> + + )} + /> + } /> + + + , + ) +} + +describe('Scenario catalog-to-run integration', () => { + beforeEach(() => { + jest.clearAllMocks() + mockListCatalog.mockResolvedValue({ + items: [SCENARIO], + pagination: { limit: 200, has_more: false }, + }) + mockGetScenario.mockResolvedValue(SCENARIO) + mockListTargets.mockResolvedValue({ + items: [TARGET], + pagination: { limit: 200, has_more: false }, + }) + mockEstimateRun.mockResolvedValue(ESTIMATE) + mockStartRun.mockResolvedValue({ scenario_result_id: RUN_ID }) + mockUseScenarioRunProgress.mockReturnValue({ + state: RUN_STATE, + retry: jest.fn(), + applyRunSummary: jest.fn(), + }) + }) + + it('carries one configured request from catalog detail through estimate, launch, and run hydration', async () => { + const user = userEvent.setup() + renderFlow() + + await user.click(await screen.findByRole('link', { name: SCENARIO_NAME })) + expect(await screen.findByRole('heading', { level: 1, name: SCENARIO_NAME })).toBeInTheDocument() + + const expectedEstimateRequest = { + target_name: TARGET.target_registry_name, + techniques: ['default_technique'], + include_baseline: true, + } + await waitFor(() => expect(mockEstimateRun).toHaveBeenLastCalledWith( + SCENARIO_NAME, + expectedEstimateRequest, + expect.any(AbortSignal), + )) + expect(within(screen.getByRole('complementary', { name: 'Run preview' })) + .getByText('2 planned attacks')).toBeInTheDocument() + + await user.click(screen.getByTestId('launch-scenario-btn')) + + await waitFor(() => expect(mockStartRun).toHaveBeenCalledWith({ + scenario_name: SCENARIO_NAME, + target_name: TARGET.target_registry_name, + techniques: expectedEstimateRequest.techniques, + max_concurrency: 10, + max_retries: 0, + include_baseline: expectedEstimateRequest.include_baseline, + labels: { operator: 'integration-test' }, + })) + expect(await screen.findByTestId('scenario-run-page')).toBeInTheDocument() + expect(screen.getByLabelText('Current route')).toHaveTextContent( + `/scenario-history/${RUN_ID}`, + ) + expect(screen.getByRole('heading', { level: 1, name: SCENARIO_NAME })).toBeInTheDocument() + }) +}) diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.styles.ts b/frontend/src/components/Scenarios/ScenarioRunPage.styles.ts new file mode 100644 index 0000000000..4620bffa77 --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioRunPage.styles.ts @@ -0,0 +1,298 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { + MINIMUM_TOUCH_TARGET_SIZE, + NARROW_VIEWPORT_QUERY, + mobileTouchTarget, +} from '@/styles/touchTargets' + +export const useScenarioRunPageStyles = makeStyles({ + root: { + display: 'flex', + flexDirection: 'column', + width: '100%', + height: '100%', + minWidth: 0, + overflowY: 'auto', + overflowX: 'hidden', + backgroundColor: tokens.colorNeutralBackground2, + }, + content: { + display: 'flex', + flexDirection: 'column', + width: '100%', + maxWidth: '96rem', + gap: tokens.spacingVerticalXL, + padding: tokens.spacingVerticalXXL, + marginInline: 'auto', + [NARROW_VIEWPORT_QUERY]: { + padding: `${tokens.spacingVerticalL} ${tokens.spacingHorizontalM}`, + gap: tokens.spacingVerticalL, + }, + }, + backLink: { + display: 'inline-flex', + alignItems: 'center', + alignSelf: 'flex-start', + gap: tokens.spacingHorizontalXS, + minHeight: MINIMUM_TOUCH_TARGET_SIZE, + color: tokens.colorBrandForegroundLink, + textDecorationLine: 'none', + ':hover': { + textDecorationLine: 'underline', + }, + ':focus-visible': { + outline: `2px solid ${tokens.colorStrokeFocus2}`, + outlineOffset: '2px', + }, + }, + header: { + display: 'flex', + alignItems: 'flex-start', + justifyContent: 'space-between', + gap: tokens.spacingHorizontalXL, + [NARROW_VIEWPORT_QUERY]: { + flexDirection: 'column', + alignItems: 'stretch', + }, + }, + headerIdentity: { + display: 'flex', + flexDirection: 'column', + minWidth: 0, + gap: tokens.spacingVerticalXS, + }, + titleRow: { + display: 'flex', + alignItems: 'center', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalS, + }, + runId: { + color: tokens.colorNeutralForeground3, + overflowWrap: 'anywhere', + }, + headerActions: { + display: 'flex', + flexShrink: 0, + gap: tokens.spacingHorizontalS, + [NARROW_VIEWPORT_QUERY]: { + width: '100%', + }, + }, + touchTarget: { + ...mobileTouchTarget, + }, + wideButton: { + [NARROW_VIEWPORT_QUERY]: { + flexGrow: 1, + }, + }, + metadata: { + display: 'grid', + gridTemplateColumns: 'repeat(3, minmax(10rem, 1fr))', + gap: `${tokens.spacingVerticalS} ${tokens.spacingHorizontalXL}`, + paddingTop: tokens.spacingVerticalM, + borderTop: `1px solid ${tokens.colorNeutralStroke2}`, + [NARROW_VIEWPORT_QUERY]: { + gridTemplateColumns: '1fr', + }, + }, + metadataItem: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXXS, + minWidth: 0, + }, + metadataLabel: { + color: tokens.colorNeutralForeground3, + }, + section: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + }, + sectionHeading: { + display: 'flex', + alignItems: 'baseline', + justifyContent: 'space-between', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalM, + }, + sectionHint: { + color: tokens.colorNeutralForeground3, + }, + progressSurface: { + display: 'grid', + gridTemplateColumns: 'minmax(14rem, 2fr) repeat(2, minmax(8rem, 1fr))', + gap: tokens.spacingHorizontalXL, + alignItems: 'center', + padding: tokens.spacingVerticalL, + border: `1px solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusLarge, + backgroundColor: tokens.colorNeutralBackground1, + [NARROW_VIEWPORT_QUERY]: { + gridTemplateColumns: '1fr', + gap: tokens.spacingVerticalM, + }, + }, + progressPrimary: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalS, + minWidth: 0, + }, + progressText: { + display: 'flex', + alignItems: 'baseline', + justifyContent: 'space-between', + gap: tokens.spacingHorizontalM, + }, + metric: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXXS, + }, + metricLabel: { + color: tokens.colorNeutralForeground3, + }, + metricValue: { + fontVariantNumeric: 'tabular-nums', + }, + summaryGrid: { + display: 'grid', + gridTemplateColumns: 'repeat(auto-fit, minmax(15rem, 1fr))', + gap: tokens.spacingHorizontalM, + }, + summaryItem: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalS, + padding: tokens.spacingVerticalL, + borderTop: `1px solid ${tokens.colorNeutralStroke1}`, + backgroundColor: tokens.colorNeutralBackground1, + }, + summaryTitle: { + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + gap: tokens.spacingHorizontalS, + }, + summaryStats: { + display: 'grid', + gridTemplateColumns: 'repeat(3, 1fr)', + gap: tokens.spacingHorizontalS, + }, + summaryStat: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXXS, + }, + tableScroll: { + width: '100%', + overflowX: 'auto', + border: `1px solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusLarge, + backgroundColor: tokens.colorNeutralBackground1, + }, + table: { + minWidth: '64rem', + tableLayout: 'auto', + }, + attemptsTable: { + minWidth: '68rem', + tableLayout: 'auto', + }, + clickableAttemptRow: { + cursor: 'pointer', + ':hover': { + backgroundColor: tokens.colorNeutralBackground1Hover, + }, + ':focus-visible': { + outline: `2px solid ${tokens.colorStrokeFocus2}`, + outlineOffset: '-2px', + }, + }, + nowrap: { + whiteSpace: 'nowrap', + fontVariantNumeric: 'tabular-nums', + }, + preview: { + display: 'block', + maxWidth: '24rem', + overflow: 'hidden', + whiteSpace: 'nowrap', + textOverflow: 'ellipsis', + }, + attackLink: { + display: 'inline-flex', + alignItems: 'center', + justifyContent: 'center', + minWidth: MINIMUM_TOUCH_TARGET_SIZE, + minHeight: MINIMUM_TOUCH_TARGET_SIZE, + textDecorationLine: 'none', + borderRadius: tokens.borderRadiusMedium, + ':hover': { + backgroundColor: tokens.colorSubtleBackgroundHover, + }, + ':focus-visible': { + outline: `2px solid ${tokens.colorStrokeFocus2}`, + outlineOffset: '2px', + }, + }, + objectiveButton: { + maxWidth: '26rem', + justifyContent: 'flex-start', + ...mobileTouchTarget, + }, + emptyState: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + gap: tokens.spacingVerticalS, + minHeight: '8rem', + padding: tokens.spacingVerticalXXL, + color: tokens.colorNeutralForeground3, + textAlign: 'center', + }, + centeredState: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + gap: tokens.spacingVerticalM, + minHeight: '18rem', + textAlign: 'center', + }, + loadingBlock: { + width: 'min(42rem, 100%)', + }, + dialogContent: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + overflowWrap: 'anywhere', + }, + detailGrid: { + display: 'grid', + gridTemplateColumns: 'repeat(2, minmax(0, 1fr))', + gap: `${tokens.spacingVerticalM} ${tokens.spacingHorizontalL}`, + [NARROW_VIEWPORT_QUERY]: { + gridTemplateColumns: '1fr', + }, + }, + objective: { + whiteSpace: 'pre-wrap', + overflowWrap: 'anywhere', + }, + liveStatus: { + position: 'absolute', + width: '1px', + height: '1px', + overflow: 'hidden', + clip: 'rect(0 0 0 0)', + clipPath: 'inset(50%)', + whiteSpace: 'nowrap', + }, +}) diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx new file mode 100644 index 0000000000..4c3c772308 --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx @@ -0,0 +1,359 @@ +import { fireEvent, render, screen, waitFor, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { + MemoryRouter, + Route, + Routes, + useLocation, + useNavigate, +} from 'react-router' + +import { useScenarioRunProgress } from '@/hooks/useScenarioRunProgress' +import { scenariosApi } from '@/services/api' +import type { + ScenarioProgressResult, + ScenarioRunPlan, +} from '@/types' +import { + INITIAL_SCENARIO_RUN_PROGRESS_STATE, + type ScenarioRunProgressState, +} from '@/utils/scenarioRunProgress' + +import ScenarioRunPage from './ScenarioRunPage' + +jest.mock('@/hooks/useScenarioRunProgress', () => ({ + useScenarioRunProgress: jest.fn(), +})) + +jest.mock('@/services/api', () => ({ + scenariosApi: { + cancelRun: jest.fn(), + }, +})) + +const mockUseScenarioRunProgress = useScenarioRunProgress as jest.Mock +const mockCancelRun = scenariosApi.cancelRun as jest.Mock +const mockRetry = jest.fn() +const mockApplyRunSummary = jest.fn() +const SCENARIO_RESULT_ID = '123e4567-e89b-12d3-a456-426614174000' + +const PLAN: ScenarioRunPlan = { + version: 1, + scenario_registry_name: 'test.scenario', + atomic_groups: [{ + id: 'group-1', + atomic_attack_name: 'attack-technique', + display_group: 'Technique One', + technique_eval_hash: 'eval-1', + seed_group_ids: ['seed-1'], + }], + seed_groups: [{ + id: 'seed-1', + objective_sha256: 'sha-1', + objective: 'Reveal the system prompt and all hidden configuration.', + }], +} + +const ATTEMPT: ScenarioProgressResult = { + attack_result_id: 'attack-result-1', + atomic_group_id: 'group-1', + atomic_attack_name: 'attack-technique', + seed_group_id: 'seed-1', + outcome: 'success', + execution_time_ms: 5_000, + timestamp: '2026-01-01T00:00:05Z', + total_retries: 1, + retries: [], +} + +function makeState(overrides: Partial = {}): ScenarioRunProgressState { + return { + ...INITIAL_SCENARIO_RUN_PROGRESS_STATE, + loadStatus: 'ready', + run: { + scenario_result_id: SCENARIO_RESULT_ID, + scenario_name: 'TestScenario', + scenario_registry_name: 'test.scenario', + scenario_version: 1, + status: 'IN_PROGRESS', + created_at: '2026-01-01T00:00:00Z', + }, + plan: PLAN, + planComplete: true, + activeAtomicGroupIds: ['group-1'], + results: [ATTEMPT], + cursor: 'cursor-1', + ...overrides, + } +} + +function mockHookState(state: ScenarioRunProgressState): void { + mockUseScenarioRunProgress.mockReturnValue({ + state, + retry: mockRetry, + applyRunSummary: mockApplyRunSummary, + }) +} + +function AttackRouteProbe() { + const location = useLocation() + const navigate = useNavigate() + return ( +
+ +
+ ) +} + +function renderPage(path = `/scenario-history/${SCENARIO_RESULT_ID}`) { + return render( + + + + } /> + } /> + + + , + ) +} + +describe('ScenarioRunPage', () => { + beforeEach(() => { + jest.clearAllMocks() + mockHookState(makeState()) + }) + + it('renders a live dashboard with accessible progress and semantic tables', () => { + renderPage() + + expect(screen.getByRole('heading', { name: 'test.scenario', level: 1 })).toBeInTheDocument() + expect(screen.getByTestId('run-state-badge')).toHaveTextContent('In progress') + expect(screen.getByRole('progressbar', { name: 'Overall scenario run progress' })).toHaveAttribute( + 'aria-valuetext', + '1 of 1 executable units completed', + ) + expect(screen.getByRole('table', { name: 'Atomic attack groups' })).toBeInTheDocument() + expect(screen.getByRole('table', { name: 'Logical seed groups' })).toBeInTheDocument() + expect(screen.getByRole('table', { name: 'Persisted attack attempts' })).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Cancel run' })).toBeInTheDocument() + expect(screen.queryByRole('columnheader', { name: 'Actions' })).not.toBeInTheDocument() + }) + + it('keeps legacy runs useful without misleading totals, ETA, or a progress bar', () => { + mockHookState(makeState({ planComplete: false })) + + renderPage() + + expect(screen.getByText(/legacy run has no complete persisted execution plan/i)).toBeInTheDocument() + expect(screen.getAllByText(/1 known completed units; planned total unavailable/i)).toHaveLength(2) + expect(screen.queryByRole('progressbar')).not.toBeInTheDocument() + expect(screen.getByText('Progress percentage unavailable')).toBeInTheDocument() + expect(screen.getAllByText('Unavailable').length).toBeGreaterThan(0) + expect(screen.getAllByText('1/total unavailable').length).toBeGreaterThan(0) + expect(screen.queryByText('1/1')).not.toBeInTheDocument() + expect(screen.getByRole('link', { name: 'Open attack attack-result-1' })).toBeInTheDocument() + }) + + it('shows a stale warning and retries from the explicit action', async () => { + const user = userEvent.setup() + mockHookState(makeState({ stale: true, error: 'Network unavailable' })) + + renderPage() + await user.click(screen.getByRole('button', { name: 'Retry' })) + + expect(mockRetry).toHaveBeenCalledTimes(1) + expect(screen.getByText(/showing the last successfully loaded progress/i)).toBeInTheDocument() + }) + + it('cancels after confirmation and immediately applies the returned terminal state', async () => { + const user = userEvent.setup() + const cancelledRun = { + scenario_result_id: 'run-1', + scenario_name: 'TestScenario', + scenario_registry_name: 'test.scenario', + scenario_version: 1, + status: 'CANCELLED', + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:01:00Z', + completed_at: '2026-01-01T00:01:00Z', + techniques_used: [], + total_attacks: 1, + completed_attacks: 1, + objective_achieved_rate: 100, + failed_attacks: [], + attack_retries: [], + total_retries: 0, + labels: {}, + } + mockCancelRun.mockResolvedValueOnce(cancelledRun) + + renderPage() + await user.click(screen.getByRole('button', { name: 'Cancel run' })) + const dialog = screen.getByRole('dialog', { name: 'Cancel this scenario run?' }) + await user.click(within(dialog).getByRole('button', { name: 'Cancel run' })) + + await waitFor(() => expect(mockApplyRunSummary).toHaveBeenCalledWith(cancelledRun)) + expect(mockCancelRun).toHaveBeenCalledWith(SCENARIO_RESULT_ID) + }) + + it('keeps the confirmation open and shows cancel conflicts', async () => { + const user = userEvent.setup() + mockCancelRun.mockRejectedValueOnce(new Error('Cannot cancel a completed run.')) + + renderPage() + await user.click(screen.getByRole('button', { name: 'Cancel run' })) + const dialog = screen.getByRole('dialog', { name: 'Cancel this scenario run?' }) + await user.click(within(dialog).getByRole('button', { name: 'Cancel run' })) + + expect(await within(dialog).findByText('Cannot cancel a completed run.')).toBeInTheDocument() + expect(mockApplyRunSummary).not.toHaveBeenCalled() + }) + + it('shows full objective details and restores focus on close', async () => { + const user = userEvent.setup() + renderPage() + const detailsButton = screen.getByRole('button', { + name: 'View details for attack attempt attack-result-1', + }) + + await user.click(detailsButton) + const dialog = screen.getByRole('dialog', { name: 'Attack attempt details' }) + expect(within(dialog).getByText(PLAN.seed_groups[0].objective)).toBeInTheDocument() + await user.click(within(dialog).getByRole('button', { name: 'Close' })) + + await waitFor(() => expect(detailsButton).toHaveFocus()) + }) + + it('puts the essential attack link in the first column with bounded provenance', () => { + renderPage() + + const attackLink = screen.getByRole('link', { name: 'Open attack attack-result-1' }) + expect(attackLink).toHaveAttribute( + 'href', + `/attacks/attack-result-1?scenarioResultId=${SCENARIO_RESULT_ID}`, + ) + expect(attackLink).toHaveTextContent('attack-result-1') + const attemptsTable = screen.getByRole('table', { name: 'Persisted attack attempts' }) + expect(within(attemptsTable).getByRole('columnheader', { name: 'Attack' })).toBeInTheDocument() + const firstBodyRow = within(attemptsTable).getAllByRole('row')[1] + expect(within(firstBodyRow).getAllByRole('cell')[0]).toContainElement( + attackLink, + ) + }) + + it('navigates from non-interactive row content and browser Back returns to the run', async () => { + const user = userEvent.setup() + renderPage() + + const attemptRow = screen.getByRole('row', { + name: 'Open attack attack-result-1', + }) + await user.click(within(attemptRow).getByText('Technique One')) + + expect(screen.getByTestId('attack-route')).toBeInTheDocument() + expect(screen.getByTestId('attack-route')).toHaveAttribute( + 'data-location', + `/attacks/attack-result-1?scenarioResultId=${SCENARIO_RESULT_ID}`, + ) + + await user.click(screen.getByRole('button', { name: 'Browser back' })) + + expect(screen.getByRole('heading', { name: 'test.scenario', level: 1 })).toBeInTheDocument() + }) + + it('supports Enter and Space row activation', async () => { + const user = userEvent.setup() + renderPage() + const row = screen.getByRole('row', { name: 'Open attack attack-result-1' }) + + row.focus() + await user.keyboard('{Enter}') + expect(screen.getByTestId('attack-route')).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Browser back' })) + + const restoredRow = screen.getByRole('row', { name: 'Open attack attack-result-1' }) + restoredRow.focus() + await user.keyboard(' ') + expect(screen.getByTestId('attack-route')).toBeInTheDocument() + }) + + it('does not hijack modified, non-primary, or nested-control clicks', async () => { + const user = userEvent.setup() + renderPage() + const row = screen.getByRole('row', { name: 'Open attack attack-result-1' }) + + fireEvent.click(row, { ctrlKey: true }) + fireEvent.click(row, { metaKey: true }) + fireEvent.click(row, { shiftKey: true }) + fireEvent.click(row, { altKey: true }) + fireEvent.click(row, { button: 1 }) + expect(screen.queryByTestId('attack-route')).not.toBeInTheDocument() + + await user.click(screen.getByRole('button', { + name: 'View details for attack attempt attack-result-1', + })) + expect(screen.getByRole('dialog', { name: 'Attack attempt details' })).toBeInTheDocument() + expect(screen.queryByTestId('attack-route')).not.toBeInTheDocument() + }) + + it('leaves modified first-column link clicks to native new-tab behavior', () => { + renderPage() + const link = screen.getByRole('link', { name: 'Open attack attack-result-1' }) + const modifiedClick = new MouseEvent('click', { + bubbles: true, + cancelable: true, + ctrlKey: true, + }) + + expect(link.dispatchEvent(modifiedClick)).toBe(true) + expect(modifiedClick.defaultPrevented).toBe(false) + expect(screen.queryByTestId('attack-route')).not.toBeInTheDocument() + }) + + it('renders loading, not-found, and initial error states with accessible recovery', () => { + mockHookState({ ...INITIAL_SCENARIO_RUN_PROGRESS_STATE }) + const { unmount } = renderPage() + expect(screen.getByLabelText('Loading scenario run')).toBeInTheDocument() + unmount() + + mockHookState({ + ...INITIAL_SCENARIO_RUN_PROGRESS_STATE, + loadStatus: 'not-found', + error: 'Run not found', + }) + const notFound = renderPage() + expect(screen.getByRole('heading', { name: 'Scenario run not found' })).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Retry' })).toBeInTheDocument() + notFound.unmount() + + mockHookState({ + ...INITIAL_SCENARIO_RUN_PROGRESS_STATE, + loadStatus: 'error', + error: 'Backend unavailable', + }) + renderPage() + expect(screen.getByRole('heading', { name: 'Unable to load scenario run' })).toBeInTheDocument() + expect(screen.getByText('Backend unavailable')).toBeInTheDocument() + }) + + it('decodes route IDs and does not offer cancellation for terminal runs', () => { + mockHookState(makeState({ + run: { + scenario_result_id: 'run/1', + scenario_name: 'TestScenario', + scenario_registry_name: 'test.scenario', + scenario_version: 1, + status: 'COMPLETED', + created_at: '2026-01-01T00:00:00Z', + completed_at: '2026-01-01T00:01:00Z', + }, + })) + + renderPage('/scenario-history/run%2F1') + + expect(mockUseScenarioRunProgress).toHaveBeenCalledWith('run/1') + expect(screen.queryByRole('button', { name: 'Cancel run' })).not.toBeInTheDocument() + }) +}) diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.tsx new file mode 100644 index 0000000000..3a65a0907d --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioRunPage.tsx @@ -0,0 +1,790 @@ +import { useEffect, useMemo, useRef, useState } from 'react' + +import { + Badge, + Button, + Dialog, + DialogActions, + DialogBody, + DialogContent, + DialogSurface, + DialogTitle, + MessageBar, + MessageBarActions, + MessageBarBody, + mergeClasses, + ProgressBar, + Skeleton, + SkeletonItem, + Table, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, + Text, +} from '@fluentui/react-components' +import { + ArrowLeftRegular, + ArrowSyncRegular, + CheckmarkCircleRegular, + DismissCircleRegular, + ErrorCircleRegular, + EyeRegular, + StopRegular, +} from '@fluentui/react-icons' +import { Link, useNavigate, useParams } from 'react-router' + +import { useScenarioRunProgress } from '@/hooks/useScenarioRunProgress' +import { scenariosApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { + ScenarioProgressResult, + ScenarioRunState, +} from '@/types' +import { + attackRoutePath, + routerPathParamValue, +} from '@/utils/routeParams' +import { + getAtomicGroupRollups, + getElapsedMilliseconds, + getEtaMilliseconds, + getOverallProgress, + getSeedGroupRollups, + getTechniqueRollups, + isTerminalRunState, +} from '@/utils/scenarioRunProgress' + +import { useScenarioRunPageStyles } from './ScenarioRunPage.styles' + +const CLOCK_REFRESH_INTERVAL_MS = 1_000 +const OBJECTIVE_PREVIEW_LENGTH = 96 +const INTERACTIVE_ELEMENT_SELECTOR = 'a, button, input, select, textarea, [role="button"], [role="link"]' + +const RUN_BADGE_COLORS: Record = { + CREATED: 'informative', + IN_PROGRESS: 'brand', + COMPLETED: 'success', + FAILED: 'danger', + CANCELLED: 'warning', +} + +const OUTCOME_BADGE_COLORS: Record = { + success: 'success', + failure: 'danger', + error: 'warning', + undetermined: 'informative', +} + +export default function ScenarioRunPage() { + const { scenarioResultId: encodedId } = useParams<{ scenarioResultId: string }>() + return +} + +interface ScenarioRunPageContentProps { + readonly scenarioResultId: string +} + +function ScenarioRunPageContent({ scenarioResultId }: ScenarioRunPageContentProps) { + const styles = useScenarioRunPageStyles() + const navigate = useNavigate() + const { state, retry, applyRunSummary } = useScenarioRunProgress(scenarioResultId) + const [nowMilliseconds, setNowMilliseconds] = useState(() => Date.now()) + const [cancelDialogOpen, setCancelDialogOpen] = useState(false) + const [cancelling, setCancelling] = useState(false) + const [cancelError, setCancelError] = useState(null) + const [selectedAttempt, setSelectedAttempt] = useState(null) + const detailsTriggerRef = useRef(null) + + const overall = useMemo(() => getOverallProgress(state), [state]) + const techniques = useMemo(() => getTechniqueRollups(state), [state]) + const seedGroups = useMemo(() => getSeedGroupRollups(state), [state]) + const atomicGroups = useMemo(() => getAtomicGroupRollups(state), [state]) + const seedObjectives = useMemo( + () => new Map(state.plan?.seed_groups.map((seed) => [seed.id, seed.objective]) ?? []), + [state.plan], + ) + const atomicGroupNames = useMemo( + () => new Map(atomicGroups.map((group) => [group.id, group.displayGroup])), + [atomicGroups], + ) + + useEffect(() => { + if (!state.run || isTerminalRunState(state.run.status)) { + return + } + const timer = setInterval(() => setNowMilliseconds(Date.now()), CLOCK_REFRESH_INTERVAL_MS) + return () => clearInterval(timer) + }, [state.run]) + + const closeAttemptDetails = (): void => { + setSelectedAttempt(null) + requestAnimationFrame(() => detailsTriggerRef.current?.focus()) + } + + const openAttemptDetails = ( + attempt: ScenarioProgressResult, + trigger: HTMLButtonElement, + ): void => { + detailsTriggerRef.current = trigger + setSelectedAttempt(attempt) + } + + const handleCancel = async (): Promise => { + setCancelling(true) + setCancelError(null) + try { + const run = await scenariosApi.cancelRun(scenarioResultId) + applyRunSummary(run) + setCancelDialogOpen(false) + } catch (error: unknown) { + setCancelError(toApiError(error).detail) + } finally { + setCancelling(false) + } + } + + if (state.loadStatus === 'loading' && !state.run) { + return ( +
+
+ + Back to scenarios + +
+ + +
+ +
+ + + Loading scenario run... +
+
+
+ ) + } + + if (state.loadStatus === 'not-found' && !state.run) { + return ( +
+
+ + Back to scenarios + +
+ + Scenario run not found + {state.error} + +
+
+
+ ) + } + + if (state.loadStatus === 'error' && !state.run) { + return ( +
+
+ + Back to scenarios + +
+ + Unable to load scenario run + {state.error} + +
+
+
+ ) + } + + if (!state.run) { + return null + } + + const run = state.run + const canCancel = run.status === 'CREATED' || run.status === 'IN_PROGRESS' + const elapsed = getElapsedMilliseconds(run, nowMilliseconds) + const eta = getEtaMilliseconds(state, nowMilliseconds) + const progressText = overall.planned === null + ? `${overall.completed} known completed units; planned total unavailable` + : `${overall.completed} of ${overall.planned} executable units completed` + + return ( +
+
+ + Back to scenarios + + +
+
+
+ + {run.scenario_registry_name ?? run.scenario_name} + + + {formatRunState(run.status)} + +
+ {run.scenario_registry_name && run.scenario_registry_name !== run.scenario_name && ( + {run.scenario_name} + )} + + Run ID: {run.scenario_result_id} + +
+ {canCancel && ( +
+ +
+ )} +
+ +
+
+ Scenario version + {run.scenario_version} +
+
+ Created + {formatTimestamp(run.created_at)} +
+
+ Completed + {run.completed_at ? formatTimestamp(run.completed_at) : 'Not yet'} +
+
+ + {state.stale && ( + + + Live updates paused. Showing the last successfully loaded progress. {state.error} + + + + + + )} + + {run.status === 'FAILED' && ( + + + This run ended before all planned executable units completed. Persisted attempts remain available below. + + + )} + + {!state.planComplete && ( + + + This legacy run has no complete persisted execution plan. Known groups and attempts are shown, but planned totals and ETA are unavailable. + + + )} + +
+
+ + Overall progress + + {progressText} +
+
+
+
+ {progressText} + {overall.percent !== null && {overall.percent}%} +
+ {overall.percent !== null ? ( + + ) : ( + Progress percentage unavailable + )} +
+
+ Elapsed + + {formatDuration(elapsed)} + +
+
+ Estimated remaining + + {eta === null ? 'Unavailable' : formatDuration(eta)} + +
+
+ + {isTerminalRunState(run.status) ? `Run ${formatRunState(run.status)}` : ''} + +
+ +
+
+ + Technique summary + + Success is measured over evaluated non-error units. +
+ {techniques.length === 0 ? ( + + ) : ( +
+ {techniques.map((technique) => ( +
+
+ {technique.displayGroup} + {formatSuccess(technique.succeeded, technique.evaluated, technique.successPercent)} +
+ + {technique.atomicAttackNames.join(', ')} + +
+ + + +
+
+ ))} +
+ )} +
+ +
+
+ + Atomic attack groups + + Running groups are listed first. +
+ {atomicGroups.length === 0 ? ( + + ) : ( +
+ + + + Status + Display group + Attack + Completed + Success + Errors + Retries + + + + {atomicGroups.map((group) => ( + + + {group.displayGroup} + {group.atomicAttackName || 'Persisted attack'} + + {formatCompletion(group.completed, group.planned, state.planComplete)} + + + {formatSuccess(group.succeeded, group.evaluated, group.successPercent)} + + {group.errors} + {group.retries} + + ))} + +
+
+ )} +
+ +
+
+ + Logical seed groups + + Aggregated across techniques. +
+ {seedGroups.length === 0 ? ( + + ) : ( +
+ + + + Objective + Completed + Success + Errors + Retries + + + + {seedGroups.map((seed) => ( + + + {objectivePreview(seed.objective, seed.id)} + + + {formatCompletion(seed.completed, seed.planned, state.planComplete)} + + + {formatSuccess(seed.succeeded, seed.evaluated, seed.successPercent)} + + {seed.errors} + {seed.retries} + + ))} + +
+
+ )} +
+ +
+
+ + Persisted attack attempts + + {state.results.length} attempts +
+ {state.results.length === 0 ? ( + + ) : ( +
+ + + + Attack + Outcome + Group + Seed + Objective + Execution + Retries / error + Timestamp + + + + {[...state.results].reverse().map((attempt) => { + const attackDestination = attackRoutePath( + attempt.attack_result_id, + scenarioResultId, + ) + return ( + { + if (!shouldIgnoreAttemptRowClick(event)) { + navigate(attackDestination) + } + }} + onKeyDown={(event) => { + if ( + (event.key === 'Enter' || event.key === ' ') + && !hasActivationModifier(event) + && !isInteractiveTarget(event.target) + ) { + event.preventDefault() + navigate(attackDestination) + } + }} + > + + event.stopPropagation()} + > + + {attempt.attack_result_id} + + + + + + {formatOutcome(attempt.outcome)} + + + {atomicGroupNames.get(attempt.atomic_group_id) ?? attempt.atomic_attack_name} + {attempt.seed_group_id} + + + + {formatDuration(attempt.execution_time_ms)} + + {attempt.outcome === 'error' + ? attempt.error_message ?? attempt.error_type ?? 'Error' + : `${attempt.total_retries} retries`} + + {formatTimestamp(attempt.timestamp)} + + ) + })} + +
+
+ )} +
+
+ + { + if (!cancelling) { + setCancelDialogOpen(data.open) + } + }} + > + + + Cancel this scenario run? + + + In-flight work will be stopped. Attempts already persisted will remain available in this dashboard. + + {cancelError && ( + + {cancelError} + + )} + + + + + + + + + + { + if (!data.open) { + closeAttemptDetails() + } + }} + > + + + Attack attempt details + {selectedAttempt && ( + +
+ Objective + + {seedObjectives.get(selectedAttempt.seed_group_id) ?? 'Objective text unavailable for this legacy attempt.'} + +
+
+ + + + + + + + +
+ {selectedAttempt.outcome === 'error' && ( + + + {selectedAttempt.error_type ? `${selectedAttempt.error_type}: ` : ''} + {selectedAttempt.error_message ?? 'No error detail was persisted.'} + + + )} + + )} + + + + + + +
+ ) +} + +interface MetricProps { + readonly label: string + readonly value: string +} + +function Metric({ label, value }: MetricProps) { + const styles = useScenarioRunPageStyles() + return ( +
+ {label} + {value} +
+ ) +} + +interface EmptyStateProps { + readonly text: string +} + +function EmptyState({ text }: EmptyStateProps) { + const styles = useScenarioRunPageStyles() + return ( +
+ {text} +
+ ) +} + +interface AtomicStatusBadgeProps { + readonly status: 'Running' | 'Pending' | 'Incomplete' | 'Completed' +} + +function AtomicStatusBadge({ status }: AtomicStatusBadgeProps) { + const color = status === 'Running' + ? 'brand' + : status === 'Completed' + ? 'success' + : status === 'Incomplete' + ? 'warning' + : 'informative' + return {status} +} + +function formatRunState(status: ScenarioRunState): string { + return status.toLowerCase().replace('_', ' ').replace(/^\w/, (letter) => letter.toUpperCase()) +} + +function formatOutcome(outcome: ScenarioProgressResult['outcome']): string { + return outcome.replace(/^\w/, (letter) => letter.toUpperCase()) +} + +function statusIcon(status: ScenarioRunState): React.ReactElement { + if (status === 'COMPLETED') { + return + } + if (status === 'FAILED') { + return + } + if (status === 'CANCELLED') { + return + } + return +} + +function formatTimestamp(timestamp: string): string { + const date = new Date(timestamp) + if (Number.isNaN(date.getTime())) { + return 'Unavailable' + } + return date.toLocaleString(undefined, { + month: 'short', + day: 'numeric', + year: 'numeric', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }) +} + +function formatDuration(milliseconds: number): string { + if (!Number.isFinite(milliseconds) || milliseconds < 0) { + return 'Unavailable' + } + const totalSeconds = Math.floor(milliseconds / 1_000) + const hours = Math.floor(totalSeconds / 3_600) + const minutes = Math.floor((totalSeconds % 3_600) / 60) + const seconds = totalSeconds % 60 + if (hours > 0) { + return `${hours}h ${minutes}m` + } + if (minutes > 0) { + return `${minutes}m ${seconds}s` + } + return `${seconds}s` +} + +function formatSuccess(succeeded: number, evaluated: number, percent: number | null): string { + return percent === null ? `${succeeded}/${evaluated} —` : `${succeeded}/${evaluated} (${percent}%)` +} + +function formatCompletion(completed: number, planned: number, planComplete: boolean): string { + return planComplete ? `${completed}/${planned}` : `${completed}/total unavailable` +} + +function objectivePreview(objective: string | null, fallbackId: string): string { + if (!objective) { + return `Objective unavailable (${fallbackId})` + } + if (objective.length <= OBJECTIVE_PREVIEW_LENGTH) { + return objective + } + return `${objective.slice(0, OBJECTIVE_PREVIEW_LENGTH - 1)}…` +} + +function shouldIgnoreAttemptRowClick(event: React.MouseEvent): boolean { + return event.button !== 0 + || hasActivationModifier(event) + || isInteractiveTarget(event.target) +} + +function hasActivationModifier( + event: Pick + | Pick, +): boolean { + return event.altKey || event.ctrlKey || event.metaKey || event.shiftKey +} + +function isInteractiveTarget(target: EventTarget): boolean { + return target instanceof Element && target.closest(INTERACTIVE_ELEMENT_SELECTOR) !== null +} diff --git a/frontend/src/components/Scenarios/ScenarioRunStarted.styles.ts b/frontend/src/components/Scenarios/ScenarioRunStarted.styles.ts deleted file mode 100644 index a405d923c1..0000000000 --- a/frontend/src/components/Scenarios/ScenarioRunStarted.styles.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { makeStyles, tokens } from '@fluentui/react-components' -import { NARROW_VIEWPORT_QUERY } from '@/styles/touchTargets' - -export const useScenarioRunStartedStyles = makeStyles({ - root: { - display: 'flex', - flexDirection: 'column', - height: '100%', - width: '100%', - minWidth: 0, - maxWidth: '40rem', - padding: tokens.spacingVerticalXXL, - overflowX: 'hidden', - overflowY: 'auto', - backgroundColor: tokens.colorNeutralBackground2, - gap: tokens.spacingVerticalM, - [NARROW_VIEWPORT_QUERY]: { - padding: `${tokens.spacingVerticalL} ${tokens.spacingHorizontalM}`, - }, - }, - backLink: { - alignSelf: 'flex-start', - }, - hint: { - color: tokens.colorNeutralForeground3, - }, - section: { - display: 'flex', - flexDirection: 'column', - gap: tokens.spacingVerticalXS, - padding: tokens.spacingVerticalL, - border: `1px solid ${tokens.colorNeutralStroke2}`, - borderRadius: tokens.borderRadiusLarge, - backgroundColor: tokens.colorNeutralBackground1, - }, - centeredState: { - display: 'flex', - flexDirection: 'column', - alignItems: 'center', - justifyContent: 'center', - gap: tokens.spacingVerticalM, - padding: tokens.spacingVerticalXXL, - }, -}) diff --git a/frontend/src/components/Scenarios/ScenarioRunStarted.test.tsx b/frontend/src/components/Scenarios/ScenarioRunStarted.test.tsx deleted file mode 100644 index 567483cf45..0000000000 --- a/frontend/src/components/Scenarios/ScenarioRunStarted.test.tsx +++ /dev/null @@ -1,139 +0,0 @@ -import { render, screen, waitFor } from '@testing-library/react' -import userEvent from '@testing-library/user-event' -import { FluentProvider, webLightTheme } from '@fluentui/react-components' -import { MemoryRouter, Route, Routes } from 'react-router' - -import { scenariosApi } from '@/services/api' - -import ScenarioRunStarted from './ScenarioRunStarted' - -jest.mock('@/services/api', () => ({ - scenariosApi: { - getRun: jest.fn(), - }, -})) - -const mockGetRun = scenariosApi.getRun as jest.Mock - -function renderShell(path: string, state?: unknown) { - return render( - - - - } /> - - - , - ) -} - -function makeRunSummary(overrides: Partial> = {}) { - return { - scenario_result_id: 'sr-1', - scenario_name: 'foundry.red_team_agent', - scenario_version: 0, - status: 'IN_PROGRESS', - created_at: '2026-02-15T00:00:00Z', - updated_at: '2026-02-15T00:00:00Z', - techniques_used: [], - total_attacks: 0, - completed_attacks: 0, - objective_achieved_rate: 0, - failed_attacks: [], - attack_retries: [], - total_retries: 0, - labels: {}, - ...overrides, - } -} - -describe('ScenarioRunStarted', () => { - beforeEach(() => { - jest.clearAllMocks() - }) - - it('renders an accessible heading and the scenario result id', async () => { - mockGetRun.mockResolvedValueOnce(makeRunSummary()) - - renderShell('/scenario-history/sr-1') - - expect(screen.getByRole('heading', { name: 'Scenario run started' })).toBeInTheDocument() - expect(screen.getByText('sr-1')).toBeInTheDocument() - await screen.findByTestId('run-status') - }) - - it('decodes a percent-encoded scenario result id from the URL and fetches by the decoded id', async () => { - mockGetRun.mockResolvedValueOnce(makeRunSummary({ scenario_result_id: 'sr/1' })) - - renderShell('/scenario-history/sr%2F1') - - await waitFor(() => expect(mockGetRun).toHaveBeenCalledWith('sr/1')) - expect(screen.getByText('sr/1')).toBeInTheDocument() - }) - - it('shows a loading state before the fetch resolves', () => { - mockGetRun.mockReturnValue(new Promise(() => {})) - renderShell('/scenario-history/sr-1') - expect(screen.getByText('Loading run status...')).toBeInTheDocument() - }) - - it('shows the run status once loaded', async () => { - mockGetRun.mockResolvedValueOnce(makeRunSummary({ status: 'COMPLETED' })) - - renderShell('/scenario-history/sr-1') - - expect(await screen.findByTestId('run-status-value')).toHaveTextContent('COMPLETED') - }) - - it('shows an error state with retry on failure, and recovers after retry', async () => { - const user = userEvent.setup() - mockGetRun - .mockRejectedValueOnce(new Error('boom')) - .mockResolvedValueOnce(makeRunSummary()) - - renderShell('/scenario-history/sr-1') - - expect(await screen.findByTestId('run-error')).toBeInTheDocument() - expect(screen.getByText('boom')).toBeInTheDocument() - - await user.click(screen.getByTestId('retry-btn')) - - expect(await screen.findByTestId('run-status')).toBeInTheDocument() - expect(mockGetRun).toHaveBeenCalledTimes(2) - }) - - it('does not poll — it fetches the run exactly once per mount', async () => { - mockGetRun.mockResolvedValueOnce(makeRunSummary()) - renderShell('/scenario-history/sr-1') - - await screen.findByTestId('run-status') - await new Promise((resolve) => setTimeout(resolve, 50)) - - expect(mockGetRun).toHaveBeenCalledTimes(1) - }) - - it('shows the scenario name from location state before the fetch resolves', () => { - mockGetRun.mockReturnValue(new Promise(() => {})) - - renderShell('/scenario-history/sr-1', { scenarioName: 'foundry.red_team_agent' }) - - // The loading spinner is showing, but the run id itself is already visible from the URL. - expect(screen.getByText('sr-1')).toBeInTheDocument() - }) - - it('works as a direct deep link with no location state at all', async () => { - mockGetRun.mockResolvedValueOnce(makeRunSummary()) - - renderShell('/scenario-history/sr-1') - - expect(await screen.findByTestId('run-status')).toBeInTheDocument() - expect(screen.getByText(/foundry\.red_team_agent/)).toBeInTheDocument() - }) - - it('links back to the scenario catalog', async () => { - mockGetRun.mockResolvedValueOnce(makeRunSummary()) - renderShell('/scenario-history/sr-1') - - expect(screen.getByRole('link', { name: /back to scenarios/i })).toHaveAttribute('href', '/scenarios') - }) -}) diff --git a/frontend/src/components/Scenarios/ScenarioRunStarted.tsx b/frontend/src/components/Scenarios/ScenarioRunStarted.tsx deleted file mode 100644 index 0c5930b358..0000000000 --- a/frontend/src/components/Scenarios/ScenarioRunStarted.tsx +++ /dev/null @@ -1,125 +0,0 @@ -import { useEffect, useState } from 'react' - -import { Button, MessageBar, MessageBarBody, Spinner, Text } from '@fluentui/react-components' -import { ArrowLeftRegular, ArrowSyncRegular } from '@fluentui/react-icons' -import { Link, useLocation, useParams } from 'react-router' - -import { scenariosApi } from '@/services/api' -import { toApiError } from '@/services/errors' -import type { ScenarioRunSummary } from '@/types' -import { routerPathParamValue } from '@/utils/routeParams' - -import { useScenarioRunStartedStyles } from './ScenarioRunStarted.styles' - -type LoadStatus = 'loading' | 'success' | 'error' - -/** Optional state forwarded by the launch form's `navigate()` call — shows a scenario name before the fetch resolves. */ -interface ScenarioRunLocationState { - scenarioName?: string -} - -/** - * Minimal acknowledgement shell shown right after launching a scenario run. - * - * Fetches the run once (no polling) to confirm it exists and show its - * current status; it intentionally does not aggregate or poll progress — - * that belongs to a full run-history view, out of scope here. - */ -export default function ScenarioRunStarted() { - const { scenarioResultId: encodedId } = useParams<{ scenarioResultId: string }>() - // Keying on the raw URL param forces a full remount (and state reset to the - // initial "loading" values) if the route ever navigates from one run id - // directly to another, without needing to reset state from inside an effect. - return -} - -interface ScenarioRunStartedContentProps { - encodedId: string | undefined -} - -function ScenarioRunStartedContent({ encodedId }: ScenarioRunStartedContentProps) { - const styles = useScenarioRunStartedStyles() - const location = useLocation() - const locationState = location.state as ScenarioRunLocationState | null - const decodedId = routerPathParamValue(encodedId) - - const [run, setRun] = useState(null) - const [status, setStatus] = useState('loading') - const [error, setError] = useState(null) - const [refetchCount, setRefetchCount] = useState(0) - - useEffect(() => { - let cancelled = false - scenariosApi - .getRun(decodedId) - .then((data) => { - if (cancelled) return - setRun(data) - setStatus('success') - setError(null) - }) - .catch((err: unknown) => { - if (cancelled) return - setRun(null) - setStatus('error') - setError(toApiError(err).detail) - }) - return () => { - cancelled = true - } - }, [decodedId, refetchCount]) - - const handleRetry = (): void => { - setStatus('loading') - setError(null) - setRefetchCount((count) => count + 1) - } - - const displayScenarioName = run?.scenario_name ?? locationState?.scenarioName - - return ( -
- - Back to scenarios - - - Scenario run started - - Run ID: {decodedId} - - - {status === 'loading' && ( -
- -
- )} - - {status === 'error' && ( -
- - {error} - - -
- )} - - {status === 'success' && run && ( -
- {displayScenarioName && ( - Scenario: {displayScenarioName} - )} - - Status: {run.status} - -
- )} -
- ) -} diff --git a/frontend/src/hooks/useScenarioRunProgress.test.tsx b/frontend/src/hooks/useScenarioRunProgress.test.tsx new file mode 100644 index 0000000000..d0a7791a08 --- /dev/null +++ b/frontend/src/hooks/useScenarioRunProgress.test.tsx @@ -0,0 +1,434 @@ +import { act, renderHook, waitFor } from '@testing-library/react' + +import { scenariosApi } from '@/services/api' +import type { + ScenarioProgressResult, + ScenarioRunProgress, + ScenarioRunSummary, +} from '@/types' + +import { + SCENARIO_RUN_POLL_INTERVAL_MS, + useScenarioRunProgress, +} from './useScenarioRunProgress' + +jest.mock('@/services/api', () => ({ + scenariosApi: { + getRunProgress: jest.fn(), + }, +})) + +const mockGetRunProgress = scenariosApi.getRunProgress as jest.Mock + +function makeResult(id: string): ScenarioProgressResult { + return { + attack_result_id: id, + atomic_group_id: 'group-1', + atomic_attack_name: 'attack-1', + seed_group_id: 'seed-1', + outcome: 'success', + execution_time_ms: 1_000, + timestamp: '2026-01-01T00:00:01Z', + total_retries: 0, + retries: [], + } +} + +function makePage(overrides: Partial = {}): ScenarioRunProgress { + return { + run: { + scenario_result_id: 'run-1', + scenario_name: 'TestScenario', + scenario_registry_name: 'test.scenario', + scenario_version: 1, + status: 'IN_PROGRESS', + created_at: '2026-01-01T00:00:00Z', + }, + plan: { + version: 1, + scenario_registry_name: 'test.scenario', + atomic_groups: [], + seed_groups: [], + }, + reset: false, + active_atomic_group_ids: [], + results: [], + next_cursor: null, + has_more: false, + plan_complete: true, + ...overrides, + } +} + +function makeSummary(overrides: Partial = {}): ScenarioRunSummary { + return { + scenario_result_id: 'run-1', + scenario_name: 'TestScenario', + scenario_version: 1, + status: 'IN_PROGRESS', + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:01Z', + techniques_used: [], + total_attacks: 1, + completed_attacks: 0, + objective_achieved_rate: 0, + failed_attacks: [], + attack_retries: [], + total_retries: 0, + labels: {}, + ...overrides, + } +} + +describe('useScenarioRunProgress', () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + afterEach(() => { + jest.useRealTimers() + }) + + it('loads the plan and immediately drains all available delta pages', async () => { + mockGetRunProgress + .mockResolvedValueOnce(makePage({ + results: [makeResult('attempt-1')], + next_cursor: 'cursor-1', + has_more: true, + })) + .mockResolvedValueOnce(makePage({ + plan: null, + results: [makeResult('attempt-2')], + next_cursor: 'cursor-2', + has_more: false, + })) + + const { result, unmount } = renderHook(() => useScenarioRunProgress('run-1')) + + await waitFor(() => expect(result.current.state.results).toHaveLength(2)) + expect(mockGetRunProgress).toHaveBeenNthCalledWith( + 1, + 'run-1', + { since: undefined, limit: 500 }, + expect.any(AbortSignal), + ) + expect(mockGetRunProgress).toHaveBeenNthCalledWith( + 2, + 'run-1', + { since: 'cursor-1', limit: 500 }, + expect.any(AbortSignal), + ) + unmount() + }) + + it('polls after 2.5 seconds from the last successfully applied cursor', async () => { + jest.useFakeTimers() + mockGetRunProgress + .mockResolvedValueOnce(makePage({ next_cursor: 'cursor-1' })) + .mockResolvedValueOnce(makePage({ plan: null, next_cursor: 'cursor-2' })) + + const { unmount } = renderHook(() => useScenarioRunProgress('run-1')) + await act(async () => Promise.resolve()) + + await act(async () => { + await jest.advanceTimersByTimeAsync(SCENARIO_RUN_POLL_INTERVAL_MS) + }) + + expect(mockGetRunProgress).toHaveBeenNthCalledWith( + 2, + 'run-1', + { since: 'cursor-1', limit: 500 }, + expect.any(AbortSignal), + ) + unmount() + }) + + it('isolates cursors when the run ID changes while preserving same-run polling', async () => { + jest.useFakeTimers() + mockGetRunProgress + .mockResolvedValueOnce(makePage({ next_cursor: 'run-a-cursor' })) + .mockResolvedValueOnce(makePage({ + run: { ...makePage().run, scenario_result_id: 'run-b' }, + next_cursor: 'run-b-cursor', + })) + .mockResolvedValueOnce(makePage({ + run: { ...makePage().run, scenario_result_id: 'run-b' }, + plan: null, + next_cursor: 'run-b-next-cursor', + })) + + const { rerender, unmount } = renderHook( + ({ runId }) => useScenarioRunProgress(runId), + { initialProps: { runId: 'run-a' } }, + ) + await act(async () => Promise.resolve()) + + rerender({ runId: 'run-b' }) + await act(async () => Promise.resolve()) + + expect(mockGetRunProgress).toHaveBeenNthCalledWith( + 2, + 'run-b', + { since: undefined, limit: 500 }, + expect.any(AbortSignal), + ) + + await act(async () => { + await jest.advanceTimersByTimeAsync(SCENARIO_RUN_POLL_INTERVAL_MS) + }) + expect(mockGetRunProgress).toHaveBeenNthCalledWith( + 3, + 'run-b', + { since: 'run-b-cursor', limit: 500 }, + expect.any(AbortSignal), + ) + unmount() + }) + + it('transitions a queued run to active progress on a later poll', async () => { + jest.useFakeTimers() + mockGetRunProgress + .mockResolvedValueOnce(makePage({ + run: { ...makePage().run, status: 'QUEUED', queue_position: 1 }, + next_cursor: 'cursor-1', + })) + .mockResolvedValueOnce(makePage({ + run: { ...makePage().run, status: 'IN_PROGRESS', queue_position: null }, + plan: null, + next_cursor: 'cursor-1', + })) + + const { result, unmount } = renderHook(() => useScenarioRunProgress('run-1')) + await waitFor(() => expect(result.current.state.run?.status).toBe('QUEUED')) + await act(async () => { + await jest.advanceTimersByTimeAsync(SCENARIO_RUN_POLL_INTERVAL_MS) + }) + + expect(result.current.state.run?.status).toBe('IN_PROGRESS') + unmount() + }) + + it('does not overlap polls while a request remains in flight', async () => { + jest.useFakeTimers() + let resolvePoll: ((page: ScenarioRunProgress) => void) | undefined + mockGetRunProgress + .mockResolvedValueOnce(makePage({ next_cursor: 'cursor-1' })) + .mockImplementationOnce(() => new Promise((resolve) => { + resolvePoll = resolve + })) + + const { unmount } = renderHook(() => useScenarioRunProgress('run-1')) + await act(async () => Promise.resolve()) + await act(async () => { + await jest.advanceTimersByTimeAsync(SCENARIO_RUN_POLL_INTERVAL_MS * 4) + }) + + expect(mockGetRunProgress).toHaveBeenCalledTimes(2) + await act(async () => { + resolvePoll?.(makePage({ plan: null, next_cursor: 'cursor-2' })) + }) + unmount() + }) + + it('stops permanently when a terminal page is received', async () => { + jest.useFakeTimers() + mockGetRunProgress.mockResolvedValueOnce(makePage({ + run: { ...makePage().run, status: 'COMPLETED', completed_at: '2026-01-01T00:01:00Z' }, + })) + + const { unmount } = renderHook(() => useScenarioRunProgress('run-1')) + await act(async () => Promise.resolve()) + await act(async () => { + await jest.advanceTimersByTimeAsync(SCENARIO_RUN_POLL_INTERVAL_MS * 3) + }) + + expect(mockGetRunProgress).toHaveBeenCalledTimes(1) + unmount() + }) + + it('aborts a stale request when the route ID changes', async () => { + const signals: AbortSignal[] = [] + mockGetRunProgress.mockImplementation( + (_runId: string, _params: unknown, signal: AbortSignal) => { + signals.push(signal) + return new Promise(() => {}) + }, + ) + + const { rerender, unmount } = renderHook( + ({ runId }) => useScenarioRunProgress(runId), + { initialProps: { runId: 'run-1' } }, + ) + await waitFor(() => expect(signals).toHaveLength(1)) + + rerender({ runId: 'run-2' }) + + expect(signals[0].aborted).toBe(true) + await waitFor(() => expect(signals).toHaveLength(2)) + unmount() + expect(signals[1].aborted).toBe(true) + }) + + it('treats a blank run ID as not found without issuing a request', async () => { + const { result } = renderHook(() => useScenarioRunProgress(' ')) + + await waitFor(() => expect(result.current.state.loadStatus).toBe('not-found')) + expect(mockGetRunProgress).not.toHaveBeenCalled() + }) + + it('treats an HTTP 404 as not found', async () => { + mockGetRunProgress.mockRejectedValueOnce({ + isAxiosError: true, + response: { + status: 404, + data: { detail: 'Scenario run not found.' }, + }, + }) + + const { result } = renderHook(() => useScenarioRunProgress('missing-run')) + + await waitFor(() => expect(result.current.state.loadStatus).toBe('not-found')) + expect(result.current.state.error).toBe('Scenario run not found.') + }) + + it('ignores a stale page that resolves after the run ID changes', async () => { + let resolveOldRequest: ((page: ScenarioRunProgress) => void) | undefined + mockGetRunProgress.mockImplementation((runId: string) => { + if (runId === 'run-1') { + return new Promise((resolve) => { + resolveOldRequest = resolve + }) + } + return Promise.resolve(makePage({ + run: { + ...makePage().run, + scenario_result_id: 'run-2', + }, + })) + }) + + const { result, rerender, unmount } = renderHook( + ({ runId }) => useScenarioRunProgress(runId), + { initialProps: { runId: 'run-1' } }, + ) + await waitFor(() => expect(mockGetRunProgress).toHaveBeenCalledTimes(1)) + rerender({ runId: 'run-2' }) + await waitFor(() => expect(result.current.state.run?.scenario_result_id).toBe('run-2')) + + await act(async () => { + resolveOldRequest?.(makePage()) + }) + expect(result.current.state.run?.scenario_result_id).toBe('run-2') + unmount() + }) + + it('ignores a stale failure after the run ID changes', async () => { + let rejectOldRequest: ((reason?: unknown) => void) | undefined + mockGetRunProgress.mockImplementation((runId: string) => { + if (runId === 'run-1') { + return new Promise((_resolve, reject) => { + rejectOldRequest = reject + }) + } + return Promise.resolve(makePage({ + run: { + ...makePage().run, + scenario_result_id: 'run-2', + }, + })) + }) + + const { result, rerender, unmount } = renderHook( + ({ runId }) => useScenarioRunProgress(runId), + { initialProps: { runId: 'run-1' } }, + ) + await waitFor(() => expect(mockGetRunProgress).toHaveBeenCalledTimes(1)) + rerender({ runId: 'run-2' }) + await waitFor(() => expect(result.current.state.run?.scenario_result_id).toBe('run-2')) + + await act(async () => { + rejectOldRequest?.(new Error('late failure')) + }) + expect(result.current.state.error).toBeNull() + unmount() + }) + + it('retries from the last good cursor after a transient failure', async () => { + jest.useFakeTimers() + mockGetRunProgress + .mockResolvedValueOnce(makePage({ next_cursor: 'cursor-1' })) + .mockRejectedValueOnce(new Error('temporary failure')) + .mockResolvedValueOnce(makePage({ plan: null, next_cursor: 'cursor-2' })) + + const { result, unmount } = renderHook(() => useScenarioRunProgress('run-1')) + await act(async () => Promise.resolve()) + await act(async () => { + await jest.advanceTimersByTimeAsync(SCENARIO_RUN_POLL_INTERVAL_MS) + }) + expect(result.current.state.stale).toBe(true) + + act(() => result.current.retry()) + await act(async () => Promise.resolve()) + + expect(mockGetRunProgress).toHaveBeenNthCalledWith( + 3, + 'run-1', + { since: 'cursor-1', limit: 500 }, + expect.any(AbortSignal), + ) + unmount() + }) + + it('fetches final persisted deltas after applying a cancellation summary', async () => { + mockGetRunProgress + .mockResolvedValueOnce(makePage({ next_cursor: 'cursor-1' })) + .mockResolvedValueOnce(makePage({ + run: { + ...makePage().run, + status: 'CANCELLED', + completed_at: '2026-01-01T00:00:02Z', + }, + plan: null, + results: [makeResult('final-attempt')], + next_cursor: 'cursor-2', + })) + + const { result, unmount } = renderHook(() => useScenarioRunProgress('run-1')) + await waitFor(() => expect(mockGetRunProgress).toHaveBeenCalledTimes(1)) + + act(() => { + result.current.applyRunSummary(makeSummary({ + status: 'CANCELLED', + updated_at: '2026-01-01T00:00:02Z', + completed_attacks: 1, + objective_achieved_rate: 100, + })) + }) + + await waitFor(() => expect(result.current.state.results).toEqual([makeResult('final-attempt')])) + expect(mockGetRunProgress).toHaveBeenLastCalledWith( + 'run-1', + { since: 'cursor-1', limit: 500 }, + expect.any(AbortSignal), + ) + unmount() + }) + + it('applies a nonterminal run summary without forcing a catch-up request', async () => { + mockGetRunProgress.mockResolvedValueOnce(makePage({ next_cursor: 'cursor-1' })) + const { result, unmount } = renderHook(() => useScenarioRunProgress('run-1')) + await waitFor(() => expect(result.current.state.cursor).toBe('cursor-1')) + mockGetRunProgress.mockClear() + + act(() => { + result.current.applyRunSummary(makeSummary({ + status: 'IN_PROGRESS', + updated_at: '2026-01-01T00:00:02Z', + })) + }) + + expect(result.current.state.run?.status).toBe('IN_PROGRESS') + expect(mockGetRunProgress).not.toHaveBeenCalled() + unmount() + }) +}) diff --git a/frontend/src/hooks/useScenarioRunProgress.tsx b/frontend/src/hooks/useScenarioRunProgress.tsx new file mode 100644 index 0000000000..58c3a7bd2b --- /dev/null +++ b/frontend/src/hooks/useScenarioRunProgress.tsx @@ -0,0 +1,129 @@ +import { useCallback, useEffect, useReducer, useRef, useState } from 'react' + +import { scenariosApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { ScenarioRunSummary } from '@/types' +import { + INITIAL_SCENARIO_RUN_PROGRESS_STATE, + isTerminalRunState, + scenarioRunProgressReducer, + type ScenarioRunProgressState, +} from '@/utils/scenarioRunProgress' + +export const SCENARIO_RUN_POLL_INTERVAL_MS = 2_500 +const PROGRESS_PAGE_LIMIT = 500 + +export interface UseScenarioRunProgressResult { + readonly state: ScenarioRunProgressState + readonly retry: () => void + readonly applyRunSummary: (run: ScenarioRunSummary) => void +} + +export function useScenarioRunProgress(scenarioResultId: string): UseScenarioRunProgressResult { + const [state, dispatch] = useReducer( + scenarioRunProgressReducer, + INITIAL_SCENARIO_RUN_PROGRESS_STATE, + ) + const [retryEpoch, setRetryEpoch] = useState(0) + const cursorRef = useRef(null) + const cursorScenarioResultIdRef = useRef(scenarioResultId) + const abortControllerRef = useRef(null) + const timerRef = useRef | null>(null) + const pollingStoppedRef = useRef(false) + + useEffect(() => { + if (cursorScenarioResultIdRef.current !== scenarioResultId) { + cursorScenarioResultIdRef.current = scenarioResultId + cursorRef.current = null + } + + let active = true + pollingStoppedRef.current = false + + const clearPollTimer = (): void => { + if (timerRef.current !== null) { + clearTimeout(timerRef.current) + timerRef.current = null + } + } + + const fetchPage = async (since: string | null): Promise => { + if (!active || pollingStoppedRef.current) { + return + } + const controller = new AbortController() + abortControllerRef.current = controller + try { + const page = await scenariosApi.getRunProgress( + scenarioResultId, + { since: since ?? undefined, limit: PROGRESS_PAGE_LIMIT }, + controller.signal, + ) + if (!active || pollingStoppedRef.current) { + return + } + + const appliedCursor = page.next_cursor ?? since + cursorRef.current = appliedCursor + dispatch({ type: 'apply-page', page, fresh: since === null }) + + if (page.has_more) { + await fetchPage(appliedCursor) + return + } + if (isTerminalRunState(page.run.status)) { + pollingStoppedRef.current = true + return + } + clearPollTimer() + timerRef.current = setTimeout(() => { + timerRef.current = null + void fetchPage(cursorRef.current) + }, SCENARIO_RUN_POLL_INTERVAL_MS) + } catch (error: unknown) { + if (!active || controller.signal.aborted) { + return + } + const apiError = toApiError(error) + dispatch({ + type: 'request-failed', + message: apiError.detail, + notFound: apiError.status === 404, + }) + } + } + + if (!scenarioResultId.trim()) { + dispatch({ + type: 'request-failed', + message: 'The scenario run ID in this URL is missing or invalid.', + notFound: true, + }) + } else { + void fetchPage(cursorRef.current) + } + + return () => { + active = false + clearPollTimer() + abortControllerRef.current?.abort() + abortControllerRef.current = null + } + }, [scenarioResultId, retryEpoch]) + + const retry = useCallback((): void => { + dispatch({ type: 'retry' }) + pollingStoppedRef.current = false + setRetryEpoch((epoch) => epoch + 1) + }, []) + + const applyRunSummary = useCallback((run: ScenarioRunSummary): void => { + dispatch({ type: 'apply-run-summary', run }) + if (isTerminalRunState(run.status)) { + pollingStoppedRef.current = false + setRetryEpoch((epoch) => epoch + 1) + } + }, []) + + return { state, retry, applyRunSummary } +} diff --git a/frontend/src/services/api.test.ts b/frontend/src/services/api.test.ts index f0d2729f3c..c874be497e 100644 --- a/frontend/src/services/api.test.ts +++ b/frontend/src/services/api.test.ts @@ -653,11 +653,37 @@ describe("api service", () => { }; (apiClient.get as jest.Mock).mockResolvedValueOnce(mockResponse); - await scenariosApi.getRunProgress("sr-1", { since: "cursor-1", limit: 50 }); + const controller = new AbortController(); + await scenariosApi.getRunProgress( + "sr-1", + { since: "cursor-1", limit: 50 }, + controller.signal, + ); expect(apiClient.get).toHaveBeenCalledWith("/scenarios/runs/sr-1/progress", { params: { since: "cursor-1", limit: 50 }, + signal: controller.signal, }); }); + + it("cancels a scenario run by id", async () => { + const mockResponse = { + data: { + scenario_result_id: "sr-1", + status: "CANCELLED", + }, + }; + const controller = new AbortController(); + (apiClient.post as jest.Mock).mockResolvedValueOnce(mockResponse); + + const result = await scenariosApi.cancelRun("sr/1", controller.signal); + + expect(apiClient.post).toHaveBeenCalledWith( + "/scenarios/runs/sr%2F1/cancel", + undefined, + { signal: controller.signal }, + ); + expect(result.status).toBe("CANCELLED"); + }); }); }); diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 770e08c1b5..551d61f261 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -394,10 +394,20 @@ export const scenariosApi = { getRunProgress: async ( scenarioResultId: string, params?: { since?: string; limit?: number }, + signal?: AbortSignal, ): Promise => { const response = await apiClient.get( `/scenarios/runs/${encodeURIComponent(scenarioResultId)}/progress`, - { params }, + { params, signal }, + ) + return response.data + }, + + cancelRun: async (scenarioResultId: string, signal?: AbortSignal): Promise => { + const response = await apiClient.post( + `/scenarios/runs/${encodeURIComponent(scenarioResultId)}/cancel`, + undefined, + { signal }, ) return response.data }, diff --git a/frontend/src/utils/routeParams.test.ts b/frontend/src/utils/routeParams.test.ts index 87bdfa2369..ec23355b50 100644 --- a/frontend/src/utils/routeParams.test.ts +++ b/frontend/src/utils/routeParams.test.ts @@ -1,6 +1,18 @@ -import { routerPathParamValue } from './routeParams' +import { + attackConversationRoutePath, + attackRoutePath, + routerPathParamValue, + scenarioRunProvenance, + scenarioRunRoutePath, +} from './routeParams' + +const SCENARIO_RESULT_ID = '123e4567-e89b-12d3-a456-426614174000' describe('routerPathParamValue', () => { + it('returns an empty value for a missing route parameter', () => { + expect(routerPathParamValue(undefined)).toBe('') + }) + it('restores slashes re-escaped by React Router', () => { expect(routerPathParamValue('foundry%2Fred_team_agent')).toBe('foundry/red_team_agent') }) @@ -10,3 +22,41 @@ describe('routerPathParamValue', () => { expect(routerPathParamValue('%zz')).toBe('%zz') }) }) + +describe('scenario run provenance routes', () => { + it('reads one canonical UUID and ignores unrelated query values', () => { + const params = new URLSearchParams(`tab=messages&scenarioResultId=${SCENARIO_RESULT_ID}`) + + expect(scenarioRunProvenance(params)).toBe(SCENARIO_RESULT_ID) + }) + + it.each([ + '', + 'scenarioResultId=run-1', + 'scenarioResultId=https%3A%2F%2Fevil.example%2Freturn', + `scenarioResultId=${'a'.repeat(100)}`, + `scenarioResultId=${SCENARIO_RESULT_ID}&scenarioResultId=${SCENARIO_RESULT_ID}`, + ])('rejects missing, unsafe, or ambiguous provenance: %s', (query: string) => { + expect(scenarioRunProvenance(new URLSearchParams(query))).toBeNull() + }) + + it('builds encoded attack and conversation destinations with bounded provenance', () => { + expect(attackRoutePath('attack/1', SCENARIO_RESULT_ID)).toBe( + `/attacks/attack%2F1?scenarioResultId=${SCENARIO_RESULT_ID}`, + ) + expect(attackConversationRoutePath('attack/1', 'conversation/1', SCENARIO_RESULT_ID)).toBe( + `/attacks/attack%2F1/conversations/conversation%2F1?scenarioResultId=${SCENARIO_RESULT_ID}`, + ) + }) + + it('omits invalid provenance instead of serializing it', () => { + expect(attackRoutePath('attack-1', 'https://evil.example')).toBe('/attacks/attack-1') + expect(attackConversationRoutePath('attack-1', 'conversation-1', 'run-1')).toBe( + '/attacks/attack-1/conversations/conversation-1', + ) + }) + + it('builds an encoded scenario-run route from a trusted persisted ID', () => { + expect(scenarioRunRoutePath('run/1')).toBe('/scenario-history/run%2F1') + }) +}) diff --git a/frontend/src/utils/routeParams.ts b/frontend/src/utils/routeParams.ts index b028a8b16b..f2c7127a41 100644 --- a/frontend/src/utils/routeParams.ts +++ b/frontend/src/utils/routeParams.ts @@ -1,3 +1,6 @@ +const SCENARIO_RESULT_ID_QUERY_KEY = 'scenarioResultId' +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + /** * Returns the original value represented by a React Router path parameter. * @@ -9,3 +12,50 @@ export function routerPathParamValue(value: string | undefined): string { return (value ?? '').replace(/%2F/gi, '/') } + +/** Returns one validated scenario-run provenance UUID from a route query. */ +export function scenarioRunProvenance(searchParams: URLSearchParams): string | null { + const values = searchParams.getAll(SCENARIO_RESULT_ID_QUERY_KEY) + if (values.length !== 1 || !UUID_PATTERN.test(values[0])) { + return null + } + return values[0] +} + +/** Builds an attack-detail route with optional bounded scenario-run provenance. */ +export function attackRoutePath( + attackResultId: string, + scenarioResultId?: string | null, +): string { + return appendScenarioRunProvenance( + `/attacks/${encodeURIComponent(attackResultId)}`, + scenarioResultId, + ) +} + +/** Builds an attack-conversation route with optional bounded scenario-run provenance. */ +export function attackConversationRoutePath( + attackResultId: string, + conversationId: string, + scenarioResultId?: string | null, +): string { + return appendScenarioRunProvenance( + `/attacks/${encodeURIComponent(attackResultId)}/conversations/${encodeURIComponent(conversationId)}`, + scenarioResultId, + ) +} + +/** Builds the route for one scenario run. Callers must pass a trusted persisted ID. */ +export function scenarioRunRoutePath(scenarioResultId: string): string { + return `/scenario-history/${encodeURIComponent(scenarioResultId)}` +} + +function appendScenarioRunProvenance(path: string, scenarioResultId?: string | null): string { + if (!scenarioResultId || !UUID_PATTERN.test(scenarioResultId)) { + return path + } + const searchParams = new URLSearchParams({ + [SCENARIO_RESULT_ID_QUERY_KEY]: scenarioResultId, + }) + return `${path}?${searchParams.toString()}` +} diff --git a/frontend/src/utils/scenarioRunProgress.test.ts b/frontend/src/utils/scenarioRunProgress.test.ts new file mode 100644 index 0000000000..414b35bd17 --- /dev/null +++ b/frontend/src/utils/scenarioRunProgress.test.ts @@ -0,0 +1,272 @@ +import type { + ScenarioProgressResult, + ScenarioRunPlan, + ScenarioRunProgress, +} from '@/types' + +import { + INITIAL_SCENARIO_RUN_PROGRESS_STATE, + getAtomicGroupRollups, + getElapsedMilliseconds, + getEtaMilliseconds, + getOverallProgress, + getSeedGroupRollups, + getTechniqueRollups, + scenarioRunProgressReducer, + type ScenarioRunProgressState, +} from './scenarioRunProgress' + +const PLAN: ScenarioRunPlan = { + version: 1, + scenario_registry_name: 'test.scenario', + atomic_groups: [ + { + id: 'group-a', + atomic_attack_name: 'attack-a', + display_group: 'Technique A', + technique_eval_hash: 'eval-a', + seed_group_ids: ['seed-1', 'seed-2'], + }, + { + id: 'group-b', + atomic_attack_name: 'attack-b', + display_group: 'Technique B', + technique_eval_hash: 'eval-b', + seed_group_ids: ['seed-1'], + }, + ], + seed_groups: [ + { id: 'seed-1', objective_sha256: 'sha-1', objective: 'First objective' }, + { id: 'seed-2', objective_sha256: 'sha-2', objective: 'Second objective' }, + ], +} + +function makeResult( + id: string, + atomicGroupId: string, + seedGroupId: string, + outcome: ScenarioProgressResult['outcome'], + minute: number, + overrides: Partial = {}, +): ScenarioProgressResult { + return { + attack_result_id: id, + atomic_group_id: atomicGroupId, + atomic_attack_name: atomicGroupId === 'group-a' ? 'attack-a' : 'attack-b', + seed_group_id: seedGroupId, + outcome, + execution_time_ms: 1_000, + timestamp: `2026-01-01T00:${String(minute).padStart(2, '0')}:00Z`, + total_retries: 0, + retries: [], + ...overrides, + } +} + +function makePage(overrides: Partial = {}): ScenarioRunProgress { + return { + run: { + scenario_result_id: 'run-1', + scenario_name: 'TestScenario', + scenario_registry_name: 'test.scenario', + scenario_version: 1, + status: 'IN_PROGRESS', + created_at: '2026-01-01T00:00:00Z', + }, + plan: PLAN, + reset: false, + active_atomic_group_ids: [], + results: [], + next_cursor: 'cursor-1', + has_more: false, + plan_complete: true, + ...overrides, + } +} + +function readyState(results: ScenarioProgressResult[]): ScenarioRunProgressState { + return scenarioRunProgressReducer(INITIAL_SCENARIO_RUN_PROGRESS_STATE, { + type: 'apply-page', + page: makePage({ results }), + fresh: true, + }) +} + +describe('scenarioRunProgressReducer', () => { + it('merges duplicated pages idempotently by attack result id', () => { + const result = makeResult('attempt-1', 'group-a', 'seed-1', 'success', 1) + const first = readyState([result]) + const duplicate = scenarioRunProgressReducer(first, { + type: 'apply-page', + page: makePage({ plan: null, results: [result], next_cursor: 'cursor-1' }), + fresh: false, + }) + + expect(duplicate.results).toEqual([result]) + expect(duplicate.cursor).toBe('cursor-1') + }) + + it('atomically resets prior results when the server requests reset', () => { + const first = readyState([makeResult('old', 'group-a', 'seed-1', 'success', 1)]) + const replacement = makeResult('new', 'group-b', 'seed-1', 'failure', 2) + const reset = scenarioRunProgressReducer(first, { + type: 'apply-page', + page: makePage({ reset: true, results: [replacement], next_cursor: 'cursor-2' }), + fresh: false, + }) + + expect(reset.results).toEqual([replacement]) + expect(reset.cursor).toBe('cursor-2') + }) + + it('retains last-good data and marks it stale after a transient failure', () => { + const first = readyState([makeResult('attempt-1', 'group-a', 'seed-1', 'success', 1)]) + const failed = scenarioRunProgressReducer(first, { + type: 'request-failed', + message: 'Network unavailable', + notFound: false, + }) + + expect(failed.results).toHaveLength(1) + expect(failed.loadStatus).toBe('ready') + expect(failed.stale).toBe(true) + expect(failed.error).toBe('Network unavailable') + }) +}) + +describe('scenario run progress calculations', () => { + it('counts executable units once across multiple attempts and completes from the latest non-error outcome', () => { + const state = readyState([ + makeResult('error-1', 'group-a', 'seed-1', 'error', 1), + makeResult('failure-1', 'group-a', 'seed-1', 'failure', 2), + makeResult('success-1', 'group-a', 'seed-1', 'success', 3), + makeResult('error-2', 'group-a', 'seed-1', 'error', 4), + ]) + + expect(getOverallProgress(state)).toEqual({ completed: 1, planned: 3, percent: 33 }) + expect(getTechniqueRollups(state)[0]).toMatchObject({ + completed: 1, + planned: 2, + succeeded: 1, + evaluated: 1, + errors: 2, + retries: 3, + }) + }) + + it('keeps an error-only unit attempted but incomplete', () => { + const state = readyState([ + makeResult('error-1', 'group-a', 'seed-1', 'error', 1, { total_retries: 2 }), + ]) + + expect(getOverallProgress(state).completed).toBe(0) + expect(getAtomicGroupRollups(state)[0]).toMatchObject({ + completed: 0, + errors: 1, + retries: 2, + status: 'Pending', + }) + }) + + it('does not infer a planned total or percentage for legacy runs', () => { + const state = { + ...readyState([makeResult('attempt-1', 'group-a', 'seed-1', 'success', 1)]), + planComplete: false, + } + + expect(getOverallProgress(state)).toEqual({ completed: 1, planned: null, percent: null }) + expect(getEtaMilliseconds(state, Date.parse('2026-01-01T00:10:00Z'))).toBeNull() + }) + + it('calculates technique and seed rollups across techniques', () => { + const state = readyState([ + makeResult('a-1', 'group-a', 'seed-1', 'success', 1), + makeResult('a-2', 'group-a', 'seed-2', 'failure', 2), + makeResult('b-1', 'group-b', 'seed-1', 'failure', 3), + ]) + + expect(getTechniqueRollups(state)).toEqual([ + expect.objectContaining({ + displayGroup: 'Technique A', + completed: 2, + planned: 2, + succeeded: 1, + evaluated: 2, + successPercent: 50, + }), + expect.objectContaining({ + displayGroup: 'Technique B', + completed: 1, + planned: 1, + succeeded: 0, + evaluated: 1, + successPercent: 0, + }), + ]) + expect(getSeedGroupRollups(state)[0]).toMatchObject({ + id: 'seed-1', + completed: 2, + planned: 2, + succeeded: 1, + evaluated: 2, + successPercent: 50, + }) + }) + + it('sorts atomic states and lets active IDs win while a run is nonterminal', () => { + const state = { + ...readyState([ + makeResult('a-1', 'group-a', 'seed-1', 'success', 1), + makeResult('a-2', 'group-a', 'seed-2', 'failure', 2), + ]), + activeAtomicGroupIds: ['group-a'], + } + + expect(getAtomicGroupRollups(state).map((group) => [group.id, group.status])).toEqual([ + ['group-a', 'Running'], + ['group-b', 'Pending'], + ]) + }) + + it('marks unfinished groups incomplete in terminal runs', () => { + const state = { + ...readyState([makeResult('a-1', 'group-a', 'seed-1', 'success', 1)]), + run: { ...makePage().run, status: 'FAILED' as const, completed_at: '2026-01-01T00:05:00Z' }, + } + + expect(getAtomicGroupRollups(state).map((group) => [group.id, group.status])).toEqual([ + ['group-a', 'Incomplete'], + ['group-b', 'Incomplete'], + ]) + }) + + it('uses now for active elapsed time and completed_at for terminal elapsed time', () => { + const active = makePage().run + expect(getElapsedMilliseconds(active, Date.parse('2026-01-01T00:05:00Z'))).toBe(300_000) + + const terminal = { + ...active, + status: 'COMPLETED' as const, + completed_at: '2026-01-01T00:03:00Z', + } + expect(getElapsedMilliseconds(terminal, Date.parse('2026-01-01T00:05:00Z'))).toBe(180_000) + }) + + it('calculates ETA from observed wall-clock completion rate and hides unsafe estimates', () => { + const state = readyState([makeResult('a-1', 'group-a', 'seed-1', 'success', 1)]) + expect(getEtaMilliseconds(state, Date.parse('2026-01-01T00:02:00Z'))).toBe(240_000) + + expect(getEtaMilliseconds( + { ...state, results: [] }, + Date.parse('2026-01-01T00:02:00Z'), + )).toBeNull() + const run = state.run + expect(run).not.toBeNull() + if (run) { + expect(getEtaMilliseconds( + { ...state, run: { ...run, status: 'COMPLETED' } }, + Date.parse('2026-01-01T00:02:00Z'), + )).toBeNull() + } + }) +}) diff --git a/frontend/src/utils/scenarioRunProgress.ts b/frontend/src/utils/scenarioRunProgress.ts new file mode 100644 index 0000000000..3b8d3fbe23 --- /dev/null +++ b/frontend/src/utils/scenarioRunProgress.ts @@ -0,0 +1,452 @@ +import type { + ScenarioProgressHeader, + ScenarioProgressResult, + ScenarioRunPlan, + ScenarioRunPlanAtomicGroup, + ScenarioRunState, + ScenarioRunSummary, +} from '@/types' + +export type ScenarioRunLoadStatus = 'loading' | 'ready' | 'not-found' | 'error' +export type AtomicGroupStatus = 'Running' | 'Pending' | 'Incomplete' | 'Completed' + +export interface ScenarioRunProgressState { + readonly loadStatus: ScenarioRunLoadStatus + readonly run: ScenarioProgressHeader | null + readonly plan: ScenarioRunPlan | null + readonly planComplete: boolean + readonly activeAtomicGroupIds: string[] + readonly results: ScenarioProgressResult[] + readonly cursor: string | null + readonly hasMore: boolean + readonly error: string | null + readonly stale: boolean +} + +export type ScenarioRunProgressAction = + | { readonly type: 'apply-page'; readonly page: import('@/types').ScenarioRunProgress; readonly fresh: boolean } + | { readonly type: 'request-failed'; readonly message: string; readonly notFound: boolean } + | { readonly type: 'retry' } + | { readonly type: 'apply-run-summary'; readonly run: ScenarioRunSummary } + +export interface OverallProgress { + readonly completed: number + readonly planned: number | null + readonly percent: number | null +} + +export interface Rollup { + readonly completed: number + readonly planned: number + readonly succeeded: number + readonly evaluated: number + readonly successPercent: number | null + readonly errors: number + readonly retries: number +} + +export interface TechniqueRollup extends Rollup { + readonly id: string + readonly displayGroup: string + readonly atomicAttackNames: string[] +} + +export interface SeedGroupRollup extends Rollup { + readonly id: string + readonly objective: string | null +} + +export interface AtomicGroupRollup extends Rollup { + readonly id: string + readonly atomicAttackName: string + readonly displayGroup: string + readonly status: AtomicGroupStatus +} + +interface UnitAttempts { + readonly atomicGroupId: string + readonly seedGroupId: string + readonly attempts: ScenarioProgressResult[] + readonly latestAttempt: ScenarioProgressResult + readonly latestNonError: ScenarioProgressResult | null +} + +const TERMINAL_STATES: ReadonlySet = new Set(['COMPLETED', 'FAILED', 'CANCELLED']) +const ATOMIC_STATUS_ORDER: Record = { + Running: 0, + Pending: 1, + Incomplete: 2, + Completed: 3, +} + +export const INITIAL_SCENARIO_RUN_PROGRESS_STATE: ScenarioRunProgressState = { + loadStatus: 'loading', + run: null, + plan: null, + planComplete: false, + activeAtomicGroupIds: [], + results: [], + cursor: null, + hasMore: false, + error: null, + stale: false, +} + +export function isTerminalRunState(status: ScenarioRunState): boolean { + return TERMINAL_STATES.has(status) +} + +export function scenarioRunProgressReducer( + state: ScenarioRunProgressState, + action: ScenarioRunProgressAction, +): ScenarioRunProgressState { + if (action.type === 'request-failed') { + const hasGoodData = state.run !== null + return { + ...state, + loadStatus: action.notFound && !hasGoodData ? 'not-found' : hasGoodData ? 'ready' : 'error', + error: action.message, + stale: hasGoodData, + hasMore: false, + } + } + + if (action.type === 'retry') { + return { + ...state, + loadStatus: state.run ? 'ready' : 'loading', + error: null, + stale: false, + } + } + + if (action.type === 'apply-run-summary') { + return { + ...state, + loadStatus: 'ready', + run: { + scenario_result_id: action.run.scenario_result_id, + scenario_name: action.run.scenario_name, + scenario_registry_name: action.run.scenario_registry_name, + scenario_version: action.run.scenario_version, + status: action.run.status, + created_at: action.run.created_at, + completed_at: action.run.completed_at, + }, + activeAtomicGroupIds: [], + error: null, + stale: false, + hasMore: false, + } + } + + const shouldReset = action.fresh || action.page.reset || action.page.plan !== null + const resultsById = new Map() + if (!shouldReset) { + for (const result of state.results) { + resultsById.set(result.attack_result_id, result) + } + } + for (const result of action.page.results) { + resultsById.set(result.attack_result_id, result) + } + + const results = [...resultsById.values()].sort(compareAttempts) + return { + loadStatus: 'ready', + run: action.page.run, + plan: action.page.plan ?? (shouldReset ? null : state.plan), + planComplete: action.page.plan_complete, + activeAtomicGroupIds: [...new Set(action.page.active_atomic_group_ids)], + results, + cursor: action.page.next_cursor ?? state.cursor, + hasMore: action.page.has_more, + error: null, + stale: false, + } +} + +export function getOverallProgress(state: ScenarioRunProgressState): OverallProgress { + const units = buildUnitAttempts(state.results) + const completed = [...units.values()].filter((unit) => unit.latestNonError !== null).length + if (!state.planComplete || !state.plan) { + return { completed, planned: null, percent: null } + } + + const planned = state.plan.atomic_groups.reduce( + (total, group) => total + new Set(group.seed_group_ids).size, + 0, + ) + const plannedKeys = buildPlannedUnitKeys(state.plan.atomic_groups) + const plannedCompleted = [...units.entries()].filter( + ([key, unit]) => plannedKeys.has(key) && unit.latestNonError !== null, + ).length + return { + completed: plannedCompleted, + planned, + percent: planned > 0 ? boundedPercent(plannedCompleted, planned) : 0, + } +} + +export function getElapsedMilliseconds( + run: ScenarioProgressHeader, + nowMilliseconds: number, +): number { + const created = Date.parse(run.created_at) + const terminalEnd = run.completed_at ? Date.parse(run.completed_at) : Number.NaN + const end = isTerminalRunState(run.status) && Number.isFinite(terminalEnd) + ? terminalEnd + : nowMilliseconds + if (!Number.isFinite(created) || !Number.isFinite(end)) { + return 0 + } + return Math.max(0, end - created) +} + +export function getEtaMilliseconds( + state: ScenarioRunProgressState, + nowMilliseconds: number, +): number | null { + if (!state.run || !state.planComplete || isTerminalRunState(state.run.status)) { + return null + } + const progress = getOverallProgress(state) + if (progress.planned === null || progress.planned <= 0 || progress.completed <= 0) { + return null + } + const remaining = Math.max(0, progress.planned - progress.completed) + if (remaining === 0) { + return 0 + } + const elapsed = getElapsedMilliseconds(state.run, nowMilliseconds) + if (elapsed <= 0) { + return null + } + const estimate = (elapsed / progress.completed) * remaining + return Number.isFinite(estimate) && estimate >= 0 ? estimate : null +} + +export function getTechniqueRollups(state: ScenarioRunProgressState): TechniqueRollup[] { + const groupMetadata = buildGroupMetadata(state) + const units = buildUnitAttempts(state.results) + const rollups = new Map() + + for (const group of groupMetadata.values()) { + const existing = rollups.get(group.display_group) + const base = existing ?? { + id: group.display_group, + displayGroup: group.display_group, + atomicAttackNames: [], + completed: 0, + planned: 0, + succeeded: 0, + evaluated: 0, + successPercent: null, + errors: 0, + retries: 0, + } + const groupRollup = aggregateGroup(group.id, group.seed_group_ids, units) + rollups.set(group.display_group, { + ...base, + atomicAttackNames: [...new Set([...base.atomicAttackNames, group.atomic_attack_name])], + completed: base.completed + groupRollup.completed, + planned: base.planned + groupRollup.planned, + succeeded: base.succeeded + groupRollup.succeeded, + evaluated: base.evaluated + groupRollup.evaluated, + successPercent: null, + errors: base.errors + groupRollup.errors, + retries: base.retries + groupRollup.retries, + }) + } + + return [...rollups.values()] + .map((rollup) => ({ + ...rollup, + successPercent: rollup.evaluated > 0 ? boundedPercent(rollup.succeeded, rollup.evaluated) : null, + })) + .sort((left, right) => left.displayGroup.localeCompare(right.displayGroup)) +} + +export function getSeedGroupRollups(state: ScenarioRunProgressState): SeedGroupRollup[] { + const groups = buildGroupMetadata(state) + const units = buildUnitAttempts(state.results) + const objectives = new Map(state.plan?.seed_groups.map((seed) => [seed.id, seed.objective]) ?? []) + const seedIds = new Set(objectives.keys()) + for (const group of groups.values()) { + for (const seedId of group.seed_group_ids) { + seedIds.add(seedId) + } + } + + return [...seedIds].map((seedId) => { + const relevantGroups = [...groups.values()].filter((group) => group.seed_group_ids.includes(seedId)) + const relevantUnits = relevantGroups + .map((group) => units.get(unitKey(group.id, seedId))) + .filter((unit): unit is UnitAttempts => unit !== undefined) + const rollup = aggregateUnits(relevantUnits, relevantGroups.length) + return { id: seedId, objective: objectives.get(seedId) ?? null, ...rollup } + }).sort((left, right) => { + const leftLabel = left.objective ?? left.id + const rightLabel = right.objective ?? right.id + return leftLabel.localeCompare(rightLabel) + }) +} + +export function getAtomicGroupRollups(state: ScenarioRunProgressState): AtomicGroupRollup[] { + const groups = buildGroupMetadata(state) + const units = buildUnitAttempts(state.results) + const terminal = state.run ? isTerminalRunState(state.run.status) : false + const activeIds = new Set(state.activeAtomicGroupIds) + + return [...groups.values()].map((group) => { + const rollup = aggregateGroup(group.id, group.seed_group_ids, units) + let status: AtomicGroupStatus + if (!terminal && activeIds.has(group.id)) { + status = 'Running' + } else if (rollup.completed >= rollup.planned && rollup.planned > 0) { + status = 'Completed' + } else if (terminal) { + status = 'Incomplete' + } else { + status = 'Pending' + } + return { + id: group.id, + atomicAttackName: group.atomic_attack_name, + displayGroup: group.display_group, + status, + ...rollup, + } + }).sort((left, right) => { + const statusDifference = ATOMIC_STATUS_ORDER[left.status] - ATOMIC_STATUS_ORDER[right.status] + if (statusDifference !== 0) { + return statusDifference + } + return left.displayGroup.localeCompare(right.displayGroup) + || left.atomicAttackName.localeCompare(right.atomicAttackName) + }) +} + +function buildGroupMetadata(state: ScenarioRunProgressState): Map { + const groups = new Map() + for (const group of state.plan?.atomic_groups ?? []) { + groups.set(group.id, { ...group, seed_group_ids: [...new Set(group.seed_group_ids)] }) + } + for (const result of state.results) { + const existing = groups.get(result.atomic_group_id) + if (existing) { + if (!existing.seed_group_ids.includes(result.seed_group_id)) { + groups.set(existing.id, { + ...existing, + seed_group_ids: [...existing.seed_group_ids, result.seed_group_id], + }) + } + continue + } + groups.set(result.atomic_group_id, { + id: result.atomic_group_id, + atomic_attack_name: result.atomic_attack_name, + display_group: result.atomic_attack_name || 'Persisted attack group', + technique_eval_hash: '', + seed_group_ids: [result.seed_group_id], + }) + } + return groups +} + +function buildUnitAttempts(results: ScenarioProgressResult[]): Map { + const grouped = new Map() + for (const result of results) { + const key = unitKey(result.atomic_group_id, result.seed_group_id) + const attempts = grouped.get(key) ?? [] + attempts.push(result) + grouped.set(key, attempts) + } + + const units = new Map() + for (const [key, unsortedAttempts] of grouped) { + const attempts = [...unsortedAttempts].sort(compareAttempts) + const latestAttempt = attempts[attempts.length - 1] + let latestNonError: ScenarioProgressResult | null = null + for (const attempt of attempts) { + if (attempt.outcome !== 'error') { + latestNonError = attempt + } + } + units.set(key, { + atomicGroupId: latestAttempt.atomic_group_id, + seedGroupId: latestAttempt.seed_group_id, + attempts, + latestAttempt, + latestNonError, + }) + } + return units +} + +function aggregateGroup( + atomicGroupId: string, + seedGroupIds: string[], + units: Map, +): Rollup { + const relevantUnits = [...new Set(seedGroupIds)] + .map((seedGroupId) => units.get(unitKey(atomicGroupId, seedGroupId))) + .filter((unit): unit is UnitAttempts => unit !== undefined) + return aggregateUnits(relevantUnits, new Set(seedGroupIds).size) +} + +function aggregateUnits(units: UnitAttempts[], planned: number): Rollup { + let completed = 0 + let succeeded = 0 + let errors = 0 + let retries = 0 + for (const unit of units) { + if (unit.latestNonError) { + completed += 1 + if (unit.latestNonError.outcome === 'success') { + succeeded += 1 + } + } + errors += unit.attempts.filter((attempt) => attempt.outcome === 'error').length + retries += Math.max(0, unit.attempts.length - 1) + retries += unit.attempts.reduce((total, attempt) => total + Math.max(0, attempt.total_retries), 0) + } + return { + completed, + planned, + succeeded, + evaluated: completed, + successPercent: completed > 0 ? boundedPercent(succeeded, completed) : null, + errors, + retries, + } +} + +function buildPlannedUnitKeys(groups: ScenarioRunPlanAtomicGroup[]): Set { + const keys = new Set() + for (const group of groups) { + for (const seedGroupId of group.seed_group_ids) { + keys.add(unitKey(group.id, seedGroupId)) + } + } + return keys +} + +function unitKey(atomicGroupId: string, seedGroupId: string): string { + return `${atomicGroupId}\u0000${seedGroupId}` +} + +function compareAttempts(left: ScenarioProgressResult, right: ScenarioProgressResult): number { + const timestampDifference = Date.parse(left.timestamp) - Date.parse(right.timestamp) + if (Number.isFinite(timestampDifference) && timestampDifference !== 0) { + return timestampDifference + } + return left.attack_result_id.localeCompare(right.attack_result_id) +} + +function boundedPercent(numerator: number, denominator: number): number { + if (denominator <= 0) { + return 0 + } + return Math.min(100, Math.max(0, Math.round((numerator / denominator) * 100))) +} diff --git a/pyrit/models/__init__.py b/pyrit/models/__init__.py index 67b420de17..c73ce8f804 100644 --- a/pyrit/models/__init__.py +++ b/pyrit/models/__init__.py @@ -108,6 +108,7 @@ ScenarioProgressResult, ScenarioRunPlan, ScenarioRunPlanAtomicGroup, + ScenarioRunPlanGroupKind, ScenarioRunPlanSeedGroup, ScenarioRunProgress, ) @@ -240,6 +241,7 @@ "ScenarioProgressResult", "ScenarioRunPlan", "ScenarioRunPlanAtomicGroup", + "ScenarioRunPlanGroupKind", "ScenarioRunPlanSeedGroup", "ScenarioRunProgress", "Seed", diff --git a/pyrit/models/scenario_progress.py b/pyrit/models/scenario_progress.py index 89fc6888c3..6ef887f09b 100644 --- a/pyrit/models/scenario_progress.py +++ b/pyrit/models/scenario_progress.py @@ -4,6 +4,7 @@ """Canonical models for durable scenario run plans and incremental progress.""" from datetime import datetime +from enum import Enum from typing import Any, Literal from pydantic import AwareDatetime, BaseModel, Field, model_validator @@ -17,6 +18,14 @@ SCENARIO_RUN_PLAN_VERSION = 1 +class ScenarioRunPlanGroupKind(str, Enum): + """Semantic kind of a planned scenario progress group.""" + + __slots__ = () + + ATTACK = "attack" + + class ScenarioRunPlanSeedGroup(BaseModel): """A de-duplicated logical seed group in a scenario run plan.""" @@ -33,6 +42,7 @@ class ScenarioRunPlanAtomicGroup(BaseModel): display_group: str technique_eval_hash: str seed_group_ids: list[str] + group_kind: ScenarioRunPlanGroupKind | None = None class ScenarioRunPlan(BaseModel): diff --git a/pyrit/scenario/core/scenario.py b/pyrit/scenario/core/scenario.py index 750bfc6405..75f236e627 100644 --- a/pyrit/scenario/core/scenario.py +++ b/pyrit/scenario/core/scenario.py @@ -15,7 +15,7 @@ from collections.abc import Sequence from enum import Enum from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar, final +from typing import TYPE_CHECKING, Any, ClassVar, Literal, final try: # Built-in on Python 3.11+. Fall back to the ``exceptiongroup`` backport on 3.10 @@ -45,6 +45,7 @@ ScenarioResult, ScenarioRunPlan, ScenarioRunPlanAtomicGroup, + ScenarioRunPlanGroupKind, ScenarioRunPlanSeedGroup, ScenarioRunSizeComponent, ScenarioRunSizeEstimateStatus, @@ -58,7 +59,11 @@ from pyrit.registry import ScorerRegistry from pyrit.registry.resolution import resolve_declared_params, resolve_reference_value from pyrit.scenario.core.atomic_attack import AtomicAttack -from pyrit.scenario.core.dataset_configuration import DatasetAttackConfiguration, read_only_dataset_resolution +from pyrit.scenario.core.dataset_configuration import ( + CompoundDatasetAttackConfiguration, + DatasetAttackConfiguration, + read_only_dataset_resolution, +) from pyrit.scenario.core.scenario_context import ScenarioContext from pyrit.scenario.core.scenario_target_defaults import get_default_scorer_target from pyrit.scenario.core.scenario_technique import ScenarioTechnique @@ -138,6 +143,10 @@ class Scenario(ABC): #: Whether the default estimator must mirror matrix-builder seed compatibility. RUN_SIZE_USES_FACTORY_COMPATIBILITY: ClassVar[bool] = False + #: How a generic dataset-size run override is interpreted. ``None`` derives the + #: standard behavior from the default configuration. + DATASET_SIZE_LIMIT_OVERRIDE_SCOPE: ClassVar[Literal["per_dataset", "combined", "unsupported"] | None] = None + def __init_subclass__(cls, **kwargs: Any) -> None: """ Enforce the keyword-only constructor contract on subclasses. @@ -259,6 +268,19 @@ def __init__( # before _build_atomic_attacks_async is awaited so overrides can read it. self._include_baseline: bool = False + def get_dataset_size_limit_override_scope(self) -> Literal["per_dataset", "combined", "unsupported"]: + """ + Return how this scenario interprets a generic dataset-size run override. + + Returns: + Literal: The explicit override scope exposed through the scenario catalog. + """ + if self.DATASET_SIZE_LIMIT_OVERRIDE_SCOPE is not None: + return self.DATASET_SIZE_LIMIT_OVERRIDE_SCOPE + if isinstance(self._default_dataset_config, CompoundDatasetAttackConfiguration): + return "per_dataset" + return "per_dataset" if len(self._default_dataset_config.dataset_names) <= 1 else "combined" + @property def name(self) -> str: """The name of the scenario.""" @@ -717,7 +739,7 @@ async def _resolve_dataset_groups_for_estimate_async( selected_count = len(selected_groups.get(name, [])) selection_note = None if selected_count != logical_count: - selection_note = f"The default selection uses {selected_count} of {logical_count} logical seed groups." + selection_note = f"The default selection uses {selected_count} of {logical_count} available objectives." datasets.append( ScenarioDatasetSummary( name=name, @@ -879,7 +901,7 @@ async def initialize_async(self) -> None: self._apply_persisted_objectives(stored_result=stored_result) reconstructed_plan = self._build_run_plan() metadata = dict(stored_result.metadata) - metadata[SCENARIO_RUN_PLAN_METADATA_KEY] = reconstructed_plan.model_dump(mode="json") + metadata[SCENARIO_RUN_PLAN_METADATA_KEY] = reconstructed_plan.model_dump(mode="json", exclude_none=True) self._memory.update_scenario_metadata( scenario_result_id=self._scenario_result_id, metadata=metadata, @@ -935,7 +957,7 @@ def _build_initial_scenario_metadata(self) -> dict[str, Any]: seen.add(sha) hashes.append(sha) metadata["objective_hashes"] = hashes - metadata[SCENARIO_RUN_PLAN_METADATA_KEY] = self._build_run_plan().model_dump(mode="json") + metadata[SCENARIO_RUN_PLAN_METADATA_KEY] = self._build_run_plan().model_dump(mode="json", exclude_none=True) return metadata def _build_run_plan(self) -> ScenarioRunPlan: @@ -973,6 +995,11 @@ def _build_run_plan(self) -> ScenarioRunPlan: display_group=atomic_attack.display_group, technique_eval_hash=technique_eval_hash, seed_group_ids=seed_group_ids, + group_kind=getattr( + atomic_attack, + "_progress_group_kind", + ScenarioRunPlanGroupKind.ATTACK, + ), ) ) return ScenarioRunPlan(