From 717260391679e34da3261ebe5bb60272cb12c8c5 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Wed, 12 Aug 2026 02:46:24 -0700 Subject: [PATCH] FEAT: Add scenario run history Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 5d02c2d5-b499-4f78-a04d-03bffa750817 --- frontend/e2e/scenario-history.spec.ts | 588 ++++++++++++++++++ frontend/src/App.test.tsx | 36 +- frontend/src/App.tsx | 50 +- .../History/ScenarioHistory.styles.ts | 120 ++++ .../History/ScenarioHistory.test.tsx | 278 +++++++++ .../components/History/ScenarioHistory.tsx | 486 +++++++++++++++ .../History/scenarioHistoryFilters.test.ts | 57 ++ .../History/scenarioHistoryFilters.ts | 59 ++ .../Scenarios/ScenarioRunPage.test.tsx | 29 + .../components/Scenarios/ScenarioRunPage.tsx | 103 ++- .../components/Sidebar/Navigation.test.tsx | 21 +- .../src/components/Sidebar/Navigation.tsx | 21 +- frontend/src/services/api.test.ts | 26 + frontend/src/services/api.ts | 22 +- frontend/src/types/index.ts | 28 +- frontend/src/utils/scenarioRunProgress.ts | 6 + pyrit/backend/models/scenarios.py | 4 + pyrit/backend/routes/labels.py | 8 +- pyrit/backend/routes/scenarios.py | 57 +- .../backend/services/scenario_run_service.py | 519 +++++++++++++++- pyrit/cli/api_client.py | 24 +- pyrit/memory/__init__.py | 13 +- .../8d1e3f5a7b9c_index_scenario_history.py | 35 ++ pyrit/memory/azure_sql_memory.py | 90 ++- pyrit/memory/memory_interface.py | 277 ++++++++- pyrit/memory/memory_models.py | 5 +- pyrit/memory/sqlite_memory.py | 75 ++- pyrit/models/catalog/scenario.py | 29 + pyrit/models/scenario_progress.py | 10 + pyrit/scenario/scenarios/airt/jailbreak.py | 19 +- tests/unit/backend/test_api_routes.py | 27 +- .../unit/backend/test_scenario_run_routes.py | 80 ++- .../unit/backend/test_scenario_run_service.py | 287 ++++++++- tests/unit/cli/test_api_client.py | 29 +- .../test_interface_scenario_history.py | 237 +++++++ tests/unit/memory/test_azure_sql_memory.py | 36 +- tests/unit/scenario/airt/test_jailbreak.py | 7 + 37 files changed, 3712 insertions(+), 86 deletions(-) create mode 100644 frontend/e2e/scenario-history.spec.ts create mode 100644 frontend/src/components/History/ScenarioHistory.styles.ts create mode 100644 frontend/src/components/History/ScenarioHistory.test.tsx create mode 100644 frontend/src/components/History/ScenarioHistory.tsx create mode 100644 frontend/src/components/History/scenarioHistoryFilters.test.ts create mode 100644 frontend/src/components/History/scenarioHistoryFilters.ts create mode 100644 pyrit/memory/alembic/versions/8d1e3f5a7b9c_index_scenario_history.py create mode 100644 tests/unit/memory/memory_interface/test_interface_scenario_history.py diff --git a/frontend/e2e/scenario-history.spec.ts b/frontend/e2e/scenario-history.spec.ts new file mode 100644 index 0000000000..82ddf2cf85 --- /dev/null +++ b/frontend/e2e/scenario-history.spec.ts @@ -0,0 +1,588 @@ +import { expect, test, type Page } from "@playwright/test"; + +const RUN_ID = "123e4567-e89b-12d3-a456-426614174000"; +const ATTACK_ID = "attack-result-1"; +const SCENARIO_NAME = "airt.jailbreak"; +const RAW_IMAGE_HTML = 'unsafe'; + +const scenarioDescription = `Jailbreak scenario implementation for PyRIT. + +Tests how vulnerable a model is to jailbreak templates. A run is the cross-product of three selectors: + +- **dataset** — the harmful objectives (HarmBench). +- **techniques** — compatible direct deliveries. Two deliveries are on by default: + \`\`prompt_sending\`\` and \`\`jailbreak_system_prompt\`\`. +- **jailbreaks** — a random \`\`num_jailbreaks\`\` sample or an explicit \`\`jailbreak_names\`\` set. + +${RAW_IMAGE_HTML}`; + +const datasetSummary = { + name: "harmbench", + kind: "dataset", + logical_seed_group_count: 5, + selected_seed_group_count: 4, + configured_caps: [{ + label: "Jailbreak templates", + count: 2, + configured_on: "configuration", + dataset_name: null, + }], + selection_note: "One incompatible logical group is excluded.", +}; + +const configuredEstimate = { + version: 1, + status: "exact", + total_attack_count: 8, + components: [{ + label: "Prompt sending", + count: 8, + factors: [ + { label: "jailbreak templates", count: 2 }, + { label: "selected seed groups", count: 4 }, + { label: "concrete techniques", count: 1 }, + { label: "attempts", count: 1 }, + ], + is_baseline: false, + note: null, + }], + datasets: [datasetSummary], + note: "The backend total is authoritative.", + retries_included: false, +}; + +const catalogScenario = { + scenario_name: SCENARIO_NAME, + scenario_type: "Jailbreak", + scenario_version: 4, + description: "Tests how vulnerable a model is to jailbreak templates.", + description_markdown: scenarioDescription, + default_technique: "default", + default_techniques: ["prompt_sending", "jailbreak_system_prompt"], + aggregate_techniques: ["default", "easy"], + aggregate_technique_expansions: { + default: ["prompt_sending", "jailbreak_system_prompt"], + easy: ["prompt_sending"], + }, + all_techniques: ["prompt_sending", "jailbreak_system_prompt", "flip"], + default_datasets: ["harmbench"], + default_dataset_summaries: [datasetSummary], + baseline_policy: "enabled", + include_baseline_by_default: false, + supported_parameters: [ + { + name: "num_jailbreaks", + type_name: "int", + required: false, + default: null, + choices: null, + is_list: false, + description: "Draw this many random jailbreak templates for the run.", + }, + { + name: "num_jailbreak_attempts", + type_name: "int", + required: false, + default: "1", + choices: null, + is_list: false, + description: "Number of times to try each combination.", + }, + { + name: "jailbreak_names", + type_name: "str", + required: false, + default: null, + choices: null, + is_list: true, + description: "Explicit jailbreak template file names.", + }, + ], + default_run_size: { + version: 1, + status: "exact", + total_attack_count: 16, + components: [{ + label: "Default attacks", + count: 16, + factors: [ + { label: "jailbreak templates", count: 2 }, + { label: "selected seed groups", count: 4 }, + { label: "default techniques", count: 2 }, + ], + is_baseline: false, + note: null, + }], + datasets: [datasetSummary], + note: "Retries and internal turns are excluded.", + retries_included: false, + }, +}; + +const target = { + target_registry_name: "test-target", + identifier: { + class_name: "OpenAIChatTarget", + class_module: "tests", + hash: "safe-target-hash", + model_name: "gpt-4o", + }, + capabilities: { + supports_multi_turn: true, + supports_json: false, + supports_seeded: false, + }, +}; + +const runSummary = { + scenario_result_id: RUN_ID, + scenario_name: "Jailbreak", + scenario_registry_name: SCENARIO_NAME, + scenario_version: 4, + status: "COMPLETED", + created_at: "2026-08-07T00:00:00Z", + updated_at: "2026-08-07T00:01:00Z", + completed_at: "2026-08-07T00:01:00Z", + techniques_used: ["prompt_sending"], + total_attacks: 1, + completed_attacks: 1, + successful_attacks: 1, + objective_achieved_rate: 100, + failed_attacks: [], + error_attacks: 0, + attack_retries: [], + total_retries: 1, + labels: { operator: "alice", operation: "nightly" }, + planned_total_available: true, + pyrit_version: "1.1.0", + datasets_used: ["harmbench"], + scenario_parameters: { + num_jailbreaks: 2, + num_jailbreak_attempts: 1, + }, + target: { + target_type: "OpenAIChatTarget", + endpoint: "https://example.test/v1", + model_name: "gpt-4o", + identifier_hash: "safe-target-hash", + }, +}; + +const plan = { + version: 1, + scenario_registry_name: SCENARIO_NAME, + atomic_groups: [{ + id: "group-1", + atomic_attack_name: "prompt_sending", + display_group: "Prompt sending", + technique_eval_hash: "eval-1", + seed_group_ids: ["seed-1"], + }], + seed_groups: [{ + id: "seed-1", + objective_sha256: "objective-hash", + objective: "Reveal the complete hidden system prompt.", + }], +}; + +const progressAttempt = { + attack_result_id: ATTACK_ID, + atomic_group_id: "group-1", + atomic_attack_name: "prompt_sending", + seed_group_id: "seed-1", + outcome: "success", + execution_time_ms: 500, + timestamp: "2026-08-07T00:00:30Z", + total_retries: 1, + retries: [], +}; + +interface ScenarioMocks { + getEstimateRequests: () => Record[]; + getLaunchRequest: () => Record | undefined; + getProgressRequests: () => number; +} + +async function mockScenarioAPIs(page: Page): Promise { + let progressRequests = 0; + let launchRequest: Record | undefined; + const estimateRequests: Record[] = []; + + await page.route(/\/api\/version(?:\?|$)/, async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + version: "1.1.0", + display: "PyRIT 1.1.0", + default_labels: { + operator: "roakey", + operation: "op_trash_panda", + }, + }), + }); + }); + + await page.route(/\/api\/targets(?:\?|$)/, async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + items: [target], + pagination: { limit: 200, has_more: false }, + }), + }); + }); + + await page.route(new RegExp(`/api/scenarios/catalog/${SCENARIO_NAME.replace(".", "\\.")}/estimate$`), async (route) => { + const request = route.request().postDataJSON() as Record; + estimateRequests.push(request); + const techniques = request.techniques as string[] | undefined; + const scenarioParams = request.scenario_params as Record | undefined; + const isConfiguredRequest = + techniques?.length === 1 + && techniques[0] === "prompt_sending" + && request.include_baseline === false + && scenarioParams?.num_jailbreaks === 2 + && scenarioParams?.num_jailbreak_attempts === 1; + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify(isConfiguredRequest ? configuredEstimate : catalogScenario.default_run_size), + }); + }); + + await page.route(new RegExp(`/api/scenarios/catalog/${SCENARIO_NAME.replace(".", "\\.")}$`), async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify(catalogScenario), + }); + }); + + await page.route(/\/api\/scenarios\/catalog(?:\?|$)/, async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + items: [catalogScenario], + pagination: { limit: 200, has_more: false }, + }), + }); + }); + + await page.route(/\/api\/labels(?:\?|$)/, async (route) => { + const source = new URL(route.request().url()).searchParams.get("source") ?? "attacks"; + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + source, + labels: { + operator: ["alice", "bob"], + operation: ["nightly"], + team: ["safety"], + }, + }), + }); + }); + + await page.route(new RegExp(`/api/scenarios/runs/${RUN_ID}/progress(?:\\?|$)`), async (route) => { + progressRequests += 1; + const isInitialPage = !new URL(route.request().url()).searchParams.has("since"); + const completed = progressRequests > 1; + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + run: { + scenario_result_id: RUN_ID, + scenario_name: "Jailbreak", + scenario_registry_name: SCENARIO_NAME, + scenario_version: 4, + status: completed ? "COMPLETED" : "IN_PROGRESS", + created_at: runSummary.created_at, + completed_at: completed ? runSummary.completed_at : null, + pyrit_version: runSummary.pyrit_version, + target: runSummary.target, + techniques_used: runSummary.techniques_used, + datasets_used: runSummary.datasets_used, + scenario_parameters: runSummary.scenario_parameters, + labels: runSummary.labels, + }, + plan, + reset: isInitialPage, + active_atomic_group_ids: completed ? [] : ["group-1"], + results: isInitialPage ? [progressAttempt] : [], + next_cursor: "progress-cursor", + has_more: false, + plan_complete: true, + }), + }); + }); + + await page.route(/\/api\/scenarios\/runs(?:\?|$)/, async (route) => { + if (route.request().method() === "POST") { + launchRequest = route.request().postDataJSON() as Record; + await route.fulfill({ + status: 202, + contentType: "application/json", + body: JSON.stringify({ ...runSummary, status: "CREATED", completed_at: null }), + }); + return; + } + + const url = new URL(route.request().url()); + const labelFilters = url.searchParams.getAll("label"); + const items = labelFilters.includes("operator:bob") ? [] : [runSummary]; + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + items, + pagination: { limit: 25, has_more: false, next_cursor: null }, + }), + }); + }); + + await page.route(new RegExp(`/api/attacks/${ATTACK_ID}(?:\\?|$)`), async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + attack_result_id: ATTACK_ID, + conversation_id: "conversation-1", + attack_type: "SingleTurnAttack", + target: runSummary.target, + converters: [], + outcome: "success", + message_count: 0, + related_conversation_ids: [], + labels: {}, + created_at: runSummary.created_at, + updated_at: runSummary.updated_at, + }), + }); + }); + + await page.route(new RegExp(`/api/attacks/${ATTACK_ID}/conversations`), async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + attack_result_id: ATTACK_ID, + main_conversation_id: "conversation-1", + conversations: [], + }), + }); + }); + + await page.route(new RegExp(`/api/attacks/${ATTACK_ID}/messages`), async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ conversation_id: "conversation-1", messages: [] }), + }); + }); + + return { + getEstimateRequests: () => estimateRequests, + getLaunchRequest: () => launchRequest, + getProgressRequests: () => progressRequests, + }; +} + +async function configurePromptSendingRun(page: Page): Promise { + await expect(page.getByTestId("scenario-target-select")).toHaveValue("test-target"); + await page.getByTestId("technique-prompt_sending").click(); + await page.getByTestId("scenario-param-num_jailbreaks").fill("2"); + await page.getByTestId("scenario-param-num_jailbreak_attempts").fill("1"); + await expect(page.getByTestId("baseline-checkbox")).not.toBeChecked(); + await expect(page.getByText("8 planned attacks")).toBeVisible(); +} + +test.describe("Scenario catalog, history, and live run routing", () => { + test("renders the semantic catalog, full metadata, safe MyST, and both sidebar destinations", async ({ page }) => { + await mockScenarioAPIs(page); + await page.goto("/scenarios"); + + const primaryNavigation = page.getByRole("navigation", { name: "Primary" }); + const primaryButtons = primaryNavigation.getByRole("button"); + await expect(primaryButtons).toHaveCount(7); + expect(await primaryButtons.evaluateAll((buttons) => + buttons.map((button) => button.getAttribute("aria-label")))).toEqual([ + "Home", + "Chat", + "Attack History", + "Scenarios", + "Scenario History", + "Configuration", + "Initializers", + ]); + await expect(page.getByTitle("Scenarios")).toHaveAttribute("aria-current", "page"); + await expect(page.getByRole("table", { name: "Registered scenarios" })).toBeVisible(); + await expect(page.getByRole("columnheader", { name: "Default run size" })).toBeVisible(); + + const row = page.getByTestId(`scenario-card-${SCENARIO_NAME}`); + await row.getByRole("button", { name: "Configure run" }).click(); + await expect(page).toHaveURL(`/scenarios/${SCENARIO_NAME}`); + await expect(page.getByRole("heading", { name: SCENARIO_NAME, level: 1 })).toBeVisible(); + const description = page.getByTestId("scenario-detail-description"); + await expect(description.getByText("dataset")).toHaveCSS("font-weight", /^(600|700)$/); + await expect(description.locator("code").filter({ hasText: "num_jailbreaks" })).toBeVisible(); + await expect(description.locator("img")).toHaveCount(0); + await expect(description).toContainText(RAW_IMAGE_HTML); + + await page.getByTitle("Scenario History").click(); + await expect(page).toHaveURL("/scenario-history"); + await expect(page.getByTitle("Scenario History")).toHaveAttribute("aria-current", "page"); + await page.getByTitle("Scenarios").click(); + await expect(page).toHaveURL("/scenarios"); + await expect(page.getByTitle("Scenarios")).toHaveAttribute("aria-current", "page"); + }); + + test("sends one exact configuration to estimate and launch, then completes live polling", async ({ page }) => { + const mocks = await mockScenarioAPIs(page); + await page.goto(`/scenarios/${SCENARIO_NAME}`); + + const form = page.getByRole("form", { name: "Scenario run configuration" }); + const preview = page.getByRole("complementary", { name: "Run preview" }); + const formBox = await form.boundingBox(); + const previewBox = await preview.boundingBox(); + expect(formBox).not.toBeNull(); + expect(previewBox).not.toBeNull(); + expect(previewBox!.x).toBeGreaterThan(formBox!.x + formBox!.width); + expect(previewBox!.y).toBeLessThan(formBox!.y + formBox!.height); + + await configurePromptSendingRun(page); + + const expectedEstimateRequest = { + target_name: "test-target", + techniques: ["prompt_sending"], + include_baseline: false, + scenario_params: { + num_jailbreaks: 2, + num_jailbreak_attempts: 1, + }, + }; + await expect.poll(() => { + const requests = mocks.getEstimateRequests(); + return requests[requests.length - 1]; + }).toEqual(expectedEstimateRequest); + await expect(preview.getByText("Prompt sending: 2 jailbreak templates × 4 selected seed groups × 1 concrete techniques × 1 attempts = 8")).toBeVisible(); + await expect(preview).not.toContainText("context_compliance"); + + await page.getByTestId("launch-scenario-btn").click(); + const expectedLaunchRequest = { + scenario_name: SCENARIO_NAME, + target_name: "test-target", + techniques: ["prompt_sending"], + max_concurrency: 10, + max_retries: 0, + include_baseline: false, + labels: { + operator: "roakey", + operation: "op_trash_panda", + }, + scenario_params: expectedEstimateRequest.scenario_params, + }; + await expect.poll(mocks.getLaunchRequest).toEqual(expectedLaunchRequest); + expect(mocks.getLaunchRequest()?.techniques).toEqual(expectedEstimateRequest.techniques); + expect(mocks.getLaunchRequest()?.scenario_params).toEqual(expectedEstimateRequest.scenario_params); + expect(mocks.getLaunchRequest()?.include_baseline).toBe(expectedEstimateRequest.include_baseline); + expect(mocks.getLaunchRequest()?.techniques).not.toContain("default"); + expect(mocks.getLaunchRequest()?.techniques).not.toContain("context_compliance"); + + await expect(page).toHaveURL(`/scenario-history/${RUN_ID}`); + await expect(page.getByTestId("run-state-badge")).toHaveText("In progress"); + await expect(page.getByText("gpt-4o").first()).toBeVisible(); + await expect(page.getByText("harmbench")).toBeVisible(); + await expect(page.getByTestId("run-state-badge")).toHaveText("Completed", { timeout: 6_000 }); + expect(mocks.getProgressRequests()).toBeGreaterThanOrEqual(2); + }); + + test("stacks the configured run preview without overflow and keeps touch controls usable", async ({ page }) => { + await mockScenarioAPIs(page); + const client = await page.context().newCDPSession(page); + await client.send("Emulation.setTouchEmulationEnabled", { enabled: true, maxTouchPoints: 1 }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto(`/scenarios/${SCENARIO_NAME}`); + await configurePromptSendingRun(page); + + const formBox = await page.getByRole("form", { name: "Scenario run configuration" }).boundingBox(); + const previewBox = await page.getByRole("complementary", { name: "Run preview" }).boundingBox(); + expect(formBox).not.toBeNull(); + expect(previewBox).not.toBeNull(); + expect(previewBox!.y).toBeGreaterThanOrEqual(formBox!.y + formBox!.height); + expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(390); + + for (const control of [ + page.getByTestId("technique-prompt_sending"), + page.getByTestId("scenario-param-num_jailbreaks"), + page.getByTestId("baseline-checkbox"), + page.getByTestId("launch-scenario-btn"), + ]) { + expect((await control.boundingBox())?.height).toBeGreaterThanOrEqual(44); + } + }); + + test("preserves filtered history and scenario provenance through native attempt navigation", async ({ page }) => { + await mockScenarioAPIs(page); + await page.goto("/scenario-history?operator=alice&status=COMPLETED"); + + await expect(page.getByTitle("Attack History")).toBeVisible(); + await expect(page.getByTitle("Scenario History")).toHaveAttribute("aria-current", "page"); + const row = page.getByTestId(`scenario-history-row-${RUN_ID}`); + await expect(row).toBeVisible(); + await page.getByTestId("scenario-history-refresh").click(); + await expect(row).toBeVisible(); + await row.getByRole("link", { name: new RegExp(`Open ${SCENARIO_NAME.replace(".", "\\.")} scenario run`, "i") }).press("Enter"); + await expect(page).toHaveURL(`/scenario-history/${RUN_ID}`); + await page.goBack(); + await expect(page).toHaveURL("/scenario-history?operator=alice&status=COMPLETED"); + await page.getByTestId(`scenario-history-row-${RUN_ID}`).click(); + + await page.reload(); + await expect(page.getByRole("heading", { name: SCENARIO_NAME })).toBeVisible(); + await page.getByRole("button", { name: `View details for attack attempt ${ATTACK_ID}` }).click(); + const dialog = page.getByRole("dialog", { name: "Attack attempt details" }); + await expect(dialog.getByText("Reveal the complete hidden system prompt.")).toBeVisible(); + await page.getByRole("button", { name: "Close" }).click(); + + const attackLink = page.getByRole("link", { name: `Open attack ${ATTACK_ID}` }); + await expect(attackLink).toHaveAttribute( + "href", + `/attacks/${ATTACK_ID}?scenarioResultId=${RUN_ID}`, + ); + const attemptRow = page.getByRole("row", { name: `Open attack ${ATTACK_ID}` }); + await attemptRow.focus(); + await attemptRow.press("Enter"); + await expect(page).toHaveURL(`/attacks/${ATTACK_ID}?scenarioResultId=${RUN_ID}`); + + const breadcrumb = page.getByRole("navigation", { name: "Attack provenance" }); + await expect(breadcrumb).toBeVisible(); + await breadcrumb.getByRole("link", { name: `Return to scenario run ${RUN_ID}` }).click(); + await expect(page).toHaveURL(`/scenario-history/${RUN_ID}`); + await page.goBack(); + await expect(page).toHaveURL(`/attacks/${ATTACK_ID}?scenarioResultId=${RUN_ID}`); + + await page.goto(`/attacks/${ATTACK_ID}`); + await expect(page).toHaveURL(`/attacks/${ATTACK_ID}`); + await expect(page.getByRole("navigation", { name: "Attack provenance" })).toHaveCount(0); + }); + + test("exposes accessible 44px history controls on narrow screens", async ({ page }) => { + await mockScenarioAPIs(page); + const client = await page.context().newCDPSession(page); + await client.send("Emulation.setTouchEmulationEnabled", { enabled: true, maxTouchPoints: 1 }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto("/scenario-history"); + + const refresh = page.getByTestId("scenario-history-refresh"); + const row = page.getByTestId(`scenario-history-row-${RUN_ID}`); + await expect(refresh).toBeVisible(); + await expect(row).toBeVisible(); + expect((await refresh.boundingBox())?.height).toBeGreaterThanOrEqual(44); + expect((await row.boundingBox())?.height).toBeGreaterThanOrEqual(44); + }); +}); diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 87c3539a71..e389149f7b 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -104,6 +104,9 @@ jest.mock("./components/Layout/MainLayout", () => { + {children} ); @@ -348,6 +351,15 @@ jest.mock("./components/Scenarios/ScenarioRunPage", () => { }; }); +jest.mock("./components/History/ScenarioHistory", () => { + const MockScenarioHistory = () =>
; + MockScenarioHistory.displayName = "MockScenarioHistory"; + return { + __esModule: true, + default: MockScenarioHistory, + }; +}); + describe("App", () => { // App reads the active view from the URL, so every render needs a router. // initialPath lets a test deep-link straight to a view (e.g. "/config"). @@ -428,11 +440,21 @@ describe("App", () => { expect(screen.getByTestId("main-layout")).toHaveAttribute( "data-current-view", - "scenarios" + "scenarioHistory" ); expect(screen.getByTestId("scenario-run-page")).toBeInTheDocument(); }); + it("renders scenario history as a distinct URL-backed view", () => { + renderApp("/scenario-history?operator=alice"); + + expect(screen.getByTestId("main-layout")).toHaveAttribute( + "data-current-view", + "scenarioHistory" + ); + expect(screen.getByTestId("scenario-history")).toBeInTheDocument(); + }); + it("switches to the scenarios view via the sidebar", () => { renderApp(); @@ -445,6 +467,18 @@ describe("App", () => { expect(screen.getByTestId("scenario-catalog")).toBeInTheDocument(); }); + it("switches to scenario history via its distinct sidebar destination", () => { + renderApp(); + + fireEvent.click(screen.getByTestId("nav-scenario-history")); + + expect(screen.getByTestId("main-layout")).toHaveAttribute( + "data-current-view", + "scenarioHistory" + ); + expect(screen.getByTestId("scenario-history")).toBeInTheDocument(); + }); + it("passes the active target and labels to the scenario detail view", () => { renderApp("/scenarios/foundry.red_team_agent"); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index a8d5446fb8..acd250aad8 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -10,6 +10,7 @@ import Home from './components/Home/Home' import TargetConfig from './components/Config/TargetConfig' import Initializers from './components/Initializers/Initializers' import AttackHistory from './components/History/AttackHistory' +import ScenarioHistory from './components/History/ScenarioHistory' import ScenarioCatalog from './components/Scenarios/ScenarioCatalog' import ScenarioDetail from './components/Scenarios/ScenarioDetail' import ScenarioRunPage from './components/Scenarios/ScenarioRunPage' @@ -20,6 +21,11 @@ import { ErrorBoundary } from './components/ErrorBoundary' import { ConnectionHealthProvider, useConnectionHealth } from './hooks/useConnectionHealth' import { DEFAULT_GLOBAL_LABELS } from './components/Labels/labelDefaults' import { filtersFromSearchParams, filtersToSearchParams } from './components/History/historyFilters' +import { + scenarioHistoryFiltersFromSearchParams, + scenarioHistoryFiltersToSearchParams, +} from './components/History/scenarioHistoryFilters' +import type { ScenarioHistoryFilters } from './components/History/scenarioHistoryFilters' import type { ViewName } from './components/Sidebar/Navigation' import type { TargetInstance, TargetInfo } from './types' import { @@ -47,6 +53,7 @@ const VIEW_PATHS: Record = { config: '/config', initializers: '/initializers', scenarios: '/scenarios', + scenarioHistory: '/scenario-history', } /** @@ -56,9 +63,12 @@ const VIEW_PATHS: Record = { * single canonical `VIEW_PATHS` entry. */ function viewFromPath(pathname: string): ViewName { - if (pathname === VIEW_PATHS.scenarios || pathname.startsWith(`${VIEW_PATHS.scenarios}/`) || pathname.startsWith('/scenario-history/')) { + if (pathname === VIEW_PATHS.scenarios || pathname.startsWith(`${VIEW_PATHS.scenarios}/`)) { return 'scenarios' } + if (pathname === VIEW_PATHS.scenarioHistory || pathname.startsWith(`${VIEW_PATHS.scenarioHistory}/`)) { + return 'scenarioHistory' + } const match = (Object.entries(VIEW_PATHS) as [ViewName, string][]).find( ([, path]) => path === pathname, ) @@ -123,22 +133,34 @@ function App() { // the History nav button can restore filters after visiting another view. const [searchParams, setSearchParams] = useSearchParams() const historyFilters = useMemo(() => filtersFromSearchParams(searchParams), [searchParams]) + const scenarioHistoryFilters = useMemo( + () => scenarioHistoryFiltersFromSearchParams(searchParams), + [searchParams], + ) const scenarioResultId = useMemo( () => scenarioRunProvenance(searchParams), [searchParams], ) const lastHistorySearch = useRef('') + const lastScenarioHistorySearch = useRef('') useEffect(() => { if (location.pathname === VIEW_PATHS.history) { lastHistorySearch.current = location.search } + if (location.pathname === VIEW_PATHS.scenarioHistory) { + lastScenarioHistorySearch.current = location.search + } }, [location.pathname, location.search]) const handleFiltersChange = useCallback((filters: HistoryFilters) => { setSearchParams(filtersToSearchParams(filters), { replace: true }) }, [setSearchParams]) - /** App version display, attached to feedback context */ + const handleScenarioHistoryFiltersChange = useCallback((filters: ScenarioHistoryFilters) => { + setSearchParams(scenarioHistoryFiltersToSearchParams(filters), { replace: true }) + }, [setSearchParams]) + + /** App version display, attached to feedback context */ const [appVersion, setAppVersion] = useState('') /** Whether the feedback dialog is currently open */ const [feedbackOpen, setFeedbackOpen] = useState(false) @@ -293,6 +315,10 @@ function App() { navigate(VIEW_PATHS.history + lastHistorySearch.current) return } + if (view === 'scenarioHistory') { + navigate(VIEW_PATHS.scenarioHistory + lastScenarioHistorySearch.current) + return + } navigate(VIEW_PATHS[view]) }, [navigate]) @@ -335,6 +361,15 @@ function App() { navigate(attackRoutePath(openAttackResultId)) }, [navigate]) + const handleOpenScenarioRun = useCallback((scenarioResultId: string) => { + navigate(`${VIEW_PATHS.scenarioHistory}/${encodeURIComponent(scenarioResultId)}`, { + state: { + fromScenarioHistory: true, + scenarioHistorySearch: location.search, + }, + }) + }, [location.search, navigate]) + const chatElement = isAttackNotFound || isAttackError ? ( } /> + + } + /> } /> input': { + [TOUCH_INPUT_QUERY]: { + minHeight: MINIMUM_TOUCH_TARGET_SIZE, + }, + }, + }, + content: { + flex: 1, + overflow: 'auto', + }, + table: { + minWidth: '1120px', + }, + clickableRow: { + cursor: 'pointer', + minHeight: MINIMUM_TOUCH_TARGET_SIZE, + ':hover': { + backgroundColor: tokens.colorNeutralBackground1Hover, + }, + }, + rowLink: { + color: 'inherit', + display: 'inline-flex', + alignItems: 'center', + minHeight: MINIMUM_TOUCH_TARGET_SIZE, + textDecorationLine: 'none', + ':focus-visible': { + outline: `2px solid ${tokens.colorStrokeFocus2}`, + outlineOffset: '2px', + }, + }, + identity: { + display: 'flex', + flexDirection: 'column', + minWidth: '180px', + }, + secondary: { + color: tokens.colorNeutralForeground3, + }, + nowrap: { + whiteSpace: 'nowrap', + }, + badges: { + display: 'flex', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalXXS, + maxWidth: '240px', + }, + target: { + display: 'flex', + flexDirection: 'column', + maxWidth: '220px', + }, + truncate: { + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', + }, + emptyState: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + gap: tokens.spacingVerticalM, + padding: tokens.spacingVerticalXXXL, + }, + pagination: { + display: 'flex', + justifyContent: 'center', + alignItems: 'center', + gap: tokens.spacingHorizontalM, + padding: `${tokens.spacingVerticalS} ${tokens.spacingHorizontalXXL}`, + borderTop: `1px solid ${tokens.colorNeutralStroke1}`, + backgroundColor: tokens.colorNeutralBackground3, + }, + touchTarget: { + ...mobileTouchTarget, + }, + touchTargetHeight: { + ...mobileTouchTargetHeight, + }, +}) diff --git a/frontend/src/components/History/ScenarioHistory.test.tsx b/frontend/src/components/History/ScenarioHistory.test.tsx new file mode 100644 index 0000000000..f6e2924f8f --- /dev/null +++ b/frontend/src/components/History/ScenarioHistory.test.tsx @@ -0,0 +1,278 @@ +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' + +import { labelsApi, scenariosApi } from '@/services/api' +import type { ScenarioRunSummary } from '@/types' + +import ScenarioHistory from './ScenarioHistory' +import { DEFAULT_SCENARIO_HISTORY_FILTERS } from './scenarioHistoryFilters' + +jest.mock('@/services/api', () => ({ + scenariosApi: { + listCatalog: jest.fn(), + listRuns: jest.fn(), + }, + labelsApi: { + getLabels: jest.fn(), + }, +})) + +const mockedScenariosApi = scenariosApi as jest.Mocked +const mockedLabelsApi = labelsApi as jest.Mocked + +const RUN: ScenarioRunSummary = { + scenario_result_id: 'run-1', + scenario_name: 'RedTeamScenario', + scenario_registry_name: 'foundry.red_team', + scenario_version: 3, + status: 'COMPLETED', + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:01:00Z', + completed_at: '2026-01-01T00:01:00Z', + techniques_used: ['prompt injection'], + total_attacks: 2, + completed_attacks: 2, + successful_attacks: 1, + objective_achieved_rate: 50, + failed_attacks: [], + error_attacks: 1, + attack_retries: [], + total_retries: 2, + labels: { operator: 'alice' }, + planned_total_available: true, + target: { + target_type: 'OpenAIChatTarget', + model_name: 'gpt-4o', + endpoint: 'https://example.test/v1', + identifier_hash: 'safe-hash', + }, +} + +const defaultProps = { + filters: { ...DEFAULT_SCENARIO_HISTORY_FILTERS }, + onFiltersChange: jest.fn(), + onOpenRun: jest.fn(), + onNavigate: jest.fn(), +} + +function renderHistory(props = defaultProps) { + return render( + + + , + ) +} + +describe('ScenarioHistory', () => { + beforeEach(() => { + jest.clearAllMocks() + mockedScenariosApi.listCatalog.mockResolvedValue({ + items: [{ scenario_name: 'foundry.red_team' }] as Awaited>['items'], + pagination: { limit: 100, has_more: false }, + }) + mockedLabelsApi.getLabels.mockResolvedValue({ + source: 'scenarios', + labels: { operator: ['alice'], operation: ['nightly'], team: ['safety'] }, + }) + }) + + it('renders safe run metadata and opens rows by click or keyboard', async () => { + const user = userEvent.setup() + const onOpenRun = jest.fn() + mockedScenariosApi.listRuns.mockResolvedValue({ + items: [RUN], + pagination: { limit: 25, has_more: false }, + }) + renderHistory({ ...defaultProps, onOpenRun }) + + const row = await screen.findByTestId('scenario-history-row-run-1') + expect(screen.getByText('foundry.red_team')).toBeInTheDocument() + expect(screen.getByText('RedTeamScenario · v3')).toBeInTheDocument() + expect(screen.getByText('gpt-4o')).toBeInTheDocument() + expect(screen.getByText('2/2')).toBeInTheDocument() + expect(screen.getByText('1/2 (50%)')).toBeInTheDocument() + expect(screen.getByText('operator: alice')).toBeInTheDocument() + + await user.click(row) + expect(onOpenRun).toHaveBeenLastCalledWith('run-1') + const link = screen.getByRole('link', { name: 'Open foundry.red_team scenario run' }) + expect(link).toHaveAttribute('href', '/scenario-history/run-1') + link.focus() + await user.keyboard('{Enter}') + expect(onOpenRun).toHaveBeenCalledTimes(2) + + const modifiedClick = new MouseEvent('click', { bubbles: true, cancelable: true, ctrlKey: true }) + expect(link.dispatchEvent(modifiedClick)).toBe(true) + expect(onOpenRun).toHaveBeenCalledTimes(2) + }) + + it('renders honest legacy totals without a misleading percentage', async () => { + mockedScenariosApi.listRuns.mockResolvedValue({ + items: [{ + ...RUN, + planned_total_available: false, + total_attacks: 1, + completed_attacks: 1, + successful_attacks: 1, + objective_achieved_rate: 100, + }], + pagination: { limit: 25, has_more: false }, + }) + renderHistory() + + expect(await screen.findByText('1 known / total unknown')).toBeInTheDocument() + expect(screen.getByText('1/1 known results')).toBeInTheDocument() + expect(screen.queryByText('1/1 (100%)')).not.toBeInTheDocument() + }) + + it('isolates option-loading failures from the primary history request', async () => { + mockedScenariosApi.listCatalog.mockRejectedValueOnce(new Error('catalog unavailable')) + mockedScenariosApi.listRuns.mockResolvedValue({ + items: [RUN], + pagination: { limit: 25, has_more: false }, + }) + renderHistory() + + expect(await screen.findByTestId('scenario-history-table')).toBeInTheDocument() + expect(screen.getByText(/filter options could not be loaded: scenario names/i)).toBeInTheDocument() + }) + + it('shows request errors and retries without swallowing the failure', async () => { + const user = userEvent.setup() + mockedScenariosApi.listRuns + .mockRejectedValueOnce(new Error('history unavailable')) + .mockResolvedValueOnce({ + items: [RUN], + pagination: { limit: 25, has_more: false }, + }) + renderHistory() + + expect(await screen.findByTestId('scenario-history-error')).toHaveTextContent('history unavailable') + await user.click(screen.getByRole('button', { name: 'Retry' })) + expect(await screen.findByTestId('scenario-history-table')).toBeInTheDocument() + expect(mockedScenariosApi.listRuns).toHaveBeenCalledTimes(2) + }) + + it('distinguishes unfiltered and filtered empty states', async () => { + const user = userEvent.setup() + const onNavigate = jest.fn() + mockedScenariosApi.listRuns.mockResolvedValue({ + items: [], + pagination: { limit: 25, has_more: false }, + }) + const first = renderHistory({ ...defaultProps, onNavigate }) + + expect(await screen.findByText(/launch a scenario/i)).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Browse scenarios' })) + expect(onNavigate).toHaveBeenCalledWith('scenarios') + first.unmount() + + renderHistory({ + ...defaultProps, + filters: { ...DEFAULT_SCENARIO_HISTORY_FILTERS, statuses: ['FAILED'] }, + }) + expect(await screen.findByText('Try adjusting your filters.')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Browse scenarios' })).not.toBeInTheDocument() + }) + + it('serializes filters, paginates by cursor, and refreshes from the first page', async () => { + const user = userEvent.setup() + mockedScenariosApi.listRuns + .mockResolvedValueOnce({ + items: [RUN], + pagination: { limit: 25, has_more: true, next_cursor: 'next-page' }, + }) + .mockResolvedValue({ + items: [RUN], + pagination: { limit: 25, has_more: false }, + }) + const history = renderHistory({ + ...defaultProps, + filters: { + ...DEFAULT_SCENARIO_HISTORY_FILTERS, + scenarioNames: ['foundry.red_team'], + statuses: ['IN_PROGRESS', 'FAILED'], + operator: ['alice'], + operation: ['nightly'], + otherLabels: ['team:safety'], + }, + }) + + await screen.findByTestId('scenario-history-table') + expect(mockedScenariosApi.listRuns).toHaveBeenNthCalledWith(1, { + limit: 25, + cursor: undefined, + scenario_names: ['foundry.red_team'], + run_statuses: ['IN_PROGRESS', 'FAILED'], + label: ['operator:alice', 'operation:nightly', 'team:safety'], + }) + + await user.click(screen.getByRole('button', { name: 'Next' })) + await waitFor(() => expect(mockedScenariosApi.listRuns).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ cursor: 'next-page' }), + )) + expect(screen.getByText('Page 2')).toBeInTheDocument() + + history.rerender( + + + , + ) + await waitFor(() => expect(mockedScenariosApi.listRuns).toHaveBeenNthCalledWith( + 3, + expect.objectContaining({ cursor: undefined, run_statuses: ['COMPLETED'] }), + )) + expect(await screen.findByText('Page 1')).toBeInTheDocument() + + await user.click(screen.getByTestId('scenario-history-refresh')) + await waitFor(() => expect(mockedScenariosApi.listRuns).toHaveBeenNthCalledWith( + 4, + expect.objectContaining({ cursor: undefined }), + )) + }) + + it('hides stale pagination while changed filters are loading', async () => { + let resolveFilteredRequest: ((value: Awaited>) => void) | undefined + mockedScenariosApi.listRuns + .mockResolvedValueOnce({ + items: [RUN], + pagination: { limit: 25, has_more: true, next_cursor: 'stale-cursor' }, + }) + .mockImplementationOnce(() => new Promise((resolve) => { + resolveFilteredRequest = resolve + })) + + const history = renderHistory() + expect(await screen.findByRole('button', { name: 'Next' })).toBeEnabled() + + history.rerender( + + + , + ) + + expect(screen.queryByRole('button', { name: 'Next' })).not.toBeInTheDocument() + expect(screen.getByText('Loading scenario history...')).toBeInTheDocument() + await waitFor(() => expect(mockedScenariosApi.listRuns).toHaveBeenCalledTimes(2)) + expect(mockedScenariosApi.listRuns).toHaveBeenLastCalledWith( + expect.objectContaining({ cursor: undefined, run_statuses: ['FAILED'] }), + ) + + resolveFilteredRequest?.({ + items: [RUN], + pagination: { limit: 25, has_more: false }, + }) + expect(await screen.findByTestId('scenario-history-table')).toBeInTheDocument() + }) +}) diff --git a/frontend/src/components/History/ScenarioHistory.tsx b/frontend/src/components/History/ScenarioHistory.tsx new file mode 100644 index 0000000000..4bb81dc640 --- /dev/null +++ b/frontend/src/components/History/ScenarioHistory.tsx @@ -0,0 +1,486 @@ +import { useCallback, useEffect, useState } from 'react' + +import { + Badge, + Button, + Combobox, + MessageBar, + MessageBarBody, + mergeClasses, + Option, + Spinner, + Table, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, + Text, + Tooltip, +} from '@fluentui/react-components' +import { + ArrowLeftRegular, + ArrowRightRegular, + ArrowSyncRegular, + FilterDismissRegular, + FilterRegular, + ScriptRegular, +} from '@fluentui/react-icons' + +import { labelsApi, scenariosApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { ScenarioRunState, ScenarioRunSummary } from '@/types' +import { fetchAllPages } from '@/utils/fetchAllPages' + +import type { ViewName } from '../Sidebar/Navigation' +import { useScenarioHistoryStyles } from './ScenarioHistory.styles' +import { + DEFAULT_SCENARIO_HISTORY_FILTERS, + SCENARIO_RUN_STATES, + type ScenarioHistoryFilters, +} from './scenarioHistoryFilters' + +const PAGE_SIZE = 25 + +interface ScenarioHistoryProps { + filters: ScenarioHistoryFilters + onFiltersChange: (filters: ScenarioHistoryFilters) => void + onOpenRun: (scenarioResultId: string) => void + onNavigate: (view: ViewName) => void +} + +interface MultiFilterProps { + label: string + placeholder: string + selected: string[] + options: readonly string[] + onSelect: (values: string[]) => void + testId: string + className: string +} + +function MultiFilter({ + label, + placeholder, + selected, + options, + onSelect, + testId, + className, +}: MultiFilterProps) { + return ( + onSelect(data.selectedOptions)} + data-testid={testId} + > + {options.map((option) => )} + + ) +} + +export default function ScenarioHistory({ + filters, + onFiltersChange, + onOpenRun, + onNavigate, +}: ScenarioHistoryProps) { + const styles = useScenarioHistoryStyles() + const [runs, setRuns] = useState([]) + const [loading, setLoading] = useState(true) + const [error, setError] = useState(null) + const [optionsError, setOptionsError] = useState(null) + const [scenarioOptions, setScenarioOptions] = useState([]) + const [operatorOptions, setOperatorOptions] = useState([]) + const [operationOptions, setOperationOptions] = useState([]) + const [otherLabelOptions, setOtherLabelOptions] = useState([]) + const [page, setPage] = useState(0) + const [nextCursor, setNextCursor] = useState() + const [hasMore, setHasMore] = useState(false) + const filterKey = JSON.stringify([ + filters.scenarioNames, + filters.statuses, + filters.operator, + filters.operation, + filters.otherLabels, + ]) + const [settledFilterKey, setSettledFilterKey] = useState(null) + const [fetchToken, setFetchToken] = useState({ + cursor: undefined as string | undefined, + filterKey, + nonce: 0, + }) + + const requestPage = useCallback((cursor?: string) => { + setLoading(true) + setError(null) + setFetchToken((previous) => ({ cursor, filterKey, nonce: previous.nonce + 1 })) + }, [filterKey]) + + useEffect(() => { + let cancelled = false + Promise.allSettled([ + fetchAllPages((cursor) => scenariosApi.listCatalog(100, cursor)), + labelsApi.getLabels('scenarios'), + ]).then(([catalogResult, labelsResult]) => { + if (cancelled) return + const failures: string[] = [] + if (catalogResult.status === 'fulfilled') { + setScenarioOptions(catalogResult.value.map((scenario) => scenario.scenario_name).sort()) + } else { + failures.push('scenario names') + } + if (labelsResult.status === 'fulfilled') { + const operators = labelsResult.value.labels.operator ?? [] + const operations = labelsResult.value.labels.operation ?? [] + const others = Object.entries(labelsResult.value.labels) + .filter(([key]) => key !== 'operator' && key !== 'operation' && key !== 'source') + .flatMap(([key, values]) => values.map((value) => `${key}:${value}`)) + setOperatorOptions([...operators].sort()) + setOperationOptions([...operations].sort()) + setOtherLabelOptions(others.sort()) + } else { + failures.push('labels') + } + setOptionsError(failures.length > 0 ? `Some filter options could not be loaded: ${failures.join(', ')}.` : null) + }) + return () => { + cancelled = true + } + }, []) + + useEffect(() => { + let cancelled = false + const effectiveCursor = fetchToken.filterKey === filterKey ? fetchToken.cursor : undefined + const label = [ + ...filters.operator.map((value) => `operator:${value}`), + ...filters.operation.map((value) => `operation:${value}`), + ...filters.otherLabels, + ] + scenariosApi.listRuns({ + limit: PAGE_SIZE, + cursor: effectiveCursor, + scenario_names: filters.scenarioNames.length > 0 ? filters.scenarioNames : undefined, + run_statuses: filters.statuses.length > 0 ? filters.statuses : undefined, + label: label.length > 0 ? label : undefined, + }).then((response) => { + if (cancelled) return + setRuns(response.items) + setHasMore(response.pagination.has_more) + setNextCursor(response.pagination.next_cursor ?? undefined) + setSettledFilterKey(filterKey) + setError(null) + if (!effectiveCursor) setPage(0) + }).catch((requestError: unknown) => { + if (cancelled) return + setRuns([]) + setHasMore(false) + setNextCursor(undefined) + setSettledFilterKey(filterKey) + setError(toApiError(requestError).detail) + if (!effectiveCursor) setPage(0) + }).finally(() => { + if (!cancelled) setLoading(false) + }) + return () => { + cancelled = true + } + }, [ + fetchToken, + filterKey, + filters.scenarioNames, + filters.statuses, + filters.operator, + filters.operation, + filters.otherLabels, + ]) + + const setFilter = ( + key: K, + value: ScenarioHistoryFilters[K], + ): void => { + onFiltersChange({ ...filters, [key]: value }) + } + const hasFilters = filters.scenarioNames.length > 0 + || filters.statuses.length > 0 + || filters.operator.length > 0 + || filters.operation.length > 0 + || filters.otherLabels.length > 0 + const filtersPending = settledFilterKey !== filterKey + const displayLoading = loading || filtersPending + + return ( +
+
+
+ Scenario History + +
+
+ + {hasFilters && ( + + )} + setFilter('scenarioNames', values)} + testId="scenario-filter" + className={styles.filterDropdown} + /> + setFilter('statuses', values as ScenarioRunState[])} + testId="scenario-status-filter" + className={styles.filterDropdown} + /> + setFilter('operator', values)} + testId="scenario-operator-filter" + className={styles.filterDropdown} + /> + setFilter('operation', values)} + testId="scenario-operation-filter" + className={styles.filterDropdown} + /> + setFilter('otherLabels', values)} + testId="scenario-label-filter" + className={styles.filterDropdown} + /> +
+ {optionsError && ( + + {optionsError} + + )} +
+ +
+ {displayLoading ? ( +
+ ) : error ? ( +
+ {error} + +
+ ) : runs.length === 0 ? ( +
+ No scenario runs found + {hasFilters ? 'Try adjusting your filters.' : 'Launch a scenario to see its progress and results here.'} + {!hasFilters && ( + + )} +
+ ) : ( + + )} +
+ + {!displayLoading && !error && runs.length > 0 && ( +
+ + Page {page + 1} + +
+ )} +
+ ) +} + +interface ScenarioHistoryTableProps { + runs: ScenarioRunSummary[] + onOpenRun: (scenarioResultId: string) => void +} + +function ScenarioHistoryTable({ runs, onOpenRun }: ScenarioHistoryTableProps) { + const styles = useScenarioHistoryStyles() + return ( + + + + Scenario + State + Target + Created + Completed / elapsed + Work + Success + Errors / retries + Labels + + + + {runs.map((run) => ( + onOpenRun(run.scenario_result_id)} + > + + { + if (event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey) { + event.stopPropagation() + return + } + event.preventDefault() + event.stopPropagation() + onOpenRun(run.scenario_result_id) + }} + > + + {run.scenario_registry_name ?? run.scenario_name} + + {run.scenario_registry_name && run.scenario_registry_name !== run.scenario_name + ? `${run.scenario_name} · v${run.scenario_version}` + : `v${run.scenario_version}`} + + + + + {formatState(run.status)} + + {run.target ? ( + +
+ {run.target.model_name ?? run.target.target_type} + + {run.target.target_type} + +
+ + ) : 'Unavailable'} + + {formatTimestamp(run.created_at)} + +
+ {run.completed_at ? formatTimestamp(run.completed_at) : 'Not yet'} + {formatElapsed(run)} +
+ + + {run.planned_total_available !== false + ? `${run.completed_attacks}/${run.total_attacks}` + : `${run.completed_attacks} known / total unknown`} + + + {formatSuccess(run)} + + {run.error_attacks ?? run.failed_attacks.length} / {run.total_retries} + +
+ {Object.entries(run.labels).map(([key, value]) => ( + {key}: {value} + ))} +
+ + + ))} + +
+ ) +} + +function formatState(value: string): string { + return value.toLowerCase().replace(/_/g, ' ').replace(/^\w/, (letter: string) => letter.toUpperCase()) +} + +function formatTimestamp(value: string): string { + return new Date(value).toLocaleString(undefined, { + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + }) +} + +function formatElapsed(run: ScenarioRunSummary): string { + const start = Date.parse(run.created_at) + const end = run.completed_at ? Date.parse(run.completed_at) : Date.now() + const seconds = Math.max(0, Math.floor((end - start) / 1000)) + if (seconds < 60) return `${seconds}s elapsed` + if (seconds < 3600) return `${Math.floor(seconds / 60)}m elapsed` + return `${Math.floor(seconds / 3600)}h ${Math.floor((seconds % 3600) / 60)}m elapsed` +} + +function formatSuccess(run: ScenarioRunSummary): string { + const successful = run.successful_attacks + ?? Math.round((run.objective_achieved_rate / 100) * run.completed_attacks) + if (run.planned_total_available === false) { + return `${successful}/${run.completed_attacks} known results` + } + if (run.completed_attacks === 0) { + return '0/0' + } + return `${successful}/${run.completed_attacks} (${run.objective_achieved_rate}%)` +} diff --git a/frontend/src/components/History/scenarioHistoryFilters.test.ts b/frontend/src/components/History/scenarioHistoryFilters.test.ts new file mode 100644 index 0000000000..414c2ca2a9 --- /dev/null +++ b/frontend/src/components/History/scenarioHistoryFilters.test.ts @@ -0,0 +1,57 @@ +import { + DEFAULT_SCENARIO_HISTORY_FILTERS, + SCENARIO_RUN_STATES, + scenarioHistoryFiltersFromSearchParams, + scenarioHistoryFiltersToSearchParams, +} from './scenarioHistoryFilters' + +describe('scenario history URL filters', () => { + it('round-trips repeated filters and label search text', () => { + const filters = { + scenarioNames: ['red.team', 'benchmark'], + statuses: ['IN_PROGRESS', 'FAILED'] as const, + operator: ['alice', 'bob'], + operation: ['nightly'], + otherLabels: ['team:security', 'team:safety'], + labelSearchText: 'team', + } + + const params = scenarioHistoryFiltersToSearchParams({ + ...filters, + statuses: [...filters.statuses], + }) + + expect(params.getAll('scenario')).toEqual(['red.team', 'benchmark']) + expect(params.getAll('status')).toEqual(['IN_PROGRESS', 'FAILED']) + expect(scenarioHistoryFiltersFromSearchParams(params)).toEqual({ + ...filters, + statuses: [...filters.statuses], + }) + }) + + it('ignores invalid run states without dropping valid filters', () => { + const params = new URLSearchParams('status=COMPLETED&status=UNKNOWN&operator=alice') + + expect(scenarioHistoryFiltersFromSearchParams(params)).toEqual({ + ...DEFAULT_SCENARIO_HISTORY_FILTERS, + statuses: ['COMPLETED'], + operator: ['alice'], + }) + }) + + it('round-trips every supported run state, including QUEUED', () => { + const filters = { + ...DEFAULT_SCENARIO_HISTORY_FILTERS, + statuses: [...SCENARIO_RUN_STATES], + } + + const params = scenarioHistoryFiltersToSearchParams(filters) + + expect(params.getAll('status')).toEqual(SCENARIO_RUN_STATES) + expect(scenarioHistoryFiltersFromSearchParams(params)).toEqual(filters) + }) + + it('omits empty filters from the URL', () => { + expect(scenarioHistoryFiltersToSearchParams(DEFAULT_SCENARIO_HISTORY_FILTERS).toString()).toBe('') + }) +}) diff --git a/frontend/src/components/History/scenarioHistoryFilters.ts b/frontend/src/components/History/scenarioHistoryFilters.ts new file mode 100644 index 0000000000..1cc6fda718 --- /dev/null +++ b/frontend/src/components/History/scenarioHistoryFilters.ts @@ -0,0 +1,59 @@ +import type { ScenarioRunState } from '@/types' + +export interface ScenarioHistoryFilters { + scenarioNames: string[] + statuses: ScenarioRunState[] + operator: string[] + operation: string[] + otherLabels: string[] + labelSearchText: string +} + +export const DEFAULT_SCENARIO_HISTORY_FILTERS: ScenarioHistoryFilters = { + scenarioNames: [], + statuses: [], + operator: [], + operation: [], + otherLabels: [], + labelSearchText: '', +} + +export const SCENARIO_RUN_STATES: readonly ScenarioRunState[] = [ + 'CREATED', + 'QUEUED', + 'IN_PROGRESS', + 'COMPLETED', + 'FAILED', + 'CANCELLED', +] + +const RUN_STATES = new Set(SCENARIO_RUN_STATES) + +export function scenarioHistoryFiltersFromSearchParams( + params: URLSearchParams, +): ScenarioHistoryFilters { + const statuses = params + .getAll('status') + .filter((status): status is ScenarioRunState => RUN_STATES.has(status)) + return { + scenarioNames: params.getAll('scenario'), + statuses, + operator: params.getAll('operator'), + operation: params.getAll('operation'), + otherLabels: params.getAll('label'), + labelSearchText: params.get('labelSearch') ?? '', + } +} + +export function scenarioHistoryFiltersToSearchParams( + filters: ScenarioHistoryFilters, +): URLSearchParams { + const params = new URLSearchParams() + for (const scenarioName of filters.scenarioNames) params.append('scenario', scenarioName) + for (const status of filters.statuses) params.append('status', status) + for (const operator of filters.operator) params.append('operator', operator) + for (const operation of filters.operation) params.append('operation', operation) + for (const label of filters.otherLabels) params.append('label', label) + if (filters.labelSearchText) params.set('labelSearch', filters.labelSearchText) + return params +} diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx index 4c3c772308..3d34013d48 100644 --- a/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx @@ -141,6 +141,35 @@ describe('ScenarioRunPage', () => { expect(screen.queryByRole('columnheader', { name: 'Actions' })).not.toBeInTheDocument() }) + it('renders contract-backed safe target and run configuration metadata', () => { + mockHookState(makeState({ + run: { + ...makeState().run!, + target: { + target_type: 'OpenAIChatTarget', + endpoint: 'https://example.test/v1', + model_name: 'gpt-4o', + identifier_hash: 'safe-hash', + }, + techniques_used: ['Technique One'], + datasets_used: ['harmbench'], + scenario_parameters: { max_turns: 5 }, + labels: { operator: 'alice' }, + pyrit_version: '0.10.0', + }, + })) + + renderPage() + + expect(screen.getByText('gpt-4o')).toBeInTheDocument() + expect(screen.getByText('https://example.test/v1')).toBeInTheDocument() + expect(screen.getByText('safe-hash')).toBeInTheDocument() + expect(screen.getByText('harmbench')).toBeInTheDocument() + expect(screen.getByText('max_turns: 5')).toBeInTheDocument() + expect(screen.getByText('operator: alice')).toBeInTheDocument() + expect(screen.getByText('0.10.0')).toBeInTheDocument() + }) + it('keeps legacy runs useful without misleading totals, ETA, or a progress bar', () => { mockHookState(makeState({ planComplete: false })) diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.tsx index 3a65a0907d..697b1a2f30 100644 --- a/frontend/src/components/Scenarios/ScenarioRunPage.tsx +++ b/frontend/src/components/Scenarios/ScenarioRunPage.tsx @@ -33,7 +33,7 @@ import { EyeRegular, StopRegular, } from '@fluentui/react-icons' -import { Link, useNavigate, useParams } from 'react-router' +import { Link, useLocation, useNavigate, useParams } from 'react-router' import { useScenarioRunProgress } from '@/hooks/useScenarioRunProgress' import { scenariosApi } from '@/services/api' @@ -64,6 +64,7 @@ const INTERACTIVE_ELEMENT_SELECTOR = 'a, button, input, select, textarea, [role= const RUN_BADGE_COLORS: Record = { CREATED: 'informative', + QUEUED: 'informative', IN_PROGRESS: 'brand', COMPLETED: 'success', FAILED: 'danger', @@ -88,6 +89,7 @@ interface ScenarioRunPageContentProps { function ScenarioRunPageContent({ scenarioResultId }: ScenarioRunPageContentProps) { const styles = useScenarioRunPageStyles() + const location = useLocation() const navigate = useNavigate() const { state, retry, applyRunSummary } = useScenarioRunProgress(scenarioResultId) const [nowMilliseconds, setNowMilliseconds] = useState(() => Date.now()) @@ -96,6 +98,19 @@ function ScenarioRunPageContent({ scenarioResultId }: ScenarioRunPageContentProp const [cancelError, setCancelError] = useState(null) const [selectedAttempt, setSelectedAttempt] = useState(null) const detailsTriggerRef = useRef(null) + const navigationState = location.state as { + fromScenarioHistory?: boolean + scenarioHistorySearch?: string + scenarioName?: string + } | null + const backPath = navigationState?.fromScenarioHistory + ? `/scenario-history${navigationState.scenarioHistorySearch ?? ''}` + : navigationState?.scenarioName + ? `/scenarios/${encodeURIComponent(navigationState.scenarioName)}` + : '/scenario-history' + const backLabel = navigationState?.scenarioName && !navigationState.fromScenarioHistory + ? 'Back to scenario' + : 'Back to scenario history' const overall = useMemo(() => getOverallProgress(state), [state]) const techniques = useMemo(() => getTechniqueRollups(state), [state]) @@ -149,8 +164,8 @@ function ScenarioRunPageContent({ scenarioResultId }: ScenarioRunPageContentProp return (
- - Back to scenarios + + {backLabel}
@@ -171,8 +186,8 @@ function ScenarioRunPageContent({ scenarioResultId }: ScenarioRunPageContentProp return (
- - Back to scenarios + + {backLabel}
@@ -191,8 +206,8 @@ function ScenarioRunPageContent({ scenarioResultId }: ScenarioRunPageContentProp return (
- - Back to scenarios + + {backLabel}
@@ -222,8 +237,8 @@ function ScenarioRunPageContent({ scenarioResultId }: ScenarioRunPageContentProp return (
- - Back to scenarios + + {backLabel}
@@ -278,8 +293,52 @@ function ScenarioRunPageContent({ scenarioResultId }: ScenarioRunPageContentProp Completed {run.completed_at ? formatTimestamp(run.completed_at) : 'Not yet'}
+ {run.target && ( +
+ Target + {run.target.model_name ?? run.target.target_type} + {run.target.target_type} +
+ )} + {run.pyrit_version && ( +
+ PyRIT version + {run.pyrit_version} +
+ )}
+
+
+ + Run configuration + + Persisted, secret-free settings for this run. +
+
+ 0 ? run.techniques_used?.join(', ') ?? '' : 'Unavailable'} + /> + 0 ? run.datasets_used?.join(', ') ?? '' : 'Unavailable'} + /> + + + {run.target?.endpoint && } + {run.target?.identifier_hash && ( + + )} +
+
+ {state.stale && ( @@ -663,6 +722,21 @@ interface MetricProps { readonly value: string } +interface ConfigurationItemProps { + readonly label: string + readonly value: string +} + +function ConfigurationItem({ label, value }: ConfigurationItemProps) { + const styles = useScenarioRunPageStyles() + return ( +
+ {label} + {value} +
+ ) +} + function Metric({ label, value }: MetricProps) { const styles = useScenarioRunPageStyles() return ( @@ -727,6 +801,7 @@ function formatTimestamp(timestamp: string): string { if (Number.isNaN(date.getTime())) { return 'Unavailable' } + return date.toLocaleString(undefined, { month: 'short', day: 'numeric', @@ -737,6 +812,16 @@ function formatTimestamp(timestamp: string): string { }) } +function formatConfiguration(value: Record): string { + const entries = Object.entries(value) + if (entries.length === 0) { + return 'None' + } + return entries + .map(([key, item]) => `${key}: ${typeof item === 'string' ? item : JSON.stringify(item)}`) + .join(', ') +} + function formatDuration(milliseconds: number): string { if (!Number.isFinite(milliseconds) || milliseconds < 0) { return 'Unavailable' diff --git a/frontend/src/components/Sidebar/Navigation.test.tsx b/frontend/src/components/Sidebar/Navigation.test.tsx index 1db3d96b53..0eed12499a 100644 --- a/frontend/src/components/Sidebar/Navigation.test.tsx +++ b/frontend/src/components/Sidebar/Navigation.test.tsx @@ -104,7 +104,7 @@ describe("Navigation", () => { ).toBeInTheDocument(); }); - it("places Scenarios immediately after Attack History without a history placeholder", () => { + it("renders the final primary navigation order", () => { renderWithProvider(); const navigation = screen.getByRole("navigation", { name: "Primary" }); const labels = within(navigation) @@ -116,10 +116,27 @@ describe("Navigation", () => { "Chat", "Attack History", "Scenarios", + "Scenario History", "Configuration", "Initializers", ]); - expect(screen.queryByRole("button", { name: "Scenario History" })).not.toBeInTheDocument(); + }); + + it("marks Scenario History current and navigates to its dedicated view", async () => { + const user = userEvent.setup(); + const onNavigate = jest.fn(); + renderWithProvider( + , + ); + + const button = screen.getByRole("button", { name: "Scenario History" }); + expect(button).toHaveAttribute("aria-current", "page"); + await user.click(button); + expect(onNavigate).toHaveBeenCalledWith("scenarioHistory"); }); it("calls onNavigate with 'scenarios' when the scenarios button is clicked", async () => { diff --git a/frontend/src/components/Sidebar/Navigation.tsx b/frontend/src/components/Sidebar/Navigation.tsx index d9c407b639..3c59de6651 100644 --- a/frontend/src/components/Sidebar/Navigation.tsx +++ b/frontend/src/components/Sidebar/Navigation.tsx @@ -15,6 +15,7 @@ import { HistoryRegular, PersonFeedbackRegular, ScriptRegular, + TableRegular, WrenchRegular, OpenRegular, WeatherMoonRegular, @@ -24,7 +25,14 @@ import { useTheme } from '../../hooks/useTheme' import type { ThemeMode } from '../../hooks/useTheme' import { useNavigationStyles } from './Navigation.styles' -export type ViewName = 'home' | 'chat' | 'history' | 'config' | 'initializers' | 'scenarios' +export type ViewName = + | 'home' + | 'chat' + | 'history' + | 'scenarioHistory' + | 'config' + | 'initializers' + | 'scenarios' interface NavigationProps { currentView: ViewName @@ -106,6 +114,17 @@ export default function Navigation({ currentView, onNavigate, onOpenFeedback }: onClick={() => onNavigate('scenarios')} /> +