From 4b81ccf5067add90a485489bbd14a5ab8ec1b133 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Wed, 12 Aug 2026 03:52:24 -0700 Subject: [PATCH] FEAT: Add scenario configuration sizing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 5d02c2d5-b499-4f78-a04d-03bffa750817 --- frontend/e2e/api.spec.ts | 96 +- frontend/e2e/scenario-history.spec.ts | 95 +- .../components/Parameters/ParameterField.tsx | 155 +- .../Scenarios/ScenarioCatalog.test.tsx | 4 +- .../Scenarios/ScenarioDetail.styles.ts | 28 + .../Scenarios/ScenarioDetail.test.tsx | 1473 ++++++++++++++++- .../components/Scenarios/ScenarioDetail.tsx | 1108 ++++++++++--- .../Scenarios/ScenarioFlow.test.tsx | 27 +- .../Scenarios/ScenarioRunEstimate.styles.ts | 130 +- .../Scenarios/ScenarioRunEstimate.test.tsx | 778 +++++++-- .../Scenarios/ScenarioRunEstimate.tsx | 661 ++++++-- .../Scenarios/scenarioAdaptiveCap.test.ts | 42 + .../Scenarios/scenarioAdaptiveCap.ts | 33 + .../Scenarios/scenarioDatasetCaps.test.ts | 70 + .../Scenarios/scenarioDatasetCaps.ts | 87 + .../Scenarios/scenarioTechniqueSets.test.ts | 74 + frontend/src/services/api.test.ts | 70 + frontend/src/services/api.ts | 8 + frontend/src/types/index.ts | 28 +- pyrit/analytics/technique_analysis.py | 115 +- .../backend/services/scenario_run_service.py | 23 +- pyrit/backend/services/scenario_service.py | 70 +- pyrit/models/__init__.py | 6 + pyrit/models/catalog/__init__.py | 12 + pyrit/models/catalog/scenario.py | 158 +- .../registry/components/scenario_registry.py | 47 +- pyrit/scenario/core/dataset_configuration.py | 130 +- pyrit/scenario/core/scenario.py | 9 +- pyrit/scenario/scenarios/adaptive/__init__.py | 6 + .../scenarios/adaptive/adaptive_scenario.py | 90 +- .../scenario/scenarios/adaptive/dispatcher.py | 55 +- .../adaptive/selectors/epsilon_greedy.py | 13 +- .../adaptive/selectors/technique_selector.py | 15 +- .../scenarios/adaptive/technique_identity.py | 72 + .../scenarios/adaptive/text_adaptive.py | 5 +- pyrit/scenario/scenarios/airt/jailbreak.py | 24 +- pyrit/scenario/scenarios/airt/psychosocial.py | 5 +- .../scenario/scenarios/garak/web_injection.py | 5 +- .../unit/analytics/test_technique_analysis.py | 85 +- .../unit/backend/test_scenario_run_service.py | 81 +- tests/unit/backend/test_scenario_service.py | 246 ++- tests/unit/models/test_scenario_catalog.py | 115 ++ tests/unit/registry/test_scenario_registry.py | 105 +- tests/unit/scenario/airt/test_jailbreak.py | 25 +- .../core/test_dataset_configuration.py | 99 ++ .../scenarios/adaptive/test_dispatcher.py | 41 +- .../scenarios/adaptive/test_epsilon_greedy.py | 29 +- .../adaptive/test_technique_identity.py | 23 + .../scenarios/adaptive/test_text_adaptive.py | 213 ++- .../test_default_run_size_estimates.py | 135 +- 50 files changed, 6110 insertions(+), 914 deletions(-) create mode 100644 frontend/src/components/Scenarios/scenarioAdaptiveCap.test.ts create mode 100644 frontend/src/components/Scenarios/scenarioAdaptiveCap.ts create mode 100644 frontend/src/components/Scenarios/scenarioDatasetCaps.test.ts create mode 100644 frontend/src/components/Scenarios/scenarioDatasetCaps.ts create mode 100644 frontend/src/components/Scenarios/scenarioTechniqueSets.test.ts create mode 100644 pyrit/scenario/scenarios/adaptive/technique_identity.py create mode 100644 tests/unit/scenario/scenarios/adaptive/test_technique_identity.py diff --git a/frontend/e2e/api.spec.ts b/frontend/e2e/api.spec.ts index d9a1670ef2..9b5556ef29 100644 --- a/frontend/e2e/api.spec.ts +++ b/frontend/e2e/api.spec.ts @@ -1,26 +1,31 @@ import { test, expect } from "@playwright/test"; +import type { APIRequestContext } from "@playwright/test"; // API tests go through the Vite dev server proxy (/api -> configured backend) // rather than hitting the backend directly, so they work as soon as // Playwright's webServer is ready. -test.describe("API Health Check", () => { - // The backend may still be starting when Vite is already up. - // Poll the health endpoint through the proxy until the backend is ready. - test.beforeAll(async ({ request }) => { - const maxWait = 30_000; - const interval = 1_000; - const start = Date.now(); - while (Date.now() - start < maxWait) { - try { - const resp = await request.get("/api/health", { timeout: 2_000 }); - if (resp.ok()) return; - } catch { - // Backend not ready yet +async function waitForBackend(request: APIRequestContext): Promise { + const maxWait = 30_000; + const interval = 1_000; + const start = Date.now(); + while (Date.now() - start < maxWait) { + try { + const response = await request.get("/api/health", { timeout: 2_000 }); + if (response.ok()) { + return; } - await new Promise((r) => setTimeout(r, interval)); + } catch { + // Backend not ready yet } - throw new Error("Backend did not become healthy within 30 seconds"); + await new Promise((resolve) => setTimeout(resolve, interval)); + } + throw new Error("Backend did not become healthy within 30 seconds"); +} + +test.describe("API Health Check", () => { + test.beforeAll(async ({ request }) => { + await waitForBackend(request); }); test("should have healthy backend API @seeded", async ({ request }) => { @@ -38,24 +43,12 @@ test.describe("API Health Check", () => { const data = await response.json(); expect(data).toBeDefined(); }); + }); test.describe("Targets API", () => { test.beforeAll(async ({ request }) => { - // Wait for backend readiness - const maxWait = 30_000; - const interval = 1_000; - const start = Date.now(); - while (Date.now() - start < maxWait) { - try { - const resp = await request.get("/api/health", { timeout: 2_000 }); - if (resp.ok()) return; - } catch { - // Backend not ready yet - } - await new Promise((r) => setTimeout(r, interval)); - } - throw new Error("Backend did not become healthy within 30 seconds"); + await waitForBackend(request); }); test("should list targets @seeded", async ({ request }) => { @@ -103,19 +96,7 @@ test.describe("Targets API", () => { test.describe("Attacks API", () => { test.beforeAll(async ({ request }) => { - const maxWait = 30_000; - const interval = 1_000; - const start = Date.now(); - while (Date.now() - start < maxWait) { - try { - const resp = await request.get("/api/health", { timeout: 2_000 }); - if (resp.ok()) return; - } catch { - // Backend not ready yet - } - await new Promise((r) => setTimeout(r, interval)); - } - throw new Error("Backend did not become healthy within 30 seconds"); + await waitForBackend(request); }); test("should list attacks @seeded", async ({ request }) => { @@ -124,6 +105,37 @@ test.describe("Attacks API", () => { }); }); +test.describe("Scenarios API", () => { + test.beforeAll(async ({ request }) => { + await waitForBackend(request); + }); + + test("should expose scenario catalog details and queue state @seeded", async ({ request }) => { + test.setTimeout(90_000); + const catalogResponse = await request.get("/api/scenarios/catalog?limit=200"); + expect(catalogResponse.ok()).toBe(true); + const catalog = await catalogResponse.json(); + expect(catalog.items.length).toBeGreaterThan(0); + + const scenarioName = catalog.items[0].scenario_name as string; + const detailResponse = await request.get(`/api/scenarios/catalog/${encodeURIComponent(scenarioName)}`); + expect(detailResponse.ok()).toBe(true); + const detail = await detailResponse.json(); + expect(detail.scenario_name).toBe(scenarioName); + expect(detail.dataset_size_limit).toEqual(expect.objectContaining({ + default_scope: expect.any(String), + override_scope: expect.any(String), + })); + + const queueResponse = await request.get("/api/scenarios/runs/queue"); + expect(queueResponse.ok()).toBe(true); + await expect(queueResponse.json()).resolves.toEqual(expect.objectContaining({ + revision: expect.any(Number), + queued: expect.any(Array), + })); + }); +}); + test.describe("Error Handling", () => { test("should display UI when backend is slow", async ({ page }) => { // Intercept and delay API calls diff --git a/frontend/e2e/scenario-history.spec.ts b/frontend/e2e/scenario-history.spec.ts index dd2beff582..0830548d8b 100644 --- a/frontend/e2e/scenario-history.spec.ts +++ b/frontend/e2e/scenario-history.spec.ts @@ -36,6 +36,9 @@ const configuredEstimate = { version: 1, status: "exact", total_attack_count: 8, + minimum_attack_count: 8, + maximum_attack_count: 8, + condition: null, components: [{ label: "Prompt sending", count: 8, @@ -49,6 +52,7 @@ const configuredEstimate = { note: null, }], datasets: [datasetSummary], + adaptive_details: null, note: "The backend total is authoritative.", retries_included: false, }; @@ -68,6 +72,11 @@ const catalogScenario = { }, all_techniques: ["prompt_sending", "jailbreak_system_prompt", "flip"], default_datasets: ["harmbench"], + dataset_size_limit: { + default_scope: "per_dataset", + default_count: 4, + override_scope: "per_dataset", + }, default_dataset_summaries: [datasetSummary], baseline_policy: "enabled", include_baseline_by_default: false, @@ -104,6 +113,9 @@ const catalogScenario = { version: 1, status: "exact", total_attack_count: 16, + minimum_attack_count: 16, + maximum_attack_count: 16, + condition: null, components: [{ label: "Default attacks", count: 16, @@ -116,6 +128,7 @@ const catalogScenario = { note: null, }], datasets: [datasetSummary], + adaptive_details: null, note: "Retries and internal turns are excluded.", retries_included: false, }, @@ -197,6 +210,8 @@ const progressAttempt = { timestamp: "2026-08-07T00:00:30Z", total_retries: 1, retries: [], + result_kind: "attack", + technique_name: "prompt_sending", }; interface ScenarioMocks { @@ -236,6 +251,14 @@ async function mockScenarioAPIs(page: Page): Promise { }); }); + await page.route(/\/api\/datasets(?:\?|$)/, async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ items: [{ name: "harmbench" }] }), + }); + }); + await page.route(new RegExp(`/api/scenarios/catalog/${SCENARIO_NAME.replace(".", "\\.")}/estimate$`), async (route) => { const request = route.request().postDataJSON() as Record; estimateRequests.push(request); @@ -396,15 +419,20 @@ async function mockScenarioAPIs(page: Page): Promise { async function configurePromptSendingRun(page: Page): Promise { await expect(page.getByTestId("scenario-target-select")).toHaveValue("test-target"); - await page.getByTestId("technique-prompt_sending").click(); + await page.getByTestId("technique-mode-custom").click(); + await expect(page.getByTestId("technique-prompt_sending")).toBeChecked(); + await expect(page.getByTestId("technique-jailbreak_system_prompt")).toBeChecked(); + await page.getByTestId("technique-jailbreak_system_prompt").click(); await page.getByTestId("scenario-param-num_jailbreaks").fill("2"); await page.getByTestId("scenario-param-num_jailbreak_attempts").fill("1"); await expect(page.getByTestId("baseline-checkbox")).not.toBeChecked(); - await expect(page.getByText("8 planned attacks")).toBeVisible(); + await expect(page.getByRole("group", { + name: "1 technique multiplied by 4 objectives multiplied by 2 jailbreak templates multiplied by 1 attempt equals 8 planned attacks.", + })).toBeVisible(); } test.describe("Scenario catalog, history, and live run routing", () => { - test("renders the semantic catalog, full metadata, safe MyST, and both sidebar destinations", async ({ page }) => { + test("opens the Configure page from the semantic launch index with complete safe metadata", async ({ page }) => { await mockScenarioAPIs(page); await page.goto("/scenarios"); @@ -423,17 +451,52 @@ test.describe("Scenario catalog, history, and live run routing", () => { ]); await expect(page.getByTitle("Scenarios")).toHaveAttribute("aria-current", "page"); await expect(page.getByRole("table", { name: "Registered scenarios" })).toBeVisible(); - await expect(page.getByRole("columnheader", { name: "Default run size" })).toBeVisible(); + await expect(page.getByRole("columnheader")).toHaveText([ + "Scenario / purpose", + "Configure", + "Default dataset size", + "Default techniques", + "Default run size", + ]); const row = page.getByTestId(`scenario-card-${SCENARIO_NAME}`); - await row.getByRole("button", { name: "Configure run" }).click(); + const cells = row.getByRole("cell"); + await expect(cells).toHaveCount(5); + const configureButton = cells.nth(1).getByRole("button", { name: "Configure run" }); + await expect(configureButton).toBeVisible(); + const [scenarioCellBox, configureCellBox, datasetCellBox] = await Promise.all([ + cells.nth(0).boundingBox(), + cells.nth(1).boundingBox(), + cells.nth(2).boundingBox(), + ]); + expect(scenarioCellBox).not.toBeNull(); + expect(configureCellBox).not.toBeNull(); + expect(datasetCellBox).not.toBeNull(); + expect(configureCellBox!.x).toBeGreaterThan(scenarioCellBox!.x); + expect(configureCellBox!.x).toBeLessThan(datasetCellBox!.x); + await expect(page.getByRole("button", { name: /show details|hide details/i })).toHaveCount(0); + + await configureButton.click(); await expect(page).toHaveURL(`/scenarios/${SCENARIO_NAME}`); - await expect(page.getByRole("heading", { name: SCENARIO_NAME, level: 1 })).toBeVisible(); + await expect(page.getByText("Jailbreak · v4")).toBeVisible(); const description = page.getByTestId("scenario-detail-description"); await expect(description.getByText("dataset")).toHaveCSS("font-weight", /^(600|700)$/); await expect(description.locator("code").filter({ hasText: "num_jailbreaks" })).toBeVisible(); await expect(description.locator("img")).toHaveCount(0); await expect(description).toContainText(RAW_IMAGE_HTML); + await expect(page.getByRole("radio", { name: /Recommended \(default\).*2 techniques/ })).toBeChecked(); + await expect(page.getByRole("radio", { name: /Easy.*1 technique/ })).toBeVisible(); + await expect(page.getByRole("radio", { name: "Custom" })).toBeVisible(); + const members = page.getByTestId("selected-technique-set-members"); + await expect(members.getByText("prompt_sending")).toBeVisible(); + await expect(members.getByText("jailbreak_system_prompt")).toBeVisible(); + const preview = page.getByRole("complementary", { name: "Run preview" }); + await expect(preview.getByText("Jailbreak templates: 2")).toBeVisible(); + await expect(preview.getByRole("group", { + name: "2 techniques multiplied by 4 objectives multiplied by 2 jailbreak templates equals 16 planned attacks.", + })).toBeVisible(); + await expect(page.getByText("Include direct baseline comparison")).toBeVisible(); + await expect(page.getByText(/Also send each selected objective directly/)).toBeVisible(); await page.getByTitle("Scenario History").click(); await expect(page).toHaveURL("/scenario-history"); @@ -471,7 +534,9 @@ test.describe("Scenario catalog, history, and live run routing", () => { const requests = mocks.getEstimateRequests(); return requests[requests.length - 1]; }).toEqual(expectedEstimateRequest); - await expect(preview.getByText("Prompt sending: 2 jailbreak templates × 4 selected seed groups × 1 concrete techniques × 1 attempts = 8")).toBeVisible(); + await expect(preview.getByRole("group", { + name: "1 technique multiplied by 4 objectives multiplied by 2 jailbreak templates multiplied by 1 attempt equals 8 planned attacks.", + })).toBeVisible(); await expect(preview).not.toContainText("context_compliance"); await page.getByTestId("launch-scenario-btn").click(); @@ -508,7 +573,21 @@ test.describe("Scenario catalog, history, and live run routing", () => { const client = await page.context().newCDPSession(page); await client.send("Emulation.setTouchEmulationEnabled", { enabled: true, maxTouchPoints: 1 }); await page.setViewportSize({ width: 390, height: 844 }); - await page.goto(`/scenarios/${SCENARIO_NAME}`); + await page.goto("/scenarios"); + const catalogRow = page.getByTestId(`scenario-card-${SCENARIO_NAME}`); + const configureButton = catalogRow.getByRole("button", { name: "Configure run" }); + await expect(catalogRow).toBeVisible(); + expect(await catalogRow.getByRole("cell").allInnerTexts()).toEqual([ + expect.stringContaining("Scenario / purpose"), + expect.stringContaining("Configure"), + expect.stringContaining("Default dataset size"), + expect.stringContaining("Default techniques"), + expect.stringContaining("Default run size"), + ]); + expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(390); + expect((await configureButton.boundingBox())?.height).toBeGreaterThanOrEqual(44); + await configureButton.press("Enter"); + await expect(page).toHaveURL(`/scenarios/${SCENARIO_NAME}`); await configurePromptSendingRun(page); const formBox = await page.getByRole("form", { name: "Scenario run configuration" }).boundingBox(); diff --git a/frontend/src/components/Parameters/ParameterField.tsx b/frontend/src/components/Parameters/ParameterField.tsx index ccc498d3e9..39f6d3edea 100644 --- a/frontend/src/components/Parameters/ParameterField.tsx +++ b/frontend/src/components/Parameters/ParameterField.tsx @@ -1,8 +1,11 @@ +import { type ClipboardEvent, type KeyboardEvent, useRef } from 'react' + import { Checkbox, Field, Input, Select, + type FieldProps, } from '@fluentui/react-components' import type { Parameter } from '@/types' @@ -15,10 +18,27 @@ export interface ParameterFieldProps { value: ParameterFormValue disabled: boolean onChange: (name: string, value: ParameterFormValue) => void + displayLabel?: string + displayHint?: string + validationState?: FieldProps['validationState'] + validationMessage?: string + numberMin?: number + numberMax?: number + numberStep?: number + numberWholeOnly?: boolean + onRejectedNumberInput?: ( + name: string, + reason: RejectedNumberInputReason, + retainedValue: string, + ) => void /** Prefix for `data-testid` attributes. Defaults to `'param'` (e.g. `param-`). */ testIdPrefix?: string } +export type RejectedNumberInputReason = 'format' | 'below-min' | 'above-max' + +const BLOCKED_WHOLE_NUMBER_KEYS = new Set(['-', '+', '.', 'e', 'E']) + /** * Renders the appropriate Fluent UI control for a declared {@link Parameter}, * driven by {@link getParameterControlKind}. Shared by every dynamic @@ -34,17 +54,30 @@ export default function ParameterField({ value, disabled, onChange, + displayLabel, + displayHint, + validationState, + validationMessage, + numberMin, + numberMax, + numberStep, + numberWholeOnly = false, + onRejectedNumberInput, testIdPrefix = 'param', }: ParameterFieldProps) { const styles = useParameterFieldStyles() + const rejectedNumberSequenceRef = useRef(false) + const editSessionStartValueRef = useRef('') const kind = getParameterControlKind(parameter) - const label = parameter.required ? `${parameter.name} *` : parameter.name + const baseLabel = displayLabel ?? parameter.name + const label = parameter.required ? `${baseLabel} *` : baseLabel + const fieldHint = displayHint ?? parameter.description ?? undefined const testId = `${testIdPrefix}-${parameter.name}` if (kind === 'boolean') { const current = value === 'true' || value === 'false' ? value : '' return ( - + { + rejectedNumberSequenceRef.current = reason !== 'above-max' + if (reason === 'above-max' && numberMax !== undefined) { + editSessionStartValueRef.current = String(numberMax) + } + onRejectedNumberInput?.(parameter.name, reason, retainedValue) + } + const handleNumberKeyDown = (event: KeyboardEvent): void => { + if (!numberWholeOnly) { + return + } + if (event.key === 'Backspace' || event.key === 'Delete') { + rejectedNumberSequenceRef.current = false + return + } + if ( + event.key === 'ArrowDown' + && numberMin !== undefined + && Number(stringValue) <= numberMin + ) { + event.preventDefault() + event.stopPropagation() + return + } + if ( + event.key === 'ArrowUp' + && numberMax !== undefined + && Number(stringValue) >= numberMax + ) { + event.preventDefault() + event.stopPropagation() + return + } + if (BLOCKED_WHOLE_NUMBER_KEYS.has(event.key)) { + event.preventDefault() + event.stopPropagation() + rejectNumberInput('format', editSessionStartValueRef.current) + return + } + if (rejectedNumberSequenceRef.current && event.key.length === 1) { + event.preventDefault() + event.stopPropagation() + } + } + const handleNumberPaste = (event: ClipboardEvent): void => { + if (!numberWholeOnly) { + return + } + const pastedValue = event.clipboardData.getData('text') + if (!/^\d+$/.test(pastedValue)) { + event.preventDefault() + rejectNumberInput('format', stringValue) + return + } + if (numberMax !== undefined && Number(pastedValue) > numberMax) { + event.preventDefault() + rejectNumberInput('above-max', stringValue) + return + } + if (numberMin !== undefined && Number(pastedValue) < numberMin) { + event.preventDefault() + rejectNumberInput('below-min', stringValue) + } + } + const handleInputChange = (nextValue: string): void => { + if (numberWholeOnly && nextValue !== '' && !/^\d+$/.test(nextValue)) { + rejectNumberInput('format', stringValue) + return + } + if (numberMax !== undefined && nextValue !== '' && Number(nextValue) > numberMax) { + rejectNumberInput('above-max', stringValue) + return + } + if (numberMin !== undefined && nextValue !== '' && Number(nextValue) < numberMin) { + rejectNumberInput('below-min', stringValue) + return + } + rejectedNumberSequenceRef.current = false + if (nextValue === '') { + editSessionStartValueRef.current = '' + } + onChange(parameter.name, nextValue) + } return ( - + onChange(parameter.name, data.value)} + onKeyDown={kind === 'number' ? handleNumberKeyDown : undefined} + onPaste={kind === 'number' ? handleNumberPaste : undefined} + onFocus={() => { + editSessionStartValueRef.current = stringValue + }} + onBlur={() => { + rejectedNumberSequenceRef.current = false + }} + onChange={(_, data) => handleInputChange(data.value)} data-testid={testId} /> diff --git a/frontend/src/components/Scenarios/ScenarioCatalog.test.tsx b/frontend/src/components/Scenarios/ScenarioCatalog.test.tsx index 094176536b..3644c1e73b 100644 --- a/frontend/src/components/Scenarios/ScenarioCatalog.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioCatalog.test.tsx @@ -398,7 +398,7 @@ describe('ScenarioCatalog', () => { expect(within(row).queryByText('6 objectives')).not.toBeInTheDocument() }) - it('shows adaptive progress objectives together with the underlying attempt bound', async () => { + it('shows adaptive planned attacks together with the technique attempt bound', async () => { mockListCatalog.mockResolvedValueOnce({ items: [ makeScenario({ @@ -450,7 +450,7 @@ describe('ScenarioCatalog', () => { render() const row = await screen.findByTestId('scenario-card-adaptive.text_adaptive') - expect(within(row).getByText('up to 63 attack attempts · 21–42 progress units')).toBeInTheDocument() + expect(within(row).getByText('21–42 planned attacks · up to 42 technique attempts')).toBeInTheDocument() expect(within(row).queryByText(/objective envelope/i)).not.toBeInTheDocument() }) diff --git a/frontend/src/components/Scenarios/ScenarioDetail.styles.ts b/frontend/src/components/Scenarios/ScenarioDetail.styles.ts index f7521d1f0d..f15d035b65 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.styles.ts +++ b/frontend/src/components/Scenarios/ScenarioDetail.styles.ts @@ -38,6 +38,9 @@ export const useScenarioDetailStyles = makeStyles({ flexDirection: 'column', gap: tokens.spacingVerticalXS, }, + scenarioMetadata: { + color: tokens.colorNeutralForeground3, + }, description: { maxWidth: '75ch', color: tokens.colorNeutralForeground2, @@ -70,6 +73,9 @@ export const useScenarioDetailStyles = makeStyles({ }, control: { ...mobileTouchTargetHeight, + width: '100%', + minWidth: 0, + maxWidth: '100%', '& > select': { [TOUCH_INPUT_QUERY]: { minHeight: MINIMUM_TOUCH_TARGET_SIZE, @@ -100,6 +106,28 @@ export const useScenarioDetailStyles = makeStyles({ gap: tokens.spacingVerticalXS, paddingLeft: tokens.spacingHorizontalM, }, + datasetPickerHeader: { + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalM, + }, + datasetList: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXXS, + maxHeight: '18rem', + overflowY: 'auto', + padding: `${tokens.spacingVerticalXS} ${tokens.spacingHorizontalS}`, + border: `1px solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusMedium, + backgroundColor: tokens.colorNeutralBackground1, + }, + datasetEmptyState: { + padding: tokens.spacingVerticalM, + color: tokens.colorNeutralForeground3, + }, hint: { color: tokens.colorNeutralForeground3, }, diff --git a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx index 4fcfc96fb9..06e25b9a2f 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx @@ -3,10 +3,11 @@ import userEvent from '@testing-library/user-event' import { FluentProvider, webLightTheme } from '@fluentui/react-components' import { MemoryRouter, Route, Routes } from 'react-router' -import { scenariosApi, targetsApi } from '@/services/api' +import { datasetsApi, scenariosApi, targetsApi } from '@/services/api' import type { RegisteredScenario, ScenarioDefaultRunSizeEstimate, + ScenarioRunSizeEstimateRequest, TargetInstance, } from '@/types' @@ -21,12 +22,24 @@ jest.mock('@/services/api', () => ({ targetsApi: { listTargets: jest.fn(), }, + datasetsApi: { + listDatasets: jest.fn(), + }, })) const mockGetScenario = scenariosApi.getScenario as jest.Mock const mockEstimateRun = scenariosApi.estimateRun as jest.Mock const mockStartRun = scenariosApi.startRun as jest.Mock const mockListTargets = targetsApi.listTargets as jest.Mock +const mockListDatasets = datasetsApi.listDatasets as jest.Mock +const REMOVED_NORMAL_ESTIMATE_LABELS = new RegExp( + [ + ['Run', 'size', 'calculated'].join(' '), + ['Final', 'count', 'set', 'at', 'launch'].join(' '), + ].join('|'), + 'i', +) +const CORRECT_HIGHLIGHTED_SETTING_MESSAGE = 'Correct the highlighted setting to calculate this run.' const mockNavigate = jest.fn() const RAW_IMAGE_HTML = ['<', 'img src=x onerror="alert(1)">'].join('') @@ -52,6 +65,11 @@ function makeScenario(overrides: Partial = {}): RegisteredSc ), all_techniques: ['default_technique', 'crescendo'], default_datasets: ['harmbench'], + dataset_size_limit: { + default_scope: 'none', + default_count: null, + override_scope: 'per_dataset', + }, default_dataset_summaries: [], baseline_policy: 'enabled', include_baseline_by_default: true, @@ -60,8 +78,12 @@ function makeScenario(overrides: Partial = {}): RegisteredSc version: 1, status: 'unavailable', total_attack_count: null, + minimum_attack_count: null, + maximum_attack_count: null, + condition: null, components: [], datasets: [], + adaptive_details: null, note: 'Default sizing is unavailable.', retries_included: false, }, @@ -89,6 +111,9 @@ function makeEstimate( version: 1, status, total_attack_count: total, + minimum_attack_count: null, + maximum_attack_count: null, + condition: null, components: total === null ? [] : [ @@ -101,11 +126,122 @@ function makeEstimate( }, ], datasets: [], + adaptive_details: null, note: null, retries_included: false, } } +function makeAdaptiveScenario(): RegisteredScenario { + const defaultMembers = ['role_play_movie_script', 'many_shot'] + const defaultDatasets = [ + 'airt_hate', + 'airt_fairness', + 'airt_violence', + 'airt_sexual', + 'airt_harassment', + 'airt_misinformation', + 'airt_leakage', + ] + const aggregateTechniqueExpansions = { + default: defaultMembers, + all: [...defaultMembers, ...Array.from({ length: 15 }, (_, index) => `all_member_${index + 1}`)], + core: Array.from({ length: 14 }, (_, index) => `core_member_${index + 1}`), + extra: Array.from({ length: 3 }, (_, index) => `extra_member_${index + 1}`), + light: Array.from({ length: 9 }, (_, index) => `light_member_${index + 1}`), + multi_turn: Array.from({ length: 5 }, (_, index) => `multi_turn_member_${index + 1}`), + single_turn: Array.from({ length: 12 }, (_, index) => `single_turn_member_${index + 1}`), + } + return makeScenario({ + scenario_name: 'adaptive.text_adaptive', + scenario_type: 'TextAdaptive', + default_technique: 'default', + default_techniques: defaultMembers, + default_datasets: defaultDatasets, + dataset_size_limit: { + default_scope: 'per_dataset', + default_count: 4, + override_scope: 'per_dataset', + }, + aggregate_techniques: ['all', 'default', 'core', 'extra', 'light', 'multi_turn', 'single_turn'], + aggregate_technique_expansions: aggregateTechniqueExpansions, + all_techniques: [...new Set(Object.values(aggregateTechniqueExpansions).flat())], + supported_parameters: [ + { + name: 'max_attempts_per_objective', + type_name: 'int', + required: false, + default: 3, + choices: null, + is_list: false, + }, + ], + }) +} + +function makeAdaptiveEstimateForRequest( + scenario: RegisteredScenario, + request: ScenarioRunSizeEstimateRequest, +): ScenarioDefaultRunSizeEstimate { + const selectedSet = request.techniques?.[0] ?? 'default' + const selectedCandidateCount = scenario.aggregate_technique_expansions[selectedSet]?.length ?? 0 + const candidateCount = selectedSet === 'core' ? 5 : selectedCandidateCount + const configuredMax = Number(request.scenario_params?.max_attempts_per_objective ?? 3) + const perObjective = Math.min(candidateCount, configuredMax) + const includeBaseline = request.include_baseline !== false + return { + ...makeEstimate(null), + minimum_attack_count: includeBaseline ? 21 : null, + maximum_attack_count: includeBaseline ? 42 : 21, + components: [ + ...(includeBaseline ? [{ + label: 'Baseline', + count: 21, + factors: [{ label: 'objectives', count: 21 }], + is_baseline: true, + condition: null, + note: null, + }] : []), + { + label: 'Adaptive objectives', + count: 21, + factors: [{ label: 'compatible objectives', count: 21 }], + is_baseline: false, + condition: null, + note: null, + }, + ], + adaptive_details: { + objective_count: 21, + selected_candidate_technique_count: selectedCandidateCount, + candidate_technique_count: candidateCount, + max_attempts_per_objective: configuredMax, + techniques_per_objective_upper_bound: perObjective, + technique_attempt_count_upper_bound: 21 * perObjective, + stop_on_first_success: true, + compatibility_may_reduce_attempts: true, + }, + } +} + +function makeFullyCompatibleAdaptiveEstimateForRequest( + scenario: RegisteredScenario, + request: ScenarioRunSizeEstimateRequest, +): ScenarioDefaultRunSizeEstimate { + const estimate = makeAdaptiveEstimateForRequest(scenario, request) + const adaptiveDetails = estimate.adaptive_details + if (!adaptiveDetails) { + throw new Error('Expected Adaptive estimate details.') + } + const candidateCount = adaptiveDetails.selected_candidate_technique_count ?? 0 + const configuredMaximum = adaptiveDetails.max_attempts_per_objective + adaptiveDetails.candidate_technique_count = candidateCount + adaptiveDetails.techniques_per_objective_upper_bound = Math.min(candidateCount, configuredMaximum) + adaptiveDetails.technique_attempt_count_upper_bound = + adaptiveDetails.objective_count * adaptiveDetails.techniques_per_objective_upper_bound + return estimate +} + async function flushRenderedPromises(): Promise { await act(async () => { await Promise.resolve() @@ -155,11 +291,20 @@ describe('ScenarioDetail', () => { mockGetScenario.mockReset() mockEstimateRun.mockReset() mockListTargets.mockReset() + mockListDatasets.mockReset() mockStartRun.mockReset() mockListTargets.mockResolvedValue({ items: [makeTarget('target-a'), makeTarget('target-b')], pagination: { limit: 200, has_more: false }, }) + mockListDatasets.mockResolvedValue({ + items: [ + { name: 'harmbench' }, + { name: 'ds_a' }, + { name: 'ds_b' }, + { name: 'xstest' }, + ], + }) mockGetScenario.mockResolvedValue(makeScenario()) mockEstimateRun.mockReturnValue(new Promise(() => {})) mockStartRun.mockResolvedValue({ scenario_result_id: 'sr-default' }) @@ -316,15 +461,15 @@ describe('ScenarioDetail', () => { resolveSecond(makeEstimate(12)) await flushRenderedPromises() const preview = screen.getByRole('complementary', { name: 'Run preview' }) - expect(within(preview).getByText('12 planned attacks')).toBeInTheDocument() + expect(within(preview).getByRole('group', { name: '12 planned attacks.' })).toBeInTheDocument() resolveFirst(makeEstimate(8)) await flushRenderedPromises() - expect(within(preview).getByText('12 planned attacks')).toBeInTheDocument() - expect(within(preview).queryByText('8 planned attacks')).not.toBeInTheDocument() + expect(within(preview).getByRole('group', { name: '12 planned attacks.' })).toBeInTheDocument() + expect(within(preview).queryByRole('group', { name: '8 planned attacks.' })).not.toBeInTheDocument() }) - it('keeps the last good estimate and entered state after a transient preview failure', async () => { + it('clears prior arithmetic and keeps entered state after a transient preview failure', async () => { jest.useFakeTimers() const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) mockEstimateRun @@ -338,7 +483,7 @@ describe('ScenarioDetail', () => { await flushRenderedPromises() await advanceTimers(300) await flushRenderedPromises() - expect(screen.getByText('8 planned attacks')).toBeInTheDocument() + expect(screen.getByRole('group', { name: '8 planned attacks.' })).toBeInTheDocument() await user.selectOptions(screen.getByTestId('scenario-target-select'), 'target-b') await advanceTimers(300) @@ -346,11 +491,44 @@ describe('ScenarioDetail', () => { const preview = screen.getByRole('complementary', { name: 'Run preview' }) expect(within(preview).getByText('target-b')).toBeInTheDocument() - expect(within(preview).getByText('Previous estimate')).toBeInTheDocument() - expect(within(preview).getByText('8 planned attacks')).toBeInTheDocument() + expect(within(preview).getByText('Run size couldn’t be updated.')).toBeInTheDocument() + expect(within(preview).queryByRole('group', { name: '8 planned attacks.' })).not.toBeInTheDocument() expect(within(preview).getByText('Preview service unavailable')).toBeInTheDocument() expect(screen.getByTestId('scenario-target-select')).toHaveValue('target-b') - expect(screen.getByTestId('launch-scenario-btn')).not.toBeDisabled() + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + }) + + it('hides stale arithmetic and blocks launch after a configuration request error', async () => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + mockEstimateRun + .mockResolvedValueOnce(makeEstimate(8)) + .mockRejectedValueOnce({ + isAxiosError: true, + response: { + status: 400, + data: { + detail: "Scenario 'adaptive.text_adaptive' does not support overriding dataset names.", + }, + }, + }) + renderDetail('/scenarios/foundry.red_team_agent') + await flushRenderedPromises() + await advanceTimers(300) + await flushRenderedPromises() + expect(screen.getByRole('group', { name: '8 planned attacks.' })).toBeInTheDocument() + + await user.selectOptions(screen.getByTestId('scenario-target-select'), 'target-b') + await advanceTimers(300) + await flushRenderedPromises() + + const preview = screen.getByRole('complementary', { name: 'Run preview' }) + expect(within(preview).getByText('Run size couldn’t be updated.')).toBeInTheDocument() + expect(within(preview).queryByTestId('run-calculation')).not.toBeInTheDocument() + expect(within(preview).getByText( + "Scenario 'adaptive.text_adaptive' does not support overriding dataset names.", + )).toBeInTheDocument() + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() }) it('does not request a preview while the custom technique selection is empty', async () => { @@ -359,7 +537,7 @@ describe('ScenarioDetail', () => { renderDetail('/scenarios/foundry.red_team_agent') await flushRenderedPromises() - await user.click(screen.getByTestId('technique-crescendo')) + await user.click(screen.getByTestId('technique-mode-custom')) await user.click(screen.getByTestId('technique-crescendo')) await advanceTimers(300) @@ -369,7 +547,7 @@ describe('ScenarioDetail', () => { .toBeInTheDocument() }) - it('renders a backend conditional estimate without inventing a total', async () => { + it('renders an unknown conditional estimate without inventing a total', async () => { jest.useFakeTimers() mockEstimateRun.mockResolvedValue(makeEstimate(null)) renderDetail('/scenarios/foundry.red_team_agent') @@ -378,14 +556,16 @@ describe('ScenarioDetail', () => { await flushRenderedPromises() const preview = screen.getByRole('complementary', { name: 'Run preview' }) - expect(within(preview).getByText('Conditional estimate')).toBeInTheDocument() - expect(within(preview).getByText('Total depends on configuration')).toBeInTheDocument() + expect(within(preview).getByText('Run size is confirmed at launch.')).toBeInTheDocument() + expect(within(preview).queryByText(REMOVED_NORMAL_ESTIMATE_LABELS)).not.toBeInTheDocument() expect(within(preview).queryByText(/planned attacks/)).not.toBeInTheDocument() }) it('renders MyST literals through the shared safe Markdown renderer', async () => { mockGetScenario.mockResolvedValue( makeScenario({ + scenario_type: 'Jailbreak', + scenario_version: 4, description: 'Configure this scenario.', description_markdown: `Set \`\`num_jailbreaks\`\`.\n\n${RAW_IMAGE_HTML}unsafe`, }), @@ -393,6 +573,7 @@ describe('ScenarioDetail', () => { renderDetail('/scenarios/foundry.red_team_agent') const description = await screen.findByTestId('scenario-detail-description') + expect(screen.getByText('Jailbreak · v4')).toBeInTheDocument() expect(within(description).getByText('num_jailbreaks').tagName).toBe('CODE') expect(screen.queryByRole('img')).not.toBeInTheDocument() expect( @@ -405,7 +586,23 @@ describe('ScenarioDetail', () => { await screen.findByTestId('scenario-target-select') expect(screen.getByTestId('technique-default_technique')).toBeChecked() - expect(screen.getByTestId('technique-crescendo')).not.toBeChecked() + expect(screen.queryByRole('group', { name: 'Individual techniques' })).not.toBeInTheDocument() + expect(screen.getByTestId('selected-technique-set-members')).toHaveTextContent('crescendo') + }) + + it('marks a named technique set as the scenario default', async () => { + mockGetScenario.mockResolvedValue( + makeScenario({ + default_technique: 'easy', + default_techniques: ['crescendo'], + aggregate_techniques: ['easy'], + aggregate_technique_expansions: { easy: ['crescendo'] }, + }), + ) + + renderDetail('/scenarios/foundry.red_team_agent') + + expect(await screen.findByLabelText('Easy (default) — 1 technique')).toBeChecked() }) it('shows catalog-provided aggregate members before the configured estimate resolves', async () => { @@ -428,10 +625,740 @@ describe('ScenarioDetail', () => { expect(within(preview).getByText( 'Resolves to prompt_sending, jailbreak_system_prompt', )).toBeInTheDocument() - expect(within(preview).getByText('Loading backend run estimate...')).toBeInTheDocument() + expect(within(preview).getByText('Calculating planned attacks...')).toBeInTheDocument() + }) + + it('renders the initial Adaptive conditional estimate instead of an unavailable exact total', async () => { + mockGetScenario.mockResolvedValue(makeAdaptiveScenario()) + mockEstimateRun.mockResolvedValue({ + version: 1, + status: 'conditional', + total_attack_count: null, + minimum_attack_count: 21, + maximum_attack_count: 42, + condition: null, + components: [ + { + label: 'Baseline', + count: 21, + factors: [{ label: 'selected logical seed groups', count: 21 }], + is_baseline: true, + condition: null, + note: null, + }, + { + label: 'Adaptive attack envelopes', + count: 21, + factors: [{ label: 'compatible logical seed groups', count: 21 }], + is_baseline: false, + condition: null, + note: null, + }, + ], + datasets: [], + adaptive_details: { + objective_count: 21, + selected_candidate_technique_count: 2, + candidate_technique_count: 2, + max_attempts_per_objective: 2, + techniques_per_objective_upper_bound: 2, + technique_attempt_count_upper_bound: 42, + stop_on_first_success: true, + compatibility_may_reduce_attempts: true, + }, + note: 'Compatibility and early success may reduce the underlying attempt count.', + retries_included: false, + } satisfies ScenarioDefaultRunSizeEstimate) + + renderDetail('/scenarios/adaptive.text_adaptive') + + for (const datasetName of makeAdaptiveScenario().default_datasets) { + expect(await screen.findByTestId(`dataset-${datasetName}`)).toBeChecked() + } + expect(screen.getByText('7 datasets selected')).toBeInTheDocument() + await waitFor(() => expect(mockEstimateRun).toHaveBeenCalledWith( + 'adaptive.text_adaptive', + { + target_name: 'target-a', + techniques: ['default'], + include_baseline: true, + }, + expect.any(AbortSignal), + )) + expect(await screen.findByRole('group', { + name: '21 objectives multiplied by up to 2 techniques per objective, the smaller of 2 selected candidates and limit 2, equals up to 42 technique attempts.', + })).toBeInTheDocument() + expect(screen.queryByText('Exact total unavailable')).not.toBeInTheDocument() + }) + + it('updates Adaptive estimates for subset, restored, single, and failed dataset requests', async () => { + jest.useFakeTimers() + const scenario = makeAdaptiveScenario() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + const objectiveCounts = new Map([ + ['airt_hate', 4], + ['airt_fairness', 1], + ['airt_violence', 3], + ['airt_sexual', 3], + ['airt_harassment', 3], + ['airt_misinformation', 3], + ['airt_leakage', 4], + ]) + let failNextRequest = false + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockImplementation(async (_scenarioName, request: ScenarioRunSizeEstimateRequest) => { + if (failNextRequest) { + failNextRequest = false + throw { + isAxiosError: true, + response: { status: 503, data: { detail: 'Current estimate failed' } }, + } + } + const datasetNames = request.dataset_names ?? scenario.default_datasets + const objectiveCount = datasetNames.reduce( + (count, datasetName) => count + (objectiveCounts.get(datasetName) ?? 0), + 0, + ) + const configuredMaximum = Number(request.scenario_params?.max_attempts_per_objective ?? 3) + const effectiveMaximum = Math.min(2, configuredMaximum) + return { + ...makeEstimate(null), + minimum_attack_count: objectiveCount, + maximum_attack_count: objectiveCount * 2, + components: [ + { + label: 'Baseline', + count: objectiveCount, + factors: [{ label: 'objectives', count: objectiveCount }], + is_baseline: true, + condition: null, + note: null, + }, + { + label: 'Adaptive objectives', + count: objectiveCount, + factors: [{ label: 'objectives', count: objectiveCount }], + is_baseline: false, + condition: null, + note: null, + }, + ], + adaptive_details: { + objective_count: objectiveCount, + selected_candidate_technique_count: 2, + candidate_technique_count: 2, + max_attempts_per_objective: configuredMaximum, + techniques_per_objective_upper_bound: effectiveMaximum, + technique_attempt_count_upper_bound: objectiveCount * effectiveMaximum, + stop_on_first_success: true, + compatibility_may_reduce_attempts: true, + }, + } + }) + + renderDetail('/scenarios/adaptive.text_adaptive') + await flushRenderedPromises() + await advanceTimers(300) + await flushRenderedPromises() + await advanceTimers(300) + await flushRenderedPromises() + expect(screen.getByRole('group', { + name: '21 objectives multiplied by up to 2 techniques per objective, the smaller of 2 selected candidates and limit 2, equals up to 42 technique attempts.', + })).toBeInTheDocument() + + await user.click(screen.getByTestId('dataset-airt_fairness')) + await advanceTimers(300) + await flushRenderedPromises() + expect(mockEstimateRun.mock.calls.at(-1)?.[1].dataset_names).toEqual([ + 'airt_hate', + 'airt_violence', + 'airt_sexual', + 'airt_harassment', + 'airt_misinformation', + 'airt_leakage', + ]) + expect(screen.getByRole('group', { + name: '20 objectives multiplied by up to 2 techniques per objective, the smaller of 2 selected candidates and limit 2, equals up to 40 technique attempts.', + })).toBeInTheDocument() + + await user.click(screen.getByTestId('restore-default-datasets')) + await advanceTimers(300) + await flushRenderedPromises() + expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('dataset_names') + expect(screen.getByRole('group', { + name: '21 objectives multiplied by up to 2 techniques per objective, the smaller of 2 selected candidates and limit 2, equals up to 42 technique attempts.', + })).toBeInTheDocument() + + for (const datasetName of scenario.default_datasets.filter((name) => name !== 'airt_fairness')) { + await user.click(screen.getByTestId(`dataset-${datasetName}`)) + } + await advanceTimers(300) + await flushRenderedPromises() + expect(mockEstimateRun.mock.calls.at(-1)?.[1].dataset_names).toEqual(['airt_fairness']) + expect(screen.getByRole('group', { + name: '1 objective multiplied by up to 2 techniques per objective, the smaller of 2 selected candidates and limit 2, equals up to 2 technique attempts.', + })).toBeInTheDocument() + + failNextRequest = true + await user.click(screen.getByTestId('dataset-airt_hate')) + await advanceTimers(300) + await flushRenderedPromises() + const preview = screen.getByRole('complementary', { name: 'Run preview' }) + expect(within(preview).queryByTestId('run-calculation')).not.toBeInTheDocument() + expect(within(preview).getByText('Current estimate failed')).toBeInTheDocument() + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + }) + + it('explains Adaptive technique sets, progress objectives, and bounded attempt work', async () => { + const scenario = makeAdaptiveScenario() + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockImplementation( + async ( + _scenarioName: string, + request: ScenarioRunSizeEstimateRequest, + ): Promise => makeAdaptiveEstimateForRequest(scenario, request), + ) + const user = userEvent.setup() + + renderDetail('/scenarios/adaptive.text_adaptive') + + expect(await screen.findByLabelText('Recommended (default) — 2 techniques')).toBeChecked() + expect(screen.getByLabelText('All (17 techniques)')).not.toBeChecked() + expect(screen.getByLabelText('Core (14 techniques)')).toBeInTheDocument() + expect(screen.getByLabelText('Extra (3 techniques)')).toBeInTheDocument() + expect(screen.getByLabelText('Light (9 techniques)')).toBeInTheDocument() + expect(screen.getByLabelText('Multi-turn (5 techniques)')).toBeInTheDocument() + expect(screen.getByLabelText('Single-turn (12 techniques)')).toBeInTheDocument() + expect(screen.getByText( + 'Choose a predefined set, or choose Custom to select techniques individually.', + )).toBeInTheDocument() + expect(screen.getByText( + /All is generated from the catalog; Recommended is curated for this scenario/, + )).toBeInTheDocument() + expect(screen.getByText( + /tries no more than the configured maximum or the compatible candidate count, whichever is smaller/, + )).toBeInTheDocument() + expect(screen.getByText( + /compatibility can still change how many objectives can run/, + )).toBeInTheDocument() + expect(screen.queryByText(/aggregate preset/i)).not.toBeInTheDocument() + expect(screen.getByRole('radio', { name: 'Custom' })).not.toBeChecked() + expect(screen.queryByRole('group', { name: 'Individual techniques' })).not.toBeInTheDocument() + + expect(await screen.findByRole('group', { + name: '21 objectives multiplied by up to 2 techniques per objective, the smaller of 2 selected candidates and limit 2, equals up to 42 technique attempts.', + })).toBeInTheDocument() + + await user.click(screen.getByLabelText('Core (14 techniques)')) + expect(screen.getByLabelText('Core (14 techniques)')).toBeChecked() + const selectedMembers = screen.getByTestId('selected-technique-set-members') + expect(within(selectedMembers).getByText('core_member_14')).toBeInTheDocument() + expect(await screen.findByRole('group', { + name: '21 objectives multiplied by up to 2 techniques per objective, the smaller of 5 compatible candidates from 14 selected and limit 2, equals up to 42 technique attempts.', + })).toBeInTheDocument() + expect(screen.getAllByText( + /5 compatible candidates from 14 selected · limit 2/, + )).toHaveLength(2) + + const maxAttempts = screen.getByRole('spinbutton', { name: 'Maximum techniques per objective' }) + expect(screen.getByText( + /This is a per-objective limit, not a total-run budget/, + )).toBeInTheDocument() + expect(screen.getByText(/incompatible techniques are skipped/)).toBeInTheDocument() + expect(screen.getByText(/This is separate from retries/)).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Advanced options' })) + expect(screen.getByText( + 'Maximum times to resume the scenario after an exception. This is separate from Adaptive trying another technique.', + )).toBeInTheDocument() + await user.clear(maxAttempts) + await user.type(maxAttempts, '1') + expect(await screen.findByRole('group', { + name: '21 objectives multiplied by up to 1 technique per objective, the smaller of 5 compatible candidates from 14 selected and limit 1, equals up to 21 technique attempts.', + })).toBeInTheDocument() + expect(screen.getAllByText( + /5 compatible candidates from 14 selected · limit 1/, + )).toHaveLength(2) + + await user.clear(maxAttempts) + expect(await screen.findByRole('group', { + name: '21 objectives multiplied by up to 3 techniques per objective, the smaller of 5 compatible candidates from 14 selected and limit 3, equals up to 63 technique attempts.', + })).toBeInTheDocument() + + await user.type(maxAttempts, '0') + expect(maxAttempts).toHaveAttribute('aria-invalid', 'true') + expect(screen.getByText('Enter a whole number of 1 or more.')).toBeInTheDocument() + expect(screen.queryByText(/up to 0 technique/i)).not.toBeInTheDocument() + expect(screen.queryByText(/objective envelope/i)).not.toBeInTheDocument() + }) + + it('validates the Adaptive attempt limit locally and recomputes after correction', async () => { + jest.useFakeTimers() + const scenario = makeAdaptiveScenario() + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockImplementation( + async ( + _scenarioName: string, + request: ScenarioRunSizeEstimateRequest, + ): Promise => makeAdaptiveEstimateForRequest(scenario, request), + ) + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + + renderDetail('/scenarios/adaptive.text_adaptive') + await flushRenderedPromises() + await advanceTimers(300) + await flushRenderedPromises() + await advanceTimers(300) + await flushRenderedPromises() + + const maxAttempts = screen.getByRole('spinbutton', { name: 'Maximum techniques per objective' }) + const preview = screen.getByRole('complementary', { name: 'Run preview' }) + expect(maxAttempts).toHaveAttribute('min', '1') + expect(maxAttempts).toHaveAttribute('step', '1') + expect(maxAttempts).toHaveAttribute('inputmode', 'numeric') + expect(maxAttempts).toHaveAttribute('pattern', '[0-9]*') + expect(screen.getByText( + /Blank restores the bounded default of 2 techniques per objective for this target\./, + )).toBeInTheDocument() + expect(screen.queryByText(/Leave blank to use the default of 3/)).not.toBeInTheDocument() + expect(maxAttempts).toHaveValue(2) + expect(within(preview).getByText('up to 42')).toBeInTheDocument() + const initialRequestCount = mockEstimateRun.mock.calls.length + + await user.clear(maxAttempts) + await user.type(maxAttempts, '-8') + expect(maxAttempts).toHaveValue(null) + expect(maxAttempts).toHaveAttribute('aria-invalid', 'true') + expect(screen.getByText('Enter a whole number of 1 or more.')).toBeInTheDocument() + expect(within(preview).getByText(CORRECT_HIGHLIGHTED_SETTING_MESSAGE)) + .toBeInTheDocument() + expect(within(preview).queryByTestId('run-calculation')).not.toBeInTheDocument() + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + await advanceTimers(300) + expect(mockEstimateRun).toHaveBeenCalledTimes(initialRequestCount) + + await user.tab() + await user.click(maxAttempts) + await user.paste('-8') + expect(maxAttempts).toHaveValue(null) + expect(maxAttempts).toHaveAttribute('aria-invalid', 'true') + await advanceTimers(300) + expect(mockEstimateRun).toHaveBeenCalledTimes(initialRequestCount) + + await user.tab() + await user.click(maxAttempts) + await user.paste('0') + expect(maxAttempts).toHaveValue(null) + expect(maxAttempts).toHaveAttribute('aria-invalid', 'true') + await advanceTimers(300) + expect(mockEstimateRun).toHaveBeenCalledTimes(initialRequestCount) + + for (const invalidValue of ['1.5', '1e3', '+8']) { + await user.tab() + await user.click(maxAttempts) + await user.type(maxAttempts, invalidValue) + expect(maxAttempts).toHaveValue(null) + expect(maxAttempts).toHaveAttribute('aria-invalid', 'true') + expect(screen.getByText('Enter a whole number of 1 or more.')).toBeInTheDocument() + expect(within(preview).getByText(CORRECT_HIGHLIGHTED_SETTING_MESSAGE)) + .toBeInTheDocument() + expect(within(preview).queryByTestId('run-calculation')).not.toBeInTheDocument() + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + expect(screen.queryByText(/max_attempts_per_objective must/i)).not.toBeInTheDocument() + await advanceTimers(300) + expect(mockEstimateRun).toHaveBeenCalledTimes(initialRequestCount) + } + + await user.tab() + await user.click(maxAttempts) + await user.type(maxAttempts, '0') + expect(maxAttempts).toHaveValue(null) + expect(maxAttempts).toHaveAttribute('aria-invalid', 'true') + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + await advanceTimers(300) + expect(mockEstimateRun).toHaveBeenCalledTimes(initialRequestCount) + + await user.tab() + await user.click(maxAttempts) + await user.type(maxAttempts, '1') + await user.clear(maxAttempts) + await advanceTimers(300) + await flushRenderedPromises() + await advanceTimers(300) + await flushRenderedPromises() + expect(maxAttempts).toHaveAttribute('aria-invalid', 'false') + expect(maxAttempts).toHaveValue(2) + expect(mockEstimateRun.mock.calls.at(-1)?.[1].scenario_params).toEqual({ + max_attempts_per_objective: 2, + }) + expect(within(preview).getByText('up to 42')).toBeInTheDocument() + + await user.clear(maxAttempts) + await user.type(maxAttempts, '1') + await advanceTimers(300) + await flushRenderedPromises() + expect(maxAttempts).toHaveAttribute('aria-invalid', 'false') + expect(mockEstimateRun.mock.calls.at(-1)?.[1].scenario_params).toEqual({ + max_attempts_per_objective: 1, + }) + expect(within(within(preview).getByTestId('adaptive-work-calculation')).getByText('up to 21')) + .toBeInTheDocument() + expect(screen.getByTestId('launch-scenario-btn')).toBeEnabled() + + const correctedRequestCount = mockEstimateRun.mock.calls.length + Object.defineProperty(window.getSelection(), 'modify', { value: jest.fn(), configurable: true }) + await user.keyboard('{ArrowDown}') + expect(maxAttempts).toHaveValue(1) + await advanceTimers(300) + expect(mockEstimateRun).toHaveBeenCalledTimes(correctedRequestCount) + }) + + it('does not let a superseded estimate repopulate arithmetic after the limit becomes invalid', async () => { + jest.useFakeTimers() + const scenario = makeAdaptiveScenario() + mockGetScenario.mockResolvedValue(scenario) + let resolveEstimate: (estimate: ScenarioDefaultRunSizeEstimate) => void = () => {} + mockEstimateRun + .mockResolvedValueOnce(makeAdaptiveEstimateForRequest(scenario, { + target_name: 'target-a', + techniques: ['default'], + include_baseline: true, + })) + .mockReturnValueOnce(new Promise((resolve) => { + resolveEstimate = resolve + })) + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + + renderDetail('/scenarios/adaptive.text_adaptive') + await flushRenderedPromises() + await advanceTimers(300) + await flushRenderedPromises() + const maxAttempts = screen.getByRole('spinbutton', { name: 'Maximum techniques per objective' }) + await user.type(maxAttempts, '1') + await advanceTimers(300) + const requestSignal = mockEstimateRun.mock.calls[1][2] as AbortSignal + + await user.clear(maxAttempts) + await user.type(maxAttempts, '-8') + expect(requestSignal.aborted).toBe(true) + expect(screen.getByText(CORRECT_HIGHLIGHTED_SETTING_MESSAGE)).toBeInTheDocument() + + resolveEstimate(makeAdaptiveEstimateForRequest(scenario, { + target_name: 'target-a', + techniques: ['default'], + include_baseline: true, + scenario_params: { max_attempts_per_objective: 3 }, + })) + await flushRenderedPromises() + + expect(screen.queryByTestId('run-calculation')).not.toBeInTheDocument() + expect(screen.getByText(CORRECT_HIGHLIGHTED_SETTING_MESSAGE)).toBeInTheDocument() + expect(mockEstimateRun).toHaveBeenCalledTimes(2) + }) + + it('maps backend attempt-limit validation to the field without exposing internal copy', async () => { + jest.useFakeTimers() + mockGetScenario.mockResolvedValue(makeAdaptiveScenario()) + mockEstimateRun.mockRejectedValue({ + isAxiosError: true, + response: { + status: 400, + data: { detail: 'max_attempts_per_objective must be >= 1, got -8' }, + }, + }) + + renderDetail('/scenarios/adaptive.text_adaptive') + await flushRenderedPromises() + await advanceTimers(300) + await flushRenderedPromises() + + const maxAttempts = screen.getByRole('spinbutton', { name: 'Maximum techniques per objective' }) + expect(maxAttempts).toHaveAttribute('aria-invalid', 'true') + expect(screen.getByText('Enter a whole number of 1 or more.')).toBeInTheDocument() + expect(screen.getByText(CORRECT_HIGHLIGHTED_SETTING_MESSAGE)).toBeInTheDocument() + expect(screen.queryByText(/max_attempts_per_objective must/i)).not.toBeInTheDocument() + expect(screen.queryByTestId('run-calculation')).not.toBeInTheDocument() + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + }) + + it('presents Adaptive attempts clearly while preserving the scenario parameter wire key', async () => { + const scenario = makeAdaptiveScenario() + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockImplementation( + async ( + _scenarioName: string, + request: ScenarioRunSizeEstimateRequest, + ): Promise => makeAdaptiveEstimateForRequest(scenario, request), + ) + const user = userEvent.setup() + renderDetail('/scenarios/adaptive.text_adaptive') + + const maxAttempts = await screen.findByRole('spinbutton', { name: 'Maximum techniques per objective' }) + await screen.findByRole('group', { + name: '21 objectives multiplied by up to 2 techniques per objective, the smaller of 2 selected candidates and limit 2, equals up to 42 technique attempts.', + }) + expect(maxAttempts).toHaveAttribute('max', '2') + expect(maxAttempts).toHaveValue(2) + expect(screen.getByText( + 'The scenario default of 3 is reduced to 2 because Recommended (default) provides 2 compatible techniques for this target.', + )).toBeInTheDocument() + expect(screen.queryByText(/Maximum reached:/)).not.toBeInTheDocument() + expect(screen.getByText( + /Blank restores the bounded default of 2 techniques per objective for this target\./, + )).toBeInTheDocument() + expect(screen.queryByText('max_attempts_per_objective')).not.toBeInTheDocument() + expect(screen.getByText(/This is separate from retries/)).toBeInTheDocument() + + await user.clear(maxAttempts) + await user.type(maxAttempts, '5') + expect(maxAttempts).toHaveValue(2) + expect(screen.getByText( + 'Maximum reached: Recommended (default) provides 2 compatible techniques for this target.', + )).toBeInTheDocument() + expect(mockEstimateRun).not.toHaveBeenCalledWith( + 'adaptive.text_adaptive', + expect.objectContaining({ + scenario_params: { max_attempts_per_objective: 5 }, + }), + expect.any(AbortSignal), + ) + + await user.clear(maxAttempts) + await waitFor(() => expect(maxAttempts).toHaveValue(2)) + expect(screen.getByText( + 'The scenario default of 3 is reduced to 2 because Recommended (default) provides 2 compatible techniques for this target.', + )).toBeInTheDocument() + + await user.paste('3') + expect(maxAttempts).toHaveValue(2) + expect(screen.getByText( + 'Maximum reached: Recommended (default) provides 2 compatible techniques for this target.', + )).toBeInTheDocument() + + await user.clear(maxAttempts) + await user.type(maxAttempts, '1') + await waitFor(() => expect(mockEstimateRun).toHaveBeenLastCalledWith( + 'adaptive.text_adaptive', + expect.objectContaining({ + scenario_params: { max_attempts_per_objective: 1 }, + }), + expect.any(AbortSignal), + )) + expect(screen.queryByText(/Maximum reached:/)).not.toBeInTheDocument() + + await user.clear(maxAttempts) + await user.type(maxAttempts, '2') + await waitFor(() => expect(mockEstimateRun).toHaveBeenLastCalledWith( + 'adaptive.text_adaptive', + expect.objectContaining({ + scenario_params: { max_attempts_per_objective: 2 }, + }), + expect.any(AbortSignal), + )) + expect(await screen.findByRole('group', { + name: '21 objectives multiplied by up to 2 techniques per objective, the smaller of 2 selected candidates and limit 2, equals up to 42 technique attempts.', + })).toBeInTheDocument() + expect(screen.getByText( + 'Maximum reached: Recommended (default) provides 2 compatible techniques for this target.', + )).toBeInTheDocument() + Object.defineProperty(window.getSelection(), 'modify', { value: jest.fn(), configurable: true }) + await user.keyboard('{ArrowUp}') + expect(maxAttempts).toHaveValue(2) + const preview = screen.getByRole('complementary', { name: 'Run preview' }) + expect(within(preview).getByText('Maximum techniques per objective')).toBeInTheDocument() + expect(within(preview).queryByText('Techniques tried per objective')).not.toBeInTheDocument() + await user.click(screen.getByTestId('launch-scenario-btn')) + await waitFor(() => expect(mockStartRun).toHaveBeenCalledWith( + expect.objectContaining({ + scenario_params: { max_attempts_per_objective: 2 }, + }), + )) }) - it('switches from the default preset to a multi-technique custom selection', async () => { + it.each([ + ['Light (9 techniques)', 'Light', 9, 10], + ['Core (14 techniques)', 'Core', 14, 22], + ['All (17 techniques)', 'All', 17, 22], + ])( + 'clamps an over-limit attempt to the authoritative %s candidate count', + async (optionLabel, displayName, maximum, attemptedValue) => { + const scenario = makeAdaptiveScenario() + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockImplementation( + async ( + _scenarioName: string, + request: ScenarioRunSizeEstimateRequest, + ): Promise => { + const estimate = makeAdaptiveEstimateForRequest(scenario, request) + const adaptiveDetails = estimate.adaptive_details + if (adaptiveDetails) { + const candidateCount = adaptiveDetails.selected_candidate_technique_count ?? 0 + const configuredMaximum = adaptiveDetails.max_attempts_per_objective + adaptiveDetails.candidate_technique_count = candidateCount + adaptiveDetails.techniques_per_objective_upper_bound = Math.min( + candidateCount, + configuredMaximum, + ) + adaptiveDetails.technique_attempt_count_upper_bound = + 21 * adaptiveDetails.techniques_per_objective_upper_bound + } + return estimate + }, + ) + const user = userEvent.setup() + renderDetail('/scenarios/adaptive.text_adaptive') + + const maxAttempts = await screen.findByRole('spinbutton', { + name: 'Maximum techniques per objective', + }) + await user.click(await screen.findByLabelText(optionLabel)) + await waitFor(() => expect(maxAttempts).toHaveAttribute('max', String(maximum))) + expect(screen.getByText(/Leave blank to use the default of 3\./)).toBeInTheDocument() + expect(screen.queryByText(/Blank restores the bounded default/)).not.toBeInTheDocument() + await user.clear(maxAttempts) + await user.type(maxAttempts, String(attemptedValue)) + + expect(maxAttempts).toHaveValue(maximum) + expect(screen.getByText( + `Maximum reached: ${displayName} provides ${maximum} compatible techniques for this target.`, + )).toBeInTheDocument() + expect(await screen.findByRole('group', { + name: `21 objectives multiplied by up to ${maximum} techniques per objective, the smaller of ${maximum} selected candidates and limit ${maximum}, equals up to ${ + 21 * maximum + } technique attempts.`, + })).toBeInTheDocument() + expect(mockEstimateRun).not.toHaveBeenCalledWith( + 'adaptive.text_adaptive', + expect.objectContaining({ + scenario_params: { max_attempts_per_objective: attemptedValue }, + }), + expect.any(AbortSignal), + ) + expect(mockEstimateRun).toHaveBeenLastCalledWith( + 'adaptive.text_adaptive', + expect.objectContaining({ + scenario_params: { max_attempts_per_objective: maximum }, + }), + expect.any(AbortSignal), + ) + expect(screen.getByRole('complementary', { name: 'Run preview' })).toHaveTextContent( + new RegExp(`Maximum techniques per objective\\s*${maximum}`), + ) + await user.type(maxAttempts, '-8') + expect(maxAttempts).toHaveValue(maximum) + expect(maxAttempts).toHaveAttribute('aria-invalid', 'true') + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + }, + ) + + it('clamps an explicit limit when the selected technique set lowers the compatible maximum', async () => { + const scenario = makeAdaptiveScenario() + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockImplementation( + async ( + _scenarioName: string, + request: ScenarioRunSizeEstimateRequest, + ): Promise => makeAdaptiveEstimateForRequest(scenario, request), + ) + const user = userEvent.setup() + renderDetail('/scenarios/adaptive.text_adaptive') + + const maxAttempts = await screen.findByRole('spinbutton', { + name: 'Maximum techniques per objective', + }) + await screen.findByRole('group', { + name: '21 objectives multiplied by up to 2 techniques per objective, the smaller of 2 selected candidates and limit 2, equals up to 42 technique attempts.', + }) + + await user.click(screen.getByLabelText('Core (14 techniques)')) + await waitFor(() => expect(maxAttempts).toHaveAttribute('max', '5')) + await user.type(maxAttempts, '5') + await screen.findByRole('group', { + name: '21 objectives multiplied by up to 5 techniques per objective, the smaller of 5 compatible candidates from 14 selected and limit 5, equals up to 105 technique attempts.', + }) + + await user.click(screen.getByLabelText('Recommended (default) — 2 techniques')) + expect(maxAttempts).toBeDisabled() + await waitFor(() => expect(maxAttempts).toHaveValue(2)) + expect(maxAttempts).toHaveAttribute('max', '2') + expect(screen.getByText( + 'Reduced to 2 because Recommended (default) provides 2 compatible techniques for this target.', + )).toBeInTheDocument() + await screen.findByRole('group', { + name: '21 objectives multiplied by up to 2 techniques per objective, the smaller of 2 selected candidates and limit 2, equals up to 42 technique attempts.', + }) + expect(mockEstimateRun).not.toHaveBeenCalledWith( + 'adaptive.text_adaptive', + expect.objectContaining({ + techniques: ['default'], + scenario_params: { max_attempts_per_objective: 5 }, + }), + expect.any(AbortSignal), + ) + + await user.click(screen.getByTestId('launch-scenario-btn')) + await waitFor(() => expect(mockStartRun).toHaveBeenCalledWith( + expect.objectContaining({ + techniques: ['default'], + scenario_params: { max_attempts_per_objective: 2 }, + }), + )) + }) + + it('updates the bound for target compatibility and blocks a target with no eligible techniques', async () => { + const scenario = makeAdaptiveScenario() + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockImplementation( + async ( + _scenarioName: string, + request: ScenarioRunSizeEstimateRequest, + ): Promise => { + const estimate = makeAdaptiveEstimateForRequest(scenario, request) + if (request.target_name === 'target-b' && estimate.adaptive_details) { + const candidateCount = request.techniques?.[0] === 'core' ? 0 : 1 + estimate.adaptive_details.candidate_technique_count = candidateCount + estimate.adaptive_details.techniques_per_objective_upper_bound = candidateCount + estimate.adaptive_details.technique_attempt_count_upper_bound = 21 * candidateCount + } + return estimate + }, + ) + const user = userEvent.setup() + renderDetail('/scenarios/adaptive.text_adaptive') + + const maxAttempts = await screen.findByRole('spinbutton', { + name: 'Maximum techniques per objective', + }) + await waitFor(() => expect(maxAttempts).toHaveAttribute('max', '2')) + await user.type(maxAttempts, '2') + await waitFor(() => expect(maxAttempts).toHaveValue(2)) + await user.selectOptions(screen.getByRole('combobox', { name: 'Target' }), 'target-b') + + expect(maxAttempts).toBeDisabled() + await waitFor(() => expect(maxAttempts).toHaveValue(1)) + expect(maxAttempts).toHaveAttribute('max', '1') + expect(screen.getByText( + 'Reduced to 1 because Recommended (default) provides 1 compatible technique for this target.', + )).toBeInTheDocument() + expect(mockEstimateRun).not.toHaveBeenCalledWith( + 'adaptive.text_adaptive', + expect.objectContaining({ + target_name: 'target-b', + scenario_params: { max_attempts_per_objective: 2 }, + }), + expect.any(AbortSignal), + ) + + await user.click(screen.getByLabelText('Core (14 techniques)')) + expect(await screen.findByText( + 'No compatible techniques are available for this target. Choose a different technique set or target.', + )).toBeInTheDocument() + expect(maxAttempts).toBeDisabled() + expect(maxAttempts).not.toHaveAttribute('max') + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + expect(screen.queryByTestId('run-calculation')).not.toBeInTheDocument() + }) + + it('switches exclusively from a named set to Custom and initializes resolved members', async () => { mockGetScenario.mockResolvedValue( makeScenario({ aggregate_techniques: ['default_technique', 'all_garak'], @@ -447,11 +1374,17 @@ describe('ScenarioDetail', () => { // it must render exactly once (deduped), under the aggregate group. expect(screen.getAllByTestId('technique-all_garak')).toHaveLength(1) - await user.click(screen.getByTestId('technique-crescendo')) + expect(screen.queryByRole('group', { name: 'Individual techniques' })).not.toBeInTheDocument() + await user.click(screen.getByTestId('technique-mode-custom')) expect(screen.getByTestId('technique-default_technique')).not.toBeChecked() expect(screen.getByTestId('technique-crescendo')).toBeChecked() await user.click(screen.getByTestId('technique-prompt_sending')) + await waitFor(() => expect(mockEstimateRun).toHaveBeenLastCalledWith( + 'foundry.red_team_agent', + expect.objectContaining({ techniques: ['crescendo', 'prompt_sending'] }), + expect.any(AbortSignal), + )) await user.click(screen.getByTestId('launch-scenario-btn')) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) @@ -460,25 +1393,39 @@ describe('ScenarioDetail', () => { expect(new Set(request.techniques).size).toBe(request.techniques.length) }) - it('selecting a preset replaces the custom concrete list', async () => { + it('preserves custom choices while named sets send exactly one token', async () => { mockGetScenario.mockResolvedValue( makeScenario({ aggregate_techniques: ['default_technique', 'all_garak'], - all_techniques: ['default_technique', 'crescendo'], + aggregate_technique_expansions: { + default_technique: ['crescendo'], + all_garak: ['crescendo'], + }, + all_techniques: ['default_technique', 'crescendo', 'prompt_sending'], }), ) const user = userEvent.setup() renderDetail('/scenarios/foundry.red_team_agent') await screen.findByTestId('scenario-target-select') - await user.click(screen.getByTestId('technique-crescendo')) + await user.click(screen.getByTestId('technique-mode-custom')) + await user.click(screen.getByTestId('technique-prompt_sending')) await user.click(screen.getByTestId('technique-all_garak')) expect(screen.getByTestId('technique-all_garak')).toBeChecked() - expect(screen.getByTestId('technique-crescendo')).not.toBeChecked() + expect(screen.queryByRole('group', { name: 'Individual techniques' })).not.toBeInTheDocument() + await waitFor(() => expect(mockEstimateRun).toHaveBeenLastCalledWith( + 'foundry.red_team_agent', + expect.objectContaining({ techniques: ['all_garak'] }), + expect.any(AbortSignal), + )) await user.click(screen.getByTestId('launch-scenario-btn')) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) expect(mockStartRun.mock.calls[0][0].techniques).toEqual(['all_garak']) + + await user.click(screen.getByTestId('technique-mode-custom')) + expect(screen.getByTestId('technique-crescendo')).toBeChecked() + expect(screen.getByTestId('technique-prompt_sending')).toBeChecked() }) it('initializes a concrete default as custom and allows adding another concrete technique', async () => { @@ -493,6 +1440,7 @@ describe('ScenarioDetail', () => { renderDetail('/scenarios/foundry.red_team_agent') await screen.findByTestId('scenario-target-select') + expect(screen.getByTestId('technique-mode-custom')).toBeChecked() expect(screen.getByTestId('technique-prompt_sending')).toBeChecked() await user.click(screen.getByTestId('technique-crescendo')) expect(screen.getByTestId('technique-prompt_sending')).toBeChecked() @@ -508,7 +1456,7 @@ describe('ScenarioDetail', () => { renderDetail('/scenarios/foundry.red_team_agent') await screen.findByTestId('scenario-target-select') - await user.click(screen.getByTestId('technique-crescendo')) + await user.click(screen.getByTestId('technique-mode-custom')) await user.click(screen.getByTestId('technique-crescendo')) expect(await screen.findByRole('alert')).toHaveTextContent('Select at least one technique.') @@ -524,14 +1472,159 @@ describe('ScenarioDetail', () => { const checkbox = screen.getByTestId('baseline-checkbox') expect(checkbox).toBeChecked() + expect(screen.getByText( + /Also send each selected objective directly, without an attack technique/, + )).toBeInTheDocument() + expect(screen.getByRole('complementary', { name: 'Run preview' })).toHaveTextContent( + 'Included — direct objective without an attack technique', + ) await user.click(checkbox) + await waitFor(() => expect(mockEstimateRun).toHaveBeenLastCalledWith( + 'foundry.red_team_agent', + expect.objectContaining({ include_baseline: false }), + expect.any(AbortSignal), + )) await user.click(screen.getByTestId('launch-scenario-btn')) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) expect(mockStartRun.mock.calls[0][0].include_baseline).toBe(false) }) + it('updates Adaptive planned arithmetic ON to OFF to ON while preserving inner work', async () => { + const scenario = makeAdaptiveScenario() + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockImplementation( + async ( + _scenarioName: string, + request: ScenarioRunSizeEstimateRequest, + ): Promise => + makeFullyCompatibleAdaptiveEstimateForRequest(scenario, request), + ) + const user = userEvent.setup() + renderDetail('/scenarios/adaptive.text_adaptive') + + await user.click(await screen.findByLabelText('Core (14 techniques)')) + const maxAttempts = await screen.findByRole('spinbutton', { + name: 'Maximum techniques per objective', + }) + await waitFor(() => expect(maxAttempts).toHaveAttribute('max', '14')) + await user.clear(maxAttempts) + await user.type(maxAttempts, '14') + + const preview = screen.getByRole('complementary', { name: 'Run preview' }) + expect(await within(preview).findByRole('group', { + name: 'Direct baseline comparison is included: 21 direct baseline attacks plus up to 21 Adaptive attacks equals 21–42 planned attacks.', + })).toBeInTheDocument() + expect(within(preview).getByTestId('adaptive-work-calculation')).toHaveTextContent( + 'up to 294technique attempts', + ) + expect(screen.getByText('Adds 21 direct baseline attacks for the current objectives.')) + .toBeInTheDocument() + expect(within(preview).getByText('Included — direct objective without an attack technique')) + .toBeInTheDocument() + + const baselineCheckbox = screen.getByTestId('baseline-checkbox') + await user.click(baselineCheckbox) + expect(within(preview).getByText('Calculating planned attacks...')).toBeInTheDocument() + expect(within(preview).getByText('Not included')).toBeInTheDocument() + expect(await within(preview).findByRole('group', { + name: 'Direct baseline comparison is not included: up to 21 Adaptive attacks equals up to 21 planned attacks.', + })).toBeInTheDocument() + expect(within(preview).getByTestId('adaptive-work-calculation')).toHaveTextContent( + 'up to 294technique attempts', + ) + expect(mockEstimateRun).toHaveBeenLastCalledWith( + 'adaptive.text_adaptive', + expect.objectContaining({ + include_baseline: false, + scenario_params: { max_attempts_per_objective: 14 }, + }), + expect.any(AbortSignal), + ) + + await user.click(baselineCheckbox) + expect(within(preview).getByText('Calculating planned attacks...')).toBeInTheDocument() + expect(within(preview).getByText('Included — direct objective without an attack technique')) + .toBeInTheDocument() + expect(await within(preview).findByRole('group', { + name: 'Direct baseline comparison is included: 21 direct baseline attacks plus up to 21 Adaptive attacks equals 21–42 planned attacks.', + })).toBeInTheDocument() + expect(mockEstimateRun).toHaveBeenLastCalledWith( + 'adaptive.text_adaptive', + expect.objectContaining({ + include_baseline: true, + scenario_params: { max_attempts_per_objective: 14 }, + }), + expect.any(AbortSignal), + ) + }) + + it('ignores stale Adaptive baseline estimates after a rapid OFF to ON toggle', async () => { + const scenario = makeAdaptiveScenario() + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockImplementation( + async ( + _scenarioName: string, + request: ScenarioRunSizeEstimateRequest, + ): Promise => + makeFullyCompatibleAdaptiveEstimateForRequest(scenario, request), + ) + const user = userEvent.setup() + renderDetail('/scenarios/adaptive.text_adaptive') + + await screen.findByRole('group', { + name: 'Direct baseline comparison is included: 21 direct baseline attacks plus up to 21 Adaptive attacks equals 21–42 planned attacks.', + }) + + let resolveOff: ((estimate: ScenarioDefaultRunSizeEstimate) => void) | null = null + let resolveOn: ((estimate: ScenarioDefaultRunSizeEstimate) => void) | null = null + mockEstimateRun.mockImplementation( + async ( + _scenarioName: string, + request: ScenarioRunSizeEstimateRequest, + ): Promise => await new Promise((resolve) => { + if (request.include_baseline === false) { + resolveOff = resolve + } else { + resolveOn = resolve + } + }), + ) + + const baselineCheckbox = screen.getByTestId('baseline-checkbox') + await user.click(baselineCheckbox) + await waitFor(() => expect(resolveOff).not.toBeNull()) + await user.click(baselineCheckbox) + await waitFor(() => expect(resolveOn).not.toBeNull()) + + if (!resolveOn || !resolveOff) { + throw new Error('Expected both baseline estimate requests to be pending.') + } + resolveOn(makeFullyCompatibleAdaptiveEstimateForRequest(scenario, { + target_name: 'target-a', + techniques: ['default'], + include_baseline: true, + })) + await flushRenderedPromises() + expect(screen.getByRole('group', { + name: 'Direct baseline comparison is included: 21 direct baseline attacks plus up to 21 Adaptive attacks equals 21–42 planned attacks.', + })).toBeInTheDocument() + + resolveOff(makeFullyCompatibleAdaptiveEstimateForRequest(scenario, { + target_name: 'target-a', + techniques: ['default'], + include_baseline: false, + })) + await flushRenderedPromises() + expect(screen.getByRole('group', { + name: 'Direct baseline comparison is included: 21 direct baseline attacks plus up to 21 Adaptive attacks equals 21–42 planned attacks.', + })).toBeInTheDocument() + expect(screen.queryByRole('group', { + name: 'Direct baseline comparison is not included: up to 21 Adaptive attacks equals up to 21 planned attacks.', + })).not.toBeInTheDocument() + }) + it('defaults the baseline checkbox to unchecked when the policy is disabled with include_baseline_by_default false', async () => { mockGetScenario.mockResolvedValue( makeScenario({ baseline_policy: 'disabled', include_baseline_by_default: false }), @@ -544,6 +1637,7 @@ describe('ScenarioDetail', () => { it('disables and forces the baseline checkbox false when the policy is forbidden', async () => { mockGetScenario.mockResolvedValue(makeScenario({ baseline_policy: 'forbidden' })) + mockEstimateRun.mockResolvedValue(makeEstimate(8)) const user = userEvent.setup() renderDetail('/scenarios/foundry.red_team_agent') @@ -552,6 +1646,12 @@ describe('ScenarioDetail', () => { const checkbox = screen.getByTestId('baseline-checkbox') expect(checkbox).toBeDisabled() expect(checkbox).not.toBeChecked() + expect(checkbox).toHaveAccessibleName('Include direct baseline comparison') + expect(screen.getByText( + /This scenario does not support sending objectives directly without an attack technique/, + )).toBeInTheDocument() + expect(await screen.findByRole('group', { name: '8 planned attacks.' })).toBeInTheDocument() + expect(screen.queryByText(/direct baseline attack/)).not.toBeInTheDocument() await user.click(screen.getByTestId('launch-scenario-btn')) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) @@ -589,7 +1689,6 @@ describe('ScenarioDetail', () => { ], }), ) - const user = userEvent.setup() renderDetail('/scenarios/foundry.red_team_agent') await screen.findByTestId('scenario-target-select') @@ -598,18 +1697,20 @@ describe('ScenarioDetail', () => { // decimal (a valid *number* but not a valid *integer*) exercises the same // coercion/validation path a real user could actually trigger. fireEvent.change(screen.getByTestId('scenario-param-iterations'), { target: { value: '1.5' } }) - await user.click(screen.getByTestId('launch-scenario-btn')) - expect(await screen.findByRole('alert')).toHaveTextContent('iterations must be an integer.') + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + expect(screen.getByText('iterations must be an integer.')).toBeInTheDocument() expect(mockStartRun).not.toHaveBeenCalled() }) - it('omits the dataset override and max dataset size when left blank, sending default concurrency/retries', async () => { + it('selects scenario default datasets initially and omits the unchanged override', async () => { const user = userEvent.setup() renderDetail('/scenarios/foundry.red_team_agent') await screen.findByTestId('scenario-target-select') - await user.click(screen.getByRole('button', { name: 'Advanced options' })) + expect(screen.getByTestId('dataset-harmbench')).toBeChecked() + expect(screen.getByText('1 dataset selected')).toBeInTheDocument() + expect(screen.getByTestId('restore-default-datasets')).toBeDisabled() await user.click(screen.getByTestId('launch-scenario-btn')) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) @@ -620,26 +1721,196 @@ describe('ScenarioDetail', () => { expect(request.max_retries).toBe(0) }) - it('includes dataset override and max dataset size when provided', async () => { + it('materializes the adaptive per-dataset default while omitting unchanged and restored overrides', async () => { + const scenario = makeAdaptiveScenario() + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockImplementation( + async (_scenarioName, request: ScenarioRunSizeEstimateRequest) => + makeFullyCompatibleAdaptiveEstimateForRequest(scenario, request), + ) + const user = userEvent.setup() + renderDetail('/scenarios/adaptive.text_adaptive') + await screen.findByTestId('scenario-target-select') + await user.click(screen.getByRole('button', { name: 'Advanced options' })) + + const input = screen.getByTestId('advanced-max_dataset_size') + expect(input).toHaveValue(4) + expect(input).toHaveAttribute('min', '1') + expect(input).toHaveAttribute('step', '1') + expect(input).toHaveAttribute('inputmode', 'numeric') + expect(screen.getByText( + 'Scenario default: up to 4 objectives from each selected dataset. Enter another whole number to override it, or leave blank to use the scenario default.', + )).toBeInTheDocument() + expect(screen.getByRole('complementary', { name: 'Run preview' })) + .toHaveTextContent('4 per dataset (scenario default)') + await waitFor(() => expect(mockEstimateRun).toHaveBeenCalled()) + expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('max_dataset_size') + + await user.clear(input) + expect(input).toHaveValue(null) + expect(screen.getByRole('complementary', { name: 'Run preview' })) + .toHaveTextContent('4 per dataset (scenario default)') + await user.type(input, '3') + expect(screen.getByRole('complementary', { name: 'Run preview' })) + .toHaveTextContent('3 per dataset (override)') + await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]).toEqual( + expect.objectContaining({ max_dataset_size: 3 }), + )) + await user.click(screen.getByTestId('launch-scenario-btn')) + await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) + expect(mockStartRun.mock.calls[0][0].max_dataset_size).toBe(3) + mockStartRun.mockClear() + + await user.click(screen.getByTestId('restore-default-dataset-size')) + expect(input).toHaveValue(4) + expect(screen.getByRole('complementary', { name: 'Run preview' })) + .toHaveTextContent('4 per dataset (scenario default)') + await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]) + .not.toHaveProperty('max_dataset_size')) + await user.click(screen.getByTestId('launch-scenario-btn')) + await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) + expect(mockStartRun.mock.calls[0][0]).not.toHaveProperty('max_dataset_size') + }) + + it('keeps the inherited per-dataset default while dataset selections change', async () => { + const scenario = makeAdaptiveScenario() + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockImplementation( + async (_scenarioName, request: ScenarioRunSizeEstimateRequest) => + makeFullyCompatibleAdaptiveEstimateForRequest(scenario, request), + ) + const user = userEvent.setup() + renderDetail('/scenarios/adaptive.text_adaptive') + await screen.findByTestId('dataset-ds_a') + + await user.click(screen.getByTestId('dataset-airt_hate')) + await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]).toEqual( + expect.objectContaining({ + dataset_names: expect.arrayContaining(['airt_fairness', 'airt_violence']), + }), + )) + expect(mockEstimateRun.mock.calls.at(-1)?.[1].dataset_names).toHaveLength(6) + expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('max_dataset_size') + + for (const name of ['airt_fairness', 'airt_violence', 'airt_sexual', 'airt_harassment', 'airt_misinformation']) { + await user.click(screen.getByTestId(`dataset-${name}`)) + } + await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1].dataset_names) + .toEqual(['airt_leakage'])) + expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('max_dataset_size') + + await user.click(screen.getByTestId('restore-default-datasets')) + await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]) + .not.toHaveProperty('dataset_names')) + await user.click(screen.getByTestId('dataset-ds_a')) + await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1].dataset_names) + .toEqual([...scenario.default_datasets, 'ds_a'])) + expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('max_dataset_size') + + await user.click(screen.getByTestId('launch-scenario-btn')) + await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) + expect(mockStartRun.mock.calls[0][0].dataset_names).toEqual([...scenario.default_datasets, 'ds_a']) + expect(mockStartRun.mock.calls[0][0]).not.toHaveProperty('max_dataset_size') + }) + + it('renders accurate combined, no-cap, and heterogeneous dataset limit semantics', async () => { + const combinedScenario = makeScenario({ + default_datasets: ['harmbench', 'xstest'], + dataset_size_limit: { + default_scope: 'combined', + default_count: 5, + override_scope: 'combined', + }, + }) + mockGetScenario.mockResolvedValue(combinedScenario) + const user = userEvent.setup() + const view = renderDetail('/scenarios/foundry.red_team_agent') + await screen.findByTestId('scenario-target-select') + await user.click(screen.getByRole('button', { name: 'Advanced options' })) + expect(screen.getByRole('spinbutton', { + name: 'Maximum objectives across selected datasets', + })).toHaveValue(5) + expect(screen.getByText(/Scenario default: up to 5 objectives across the selected datasets/)) + .toBeInTheDocument() + expect(screen.getByRole('complementary', { name: 'Run preview' })) + .toHaveTextContent('5 total (scenario default)') + + view.unmount() + mockGetScenario.mockResolvedValue(makeScenario()) + const noCapView = renderDetail('/scenarios/foundry.red_team_agent') + await screen.findByTestId('scenario-target-select') + await user.click(screen.getByRole('button', { name: 'Advanced options' })) + expect(screen.getByText(/No scenario default cap/)).toBeInTheDocument() + expect(screen.getByRole('complementary', { name: 'Run preview' })) + .toHaveTextContent('No additional objective cap') + + noCapView.unmount() + mockGetScenario.mockResolvedValue(makeScenario({ + default_datasets: ['harmbench', 'xstest'], + dataset_size_limit: { + default_scope: 'heterogeneous', + default_count: null, + override_scope: 'per_dataset', + }, + })) + renderDetail('/scenarios/foundry.red_team_agent') + await screen.findByTestId('scenario-target-select') + await user.click(screen.getByRole('button', { name: 'Advanced options' })) + expect(screen.getByText(/Scenario defaults vary by dataset/)).toBeInTheDocument() + expect(screen.getByRole('complementary', { name: 'Run preview' })) + .toHaveTextContent('Varies by dataset (scenario default)') + }) + + it('disables dataset-size overrides when the scenario manages its population directly', async () => { + mockGetScenario.mockResolvedValue(makeScenario({ + dataset_size_limit: { + default_scope: 'none', + default_count: null, + override_scope: 'unsupported', + }, + })) + const user = userEvent.setup() + renderDetail('/scenarios/foundry.red_team_agent') + await screen.findByTestId('scenario-target-select') + await user.click(screen.getByRole('button', { name: 'Advanced options' })) + + expect(screen.getByRole('spinbutton', { name: 'Maximum objectives' })).toBeDisabled() + expect(screen.getByText( + 'This scenario manages its objective population directly and does not support a dataset-size override.', + )).toBeInTheDocument() + expect(screen.queryByTestId('restore-default-dataset-size')).not.toBeInTheDocument() + expect(screen.getByRole('complementary', { name: 'Run preview' })) + .toHaveTextContent('No additional objective cap') + }) + + it('filters datasets and sends the exact changed selection to estimate and launch', async () => { const user = userEvent.setup() renderDetail('/scenarios/foundry.red_team_agent') await screen.findByTestId('scenario-target-select') + await user.type(screen.getByRole('textbox', { name: 'Search datasets' }), 'ds_') + expect(screen.queryByTestId('dataset-harmbench')).not.toBeInTheDocument() + expect(screen.getByTestId('dataset-ds_a')).toBeInTheDocument() + await user.click(screen.getByTestId('dataset-ds_a')) + await user.clear(screen.getByRole('textbox', { name: 'Search datasets' })) + await user.click(screen.getByTestId('dataset-harmbench')) + expect(screen.getByText('1 dataset selected')).toBeInTheDocument() + expect(screen.getByRole('complementary', { name: 'Run preview' })).toHaveTextContent('ds_a') + expect(screen.getByRole('complementary', { name: 'Run preview' })).not.toHaveTextContent('harmbench') await user.click(screen.getByRole('button', { name: 'Advanced options' })) - await user.type(screen.getByTestId('dataset-override-input'), 'ds_a, ds_b') - await user.type(screen.getByTestId('max-dataset-size-input'), '25') + await user.type(screen.getByTestId('advanced-max_dataset_size'), '25') await user.click(screen.getByTestId('launch-scenario-btn')) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) const request = mockStartRun.mock.calls[0][0] - expect(request.dataset_names).toEqual(['ds_a', 'ds_b']) + expect(request.dataset_names).toEqual(['ds_a']) expect(request.max_dataset_size).toBe(25) await waitFor(() => expect(mockEstimateRun).toHaveBeenLastCalledWith( 'foundry.red_team_agent', expect.objectContaining({ target_name: 'target-a', techniques: ['default_technique'], - dataset_names: ['ds_a', 'ds_b'], + dataset_names: ['ds_a'], max_dataset_size: 25, include_baseline: true, }), @@ -648,21 +1919,121 @@ describe('ScenarioDetail', () => { expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('labels') }) + it('restores dataset defaults and removes the estimate and launch override', async () => { + const user = userEvent.setup() + renderDetail('/scenarios/foundry.red_team_agent') + await screen.findByTestId('dataset-ds_a') + + await user.click(screen.getByTestId('dataset-ds_a')) + await user.click(screen.getByTestId('dataset-harmbench')) + expect(screen.getByTestId('restore-default-datasets')).toBeEnabled() + await user.click(screen.getByTestId('restore-default-datasets')) + expect(screen.getByTestId('dataset-harmbench')).toBeChecked() + expect(screen.getByTestId('dataset-ds_a')).not.toBeChecked() + + await user.click(screen.getByTestId('launch-scenario-btn')) + await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) + expect(mockStartRun.mock.calls[0][0]).not.toHaveProperty('dataset_names') + await waitFor(() => expect(mockEstimateRun).toHaveBeenLastCalledWith( + 'foundry.red_team_agent', + expect.not.objectContaining({ dataset_names: expect.anything() }), + expect.any(AbortSignal), + )) + }) + + it('requires one dataset when the scenario declares defaults', async () => { + const user = userEvent.setup() + renderDetail('/scenarios/foundry.red_team_agent') + await screen.findByTestId('dataset-harmbench') + + await user.click(screen.getByTestId('dataset-harmbench')) + + expect(screen.getAllByText('Select at least one dataset.')).not.toHaveLength(0) + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + expect(mockStartRun).not.toHaveBeenCalled() + }) + + it('keeps scenario defaults usable when the dataset catalog fails', async () => { + mockListDatasets.mockRejectedValueOnce({ + isAxiosError: true, + response: { status: 503, data: { detail: 'Catalog unavailable' } }, + }) + renderDetail('/scenarios/foundry.red_team_agent') + + expect(await screen.findByTestId('dataset-catalog-error')).toHaveTextContent( + 'Registered datasets couldn’t be loaded. Scenario defaults remain available. Catalog unavailable', + ) + expect(screen.getByTestId('dataset-harmbench')).toBeChecked() + expect(screen.getByTestId('launch-scenario-btn')).not.toBeDisabled() + }) + + it('shows a loading state without hiding known scenario defaults', async () => { + mockListDatasets.mockReturnValueOnce(new Promise(() => {})) + renderDetail('/scenarios/foundry.red_team_agent') + + expect(await screen.findByTestId('dataset-catalog-loading')).toBeInTheDocument() + expect(screen.getByTestId('dataset-harmbench')).toBeChecked() + }) + + it('exposes the bounded dataset picker to keyboard and assistive technology', async () => { + const user = userEvent.setup() + renderDetail('/scenarios/foundry.red_team_agent') + await screen.findByTestId('dataset-ds_a') + + expect(screen.getByRole('group', { name: 'Datasets' })).toBeInTheDocument() + expect(screen.getByRole('textbox', { name: 'Search datasets' })).toBeInTheDocument() + const dataset = screen.getByRole('checkbox', { name: 'ds_a' }) + dataset.focus() + await user.keyboard('[Space]') + expect(dataset).toBeChecked() + }) + it('rejects a non-positive-integer max dataset size', async () => { const user = userEvent.setup() renderDetail('/scenarios/foundry.red_team_agent') await screen.findByTestId('scenario-target-select') await user.click(screen.getByRole('button', { name: 'Advanced options' })) - await user.type(screen.getByTestId('max-dataset-size-input'), '0') - await user.click(screen.getByTestId('launch-scenario-btn')) + expect(screen.getByText(/Maximum times to resume the scenario after an exception/)).toBeInTheDocument() + expect(screen.queryByText(/separate from Adaptive trying another technique/)).not.toBeInTheDocument() + await user.type(screen.getByTestId('advanced-max_dataset_size'), '0') - expect(await screen.findByRole('alert')).toHaveTextContent( - 'Max dataset size must be a positive integer.', - ) + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + expect(screen.getByText('Enter a whole number of 1 or more.')).toBeInTheDocument() expect(mockStartRun).not.toHaveBeenCalled() }) + it('blocks signed and decimal dataset-size input before it changes the controlled value', async () => { + const scenario = makeAdaptiveScenario() + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockImplementation( + async (_scenarioName, request: ScenarioRunSizeEstimateRequest) => + makeFullyCompatibleAdaptiveEstimateForRequest(scenario, request), + ) + const user = userEvent.setup() + renderDetail('/scenarios/adaptive.text_adaptive') + await screen.findByTestId('scenario-target-select') + await user.click(screen.getByRole('button', { name: 'Advanced options' })) + const input = screen.getByTestId('advanced-max_dataset_size') + await waitFor(() => expect(mockEstimateRun).toHaveBeenCalled()) + const requestCount = mockEstimateRun.mock.calls.length + + await user.clear(input) + await user.type(input, '-8') + expect(input).toHaveValue(null) + expect(input).toHaveAttribute('aria-invalid', 'true') + expect(screen.getByText('Enter a whole number of 1 or more.')).toBeInTheDocument() + await new Promise((resolve) => window.setTimeout(resolve, 350)) + expect(mockEstimateRun).toHaveBeenCalledTimes(requestCount) + + await user.click(input) + await user.paste('1.5') + expect(input).toHaveValue(null) + expect(screen.getByRole('complementary', { name: 'Run preview' })) + .toHaveTextContent('4 per dataset (scenario default)') + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + }) + it('validates advanced concurrency and retry bounds before launching', async () => { const user = userEvent.setup() renderDetail('/scenarios/foundry.red_team_agent') @@ -671,11 +2042,11 @@ describe('ScenarioDetail', () => { await user.click(screen.getByRole('button', { name: 'Advanced options' })) fireEvent.change(screen.getByTestId('max-concurrency-input'), { target: { value: '500' } }) fireEvent.blur(screen.getByTestId('max-concurrency-input')) - await user.click(screen.getByTestId('launch-scenario-btn')) - expect(await screen.findByRole('alert')).toHaveTextContent( + expect(screen.getByTestId('launch-scenario-btn')).toBeDisabled() + expect(screen.getByText( 'Max concurrency must be an integer from 1 to 100.', - ) + )).toBeInTheDocument() expect(mockStartRun).not.toHaveBeenCalled() }) @@ -740,6 +2111,9 @@ describe('ScenarioDetail', () => { version: 1, status: 'exact', total_attack_count: 8, + minimum_attack_count: null, + maximum_attack_count: null, + condition: null, components: [ { label: 'Prompt sending', @@ -771,14 +2145,16 @@ describe('ScenarioDetail', () => { selection_note: 'One incompatible group is excluded.', }, ], - note: 'The backend total is authoritative.', + adaptive_details: null, + note: 'The planned total is authoritative.', retries_included: false, }) renderDetail('/scenarios/airt.jailbreak') await screen.findByTestId('scenario-target-select') - await user.click(screen.getByTestId('technique-prompt_sending')) + await user.click(screen.getByTestId('technique-mode-custom')) + await user.click(screen.getByTestId('technique-jailbreak_system_prompt')) await user.clear(screen.getByTestId('scenario-param-num_jailbreaks')) await user.type(screen.getByTestId('scenario-param-num_jailbreaks'), '2') await user.clear(screen.getByTestId('scenario-param-num_jailbreak_attempts')) @@ -815,11 +2191,14 @@ describe('ScenarioDetail', () => { )) const preview = screen.getByRole('complementary', { name: 'Run preview' }) expect(within(preview).getByText('prompt_sending')).toBeInTheDocument() - expect(within(preview).getAllByText('harmbench')).toHaveLength(2) + expect(within(preview).getByText('harmbench')).toBeInTheDocument() expect(within(preview).getByText('Not included')).toBeInTheDocument() - expect(within(preview).getByText('8 planned attacks')).toBeInTheDocument() - expect(within(preview).getByText('Jailbreak templates: 2 (configuration)')).toBeInTheDocument() - expect(within(preview).getByText('2')).toBeInTheDocument() + expect(within(preview).getByRole('group', { + name: '1 technique multiplied by 4 objectives multiplied by 2 jailbreak templates multiplied by 1 attempt equals 8 planned attacks.', + })).toBeInTheDocument() + expect(within(preview).getByText('4 objectives from harmbench · 5 available')).toBeInTheDocument() + expect(within(preview).getByText('Jailbreak templates: 2')).toBeInTheDocument() + expect(within(preview).queryByText(/logical seed groups|selected seed groups/i)).not.toBeInTheDocument() await user.click(screen.getByTestId('launch-scenario-btn')) @@ -915,7 +2294,7 @@ describe('ScenarioDetail', () => { await screen.findByTestId('scenario-target-select') await user.selectOptions(screen.getByTestId('scenario-target-select'), 'target-b') - await user.click(screen.getByTestId('technique-crescendo')) + await user.click(screen.getByTestId('technique-mode-custom')) await user.clear(screen.getByTestId('scenario-param-attempts')) await user.type(screen.getByTestId('scenario-param-attempts'), '3') await user.click(screen.getByTestId('launch-scenario-btn')) diff --git a/frontend/src/components/Scenarios/ScenarioDetail.tsx b/frontend/src/components/Scenarios/ScenarioDetail.tsx index 408a2ef146..d4f0033fec 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.tsx @@ -23,19 +23,22 @@ import { ArrowLeftRegular, ArrowSyncRegular, SettingsRegular } from '@fluentui/r import { Link, useNavigate, useParams } from 'react-router' import MarkdownContent from '@/components/Markdown/MarkdownContent' -import ParameterField from '@/components/Parameters/ParameterField' +import ParameterField, { + type RejectedNumberInputReason, +} from '@/components/Parameters/ParameterField' import { buildParametersFromForm, getInitialFormValues, type ParameterFormValue, } from '@/components/Parameters/parameterForm' import type { ViewName } from '@/components/Sidebar/Navigation' -import { scenariosApi, targetsApi } from '@/services/api' +import { datasetsApi, scenariosApi, targetsApi } from '@/services/api' import { toApiError } from '@/services/errors' import type { Parameter, RegisteredScenario, RunScenarioRequest, + ScenarioDatasetSizeLimit, ScenarioRunEstimateResult, ScenarioRunSizeEstimateRequest, ScenarioRunEstimateState, @@ -46,8 +49,14 @@ import { routerPathParamValue } from '@/utils/routeParams' import { useScenarioDetailStyles } from './ScenarioDetail.styles' import { ScenarioRunEstimateDetails } from './ScenarioRunEstimate' +import { formatAdaptiveCapFeedback } from './scenarioAdaptiveCap' import { normalizeScenarioMarkdown } from './scenarioMarkdown' import { mapScenarioRunEstimate } from './scenarioRunEstimateAdapter' +import { + techniqueSetDisplayName, + techniqueSetMembers, + techniqueSetOptionLabel, +} from './scenarioTechniqueSets' /** Items requested per target page while paging through the full list. */ const TARGET_PAGE_SIZE = 200 @@ -77,6 +86,27 @@ const MAX_MAX_RETRIES = 20 const DEFAULT_MAX_CONCURRENCY = 10 const DEFAULT_MAX_RETRIES = 0 const ESTIMATE_DEBOUNCE_MS = 300 +const TEXT_ADAPTIVE_SCENARIO_NAME = 'adaptive.text_adaptive' +const CUSTOM_TECHNIQUE_SET_VALUE = '__custom__' +const MAX_ATTEMPTS_PARAMETER_NAME = 'max_attempts_per_objective' +const MAX_ATTEMPTS_DISPLAY_LABEL = 'Maximum techniques per objective' +const MAX_ATTEMPTS_DEFAULT_HINT = 'Leave blank to use the default of 3.' +const MAX_ATTEMPTS_BEHAVIOR_HINT = [ + 'This is a per-objective limit, not a total-run budget.', + 'Adaptive stops after the first success, and incompatible techniques are skipped.', + 'This is separate from retries.', +].join(' ') +const MAX_ATTEMPTS_VALIDATION_MESSAGE = 'Enter a whole number of 1 or more.' +const MAX_DATASET_SIZE_VALIDATION_MESSAGE = 'Enter a whole number of 1 or more.' +const CORRECT_HIGHLIGHTED_SETTING_MESSAGE = 'Correct the highlighted setting to calculate this run.' +const MAX_DATASET_SIZE_PARAMETER: Parameter = { + name: 'max_dataset_size', + type_name: 'int', + required: false, + default: null, + choices: null, + is_list: false, +} /** Resolves a Fluent `SpinButton` change event to a numeric value, preferring the parsed `value` over the raw `displayValue`. */ function resolveSpinButtonValue(data: { value?: number | null; displayValue?: string }, previous: number): number { @@ -96,13 +126,13 @@ type TechniqueSelection = } | { mode: 'custom' - techniques: string[] } interface TechniqueOptions { presets: string[] concrete: string[] defaultSelection: TechniqueSelection + initialCustomTechniques: string[] } /** Options rendered for technique selection: exclusive presets first, then concrete techniques. */ @@ -130,19 +160,21 @@ function uniqueTechniqueOptions(scenario: RegisteredScenario): TechniqueOptions } const defaultSelection: TechniqueSelection = defaultIsPreset ? { mode: 'preset', preset: scenario.default_technique } - : { mode: 'custom', techniques: [scenario.default_technique] } - return { presets, concrete, defaultSelection } + : { mode: 'custom' } + const initialCustomTechniques = defaultIsPreset ? [] : [scenario.default_technique] + return { presets, concrete, defaultSelection, initialCustomTechniques } } -function selectedTechniqueNames(selection: TechniqueSelection): string[] { - return selection.mode === 'preset' ? [selection.preset] : selection.techniques +function sameStringSet(left: string[], right: string[]): boolean { + const leftSet = new Set(left) + const rightSet = new Set(right) + return leftSet.size === rightSet.size && [...leftSet].every((value) => rightSet.has(value)) } -function parseDatasetNames(datasetOverride: string): string[] { - return datasetOverride - .split(',') - .map((entry) => entry.trim()) - .filter((entry) => entry.length > 0) +function parameterDisplayLabel(parameter: Parameter, usesAdaptiveTechniqueSelection: boolean): string { + return usesAdaptiveTechniqueSelection && parameter.name === MAX_ATTEMPTS_PARAMETER_NAME + ? MAX_ATTEMPTS_DISPLAY_LABEL + : parameter.name } function formatParameterPreview(value: ParameterFormValue | undefined): string { @@ -152,14 +184,91 @@ function formatParameterPreview(value: ParameterFormValue | undefined): string { return value?.trim() || 'Not set' } +function maxAttemptsValidationError(value: ParameterFormValue | undefined): string | undefined { + const raw = typeof value === 'string' ? value.trim() : '' + if (raw.length === 0) { + return undefined + } + const parsed = Number(raw) + return Number.isSafeInteger(parsed) && parsed >= 1 + ? undefined + : MAX_ATTEMPTS_VALIDATION_MESSAGE +} + +function datasetSizeFieldLabel(limit: ScenarioDatasetSizeLimit): string { + if (limit.override_scope === 'unsupported') { + return 'Maximum objectives' + } + return limit.override_scope === 'per_dataset' + ? 'Maximum objectives per dataset' + : 'Maximum objectives across selected datasets' +} + +function datasetSizeHint(limit: ScenarioDatasetSizeLimit): string { + if (limit.override_scope === 'unsupported') { + return 'This scenario manages its objective population directly and does not support a dataset-size override.' + } + if (limit.default_scope === 'per_dataset' && limit.default_count !== null) { + return `Scenario default: up to ${limit.default_count.toLocaleString()} objectives from each selected dataset. Enter another whole number to override it, or leave blank to use the scenario default.` + } + if (limit.default_scope === 'combined' && limit.default_count !== null) { + return `Scenario default: up to ${limit.default_count.toLocaleString()} objectives across the selected datasets. Enter another whole number to override it, or leave blank to use the scenario default.` + } + if (limit.default_scope === 'heterogeneous') { + const replacement = limit.override_scope === 'per_dataset' + ? 'a uniform per-dataset maximum' + : 'a combined maximum' + return `Scenario defaults vary by dataset. Enter a whole number to replace them with ${replacement}, or leave blank to keep the scenario defaults.` + } + const scope = limit.override_scope === 'per_dataset' + ? 'objectives from each selected dataset' + : 'objectives across the selected datasets' + return `No scenario default cap. Enter a whole number to limit ${scope}, or leave blank for no additional cap.` +} + +function formatDatasetSizePreview( + limit: ScenarioDatasetSizeLimit, + maxDatasetSize: string, + hasOverride: boolean, +): string { + const parsed = Number(maxDatasetSize.trim()) + if (hasOverride && Number.isSafeInteger(parsed) && parsed >= 1) { + return limit.override_scope === 'per_dataset' + ? `${parsed.toLocaleString()} per dataset (override)` + : `${parsed.toLocaleString()} total (override)` + } + if (limit.default_scope === 'per_dataset' && limit.default_count !== null) { + return `${limit.default_count.toLocaleString()} per dataset (scenario default)` + } + if (limit.default_scope === 'combined' && limit.default_count !== null) { + return `${limit.default_count.toLocaleString()} total (scenario default)` + } + if (limit.default_scope === 'heterogeneous') { + return 'Varies by dataset (scenario default)' + } + return 'No additional objective cap' +} + +function maxDatasetSizeValidationError(value: string): string | undefined { + const raw = value.trim() + if (raw.length === 0) { + return undefined + } + const parsed = Number(raw) + return Number.isSafeInteger(parsed) && parsed >= 1 + ? undefined + : MAX_DATASET_SIZE_VALIDATION_MESSAGE +} + interface BuildRunRequestInput { scenario: RegisteredScenario targetName: string techniques: string[] dynamicParameters: Parameter[] scenarioParamValues: Record - datasetOverride: string + selectedDatasets: string[] maxDatasetSize: string + hasMaxDatasetSizeOverride: boolean maxConcurrency: number maxRetries: number includeBaseline: boolean @@ -176,11 +285,6 @@ type BuildRunRequestResult = error: string } -type SuccessfulEstimateResult = Extract< - ScenarioRunEstimateResult, - { status: 'available' | 'conditional' } -> - type EstimateRequestState = | { status: 'resolved' @@ -190,8 +294,41 @@ type EstimateRequestState = | { status: 'error' requestKey: string - error: string + summary: string + note?: string + maxAttemptsError?: string + } + +interface MappedEstimateError { + summary: string + note?: string + maxAttemptsError?: string +} + +interface AdaptiveCandidateMetadata { + scopeKey: string + maximum: number +} + +interface AdaptiveLimitNotice { + scopeKey: string + message: string + validationState: 'none' | 'warning' +} + +function mapEstimateError(error: unknown): MappedEstimateError { + const detail = toApiError(error).detail + if (detail.includes(MAX_ATTEMPTS_PARAMETER_NAME)) { + return { + summary: CORRECT_HIGHLIGHTED_SETTING_MESSAGE, + maxAttemptsError: MAX_ATTEMPTS_VALIDATION_MESSAGE, } + } + return { + summary: 'Run size couldn’t be updated.', + note: detail, + } +} function buildRunRequest({ scenario, @@ -199,8 +336,9 @@ function buildRunRequest({ techniques, dynamicParameters, scenarioParamValues, - datasetOverride, + selectedDatasets, maxDatasetSize, + hasMaxDatasetSizeOverride, maxConcurrency, maxRetries, includeBaseline, @@ -212,6 +350,17 @@ function buildRunRequest({ if (techniques.length === 0) { return { ok: false, error: 'Select at least one technique.' } } + if (scenario.default_datasets.length > 0 && selectedDatasets.length === 0) { + return { ok: false, error: 'Select at least one dataset.' } + } + if (dynamicParameters.some((parameter) => parameter.name === MAX_ATTEMPTS_PARAMETER_NAME)) { + const maxAttemptsError = maxAttemptsValidationError( + scenarioParamValues[MAX_ATTEMPTS_PARAMETER_NAME], + ) + if (maxAttemptsError) { + return { ok: false, error: maxAttemptsError } + } + } let scenarioParams: Record | null = null if (dynamicParameters.length > 0) { @@ -229,7 +378,12 @@ function buildRunRequest({ if (!Number.isInteger(parsed) || parsed < 1) { return { ok: false, error: 'Max dataset size must be a positive integer.' } } - maxDatasetSizeValue = parsed + if (hasMaxDatasetSizeOverride) { + if (scenario.dataset_size_limit.override_scope === 'unsupported') { + return { ok: false, error: 'This scenario does not support a dataset-size override.' } + } + maxDatasetSizeValue = parsed + } } if ( !Number.isInteger(maxConcurrency) @@ -252,7 +406,6 @@ function buildRunRequest({ } } - const datasetNames = parseDatasetNames(datasetOverride) const request: RunScenarioRequest = { scenario_name: scenario.scenario_name, target_name: targetName, @@ -262,8 +415,8 @@ function buildRunRequest({ include_baseline: includeBaseline, labels, } - if (datasetNames.length > 0) { - request.dataset_names = datasetNames + if (!sameStringSet(selectedDatasets, scenario.default_datasets)) { + request.dataset_names = selectedDatasets } if (maxDatasetSizeValue !== undefined) { request.max_dataset_size = maxDatasetSizeValue @@ -295,6 +448,121 @@ function buildEstimateRequest(request: RunScenarioRequest): ScenarioRunSizeEstim return estimateRequest } +type DatasetCatalogStatus = 'loading' | 'success' | 'error' + +interface DatasetPickerProps { + availableDatasets: string[] + defaultDatasets: string[] + selectedDatasets: string[] + status: DatasetCatalogStatus + error: string | null + disabled: boolean + invalid: boolean + onChange: (name: string, checked: boolean) => void + onRestoreDefaults: () => void +} + +function DatasetPicker({ + availableDatasets, + defaultDatasets, + selectedDatasets, + status, + error, + disabled, + invalid, + onChange, + onRestoreDefaults, +}: DatasetPickerProps) { + const styles = useScenarioDetailStyles() + const [query, setQuery] = useState('') + const selectedSet = useMemo(() => new Set(selectedDatasets), [selectedDatasets]) + const defaultSet = useMemo(() => new Set(defaultDatasets), [defaultDatasets]) + const orderedDatasets = useMemo(() => { + const names = [...new Set([...availableDatasets, ...defaultDatasets])] + return names.sort((left, right) => { + const leftPriority = selectedSet.has(left) ? 0 : defaultSet.has(left) ? 1 : 2 + const rightPriority = selectedSet.has(right) ? 0 : defaultSet.has(right) ? 1 : 2 + return leftPriority - rightPriority || left.localeCompare(right) + }) + }, [availableDatasets, defaultDatasets, defaultSet, selectedSet]) + const normalizedQuery = query.trim().toLocaleLowerCase() + const visibleDatasets = normalizedQuery.length === 0 + ? orderedDatasets + : orderedDatasets.filter((name) => name.toLocaleLowerCase().includes(normalizedQuery)) + const selectedCount = selectedDatasets.length + + return ( + <> +
+ + {selectedCount.toLocaleString()} dataset{selectedCount === 1 ? '' : 's'} selected + + +
+ + setQuery(data.value)} + placeholder="Search datasets" + aria-label="Search datasets" + data-testid="dataset-search-input" + /> + + {status === 'loading' && ( + + Loading registered datasets… + + )} + {status === 'error' && ( + + + Registered datasets couldn’t be loaded. Scenario defaults remain available. + {error ? ` ${error}` : ''} + + + )} +
+ {visibleDatasets.length > 0 ? ( + visibleDatasets.map((name) => ( + onChange(name, data.checked === true)} + data-testid={`dataset-${name}`} + /> + )) + ) : ( + No datasets match this search. + )} +
+ + ) +} + interface ScenarioDetailProps { activeTarget: TargetInstance | null labels: Record @@ -485,7 +753,7 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari const navigate = useNavigate() const formId = `scenario-launch-${encodeURIComponent(scenario.scenario_name).replace(/%/g, '-')}` - const { presets, concrete, defaultSelection } = useMemo( + const { presets, concrete, defaultSelection, initialCustomTechniques } = useMemo( () => uniqueTechniqueOptions(scenario), [scenario], ) @@ -496,6 +764,7 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari [scenario.supported_parameters], ) const isBaselineForbidden = scenario.baseline_policy === 'forbidden' + const usesAdaptiveTechniqueSelection = scenario.scenario_name === TEXT_ADAPTIVE_SCENARIO_NAME const [targetName, setTargetName] = useState(() => { if (activeTarget && targets.some((target) => @@ -505,29 +774,63 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari return targets[0].target_registry_name }) const [techniqueSelection, setTechniqueSelection] = useState(() => defaultSelection) + const [customTechniques, setCustomTechniques] = useState(() => initialCustomTechniques) const [baselineChecked, setBaselineChecked] = useState( () => !isBaselineForbidden && scenario.include_baseline_by_default, ) - const [datasetOverride, setDatasetOverride] = useState('') - const [maxDatasetSize, setMaxDatasetSize] = useState('') + const [availableDatasets, setAvailableDatasets] = useState(() => [...scenario.default_datasets]) + const [selectedDatasets, setSelectedDatasets] = useState(() => [...scenario.default_datasets]) + const [datasetCatalogStatus, setDatasetCatalogStatus] = useState('loading') + const [datasetCatalogError, setDatasetCatalogError] = useState(null) + const [maxDatasetSize, setMaxDatasetSize] = useState(() => { + const limit = scenario.dataset_size_limit + return limit.default_count !== null && limit.default_scope === limit.override_scope + ? String(limit.default_count) + : '' + }) + const [hasMaxDatasetSizeOverride, setHasMaxDatasetSizeOverride] = useState(false) + const [maxDatasetSizeInputRejected, setMaxDatasetSizeInputRejected] = useState(false) const [maxConcurrency, setMaxConcurrency] = useState(DEFAULT_MAX_CONCURRENCY) const [maxRetries, setMaxRetries] = useState(DEFAULT_MAX_RETRIES) - const [scenarioParamValues, setScenarioParamValues] = useState>(() => - getInitialFormValues(dynamicParameters), - ) + const [scenarioParamValues, setScenarioParamValues] = useState>(() => { + const initialValues = getInitialFormValues(dynamicParameters) + if (usesAdaptiveTechniqueSelection && MAX_ATTEMPTS_PARAMETER_NAME in initialValues) { + initialValues[MAX_ATTEMPTS_PARAMETER_NAME] = '' + } + return initialValues + }) const [validationError, setValidationError] = useState(null) const [apiError, setApiError] = useState(null) const [submitting, setSubmitting] = useState(false) const [estimateRequestState, setEstimateRequestState] = useState(null) - const [lastGoodEstimate, setLastGoodEstimate] = useState(null) + const [launchMaxAttemptsError, setLaunchMaxAttemptsError] = useState(null) + const [maxAttemptsInputRejected, setMaxAttemptsInputRejected] = useState(false) + const [adaptiveCandidateMetadata, setAdaptiveCandidateMetadata] = + useState(null) + const [adaptiveLimitNotice, setAdaptiveLimitNotice] = useState(null) // Synchronous guard against a double-submit racing ahead of the state update. const isSubmittingRef = useRef(false) const estimateSequenceRef = useRef(0) + const customTechniquesInitializedRef = useRef(defaultSelection.mode === 'custom') + const hasResolvedAdaptiveMetadataRef = useRef(false) const techniques = useMemo( - () => selectedTechniqueNames(techniqueSelection), - [techniqueSelection], + () => techniqueSelection.mode === 'preset' + ? [techniqueSelection.preset] + : customTechniques, + [customTechniques, techniqueSelection], + ) + const adaptiveCandidateScopeKey = useMemo( + () => JSON.stringify({ targetName, techniques }), + [targetName, techniques], ) + const knownAdaptiveCandidateMaximum = + adaptiveCandidateMetadata?.scopeKey === adaptiveCandidateScopeKey + ? adaptiveCandidateMetadata.maximum + : null + const adaptiveSelectionDisplayName = techniqueSelection.mode === 'preset' + ? techniqueSetDisplayName(scenario, techniqueSelection.preset) + : 'Custom selection' const requestResult = useMemo( () => buildRunRequest({ scenario, @@ -535,8 +838,9 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari techniques, dynamicParameters, scenarioParamValues, - datasetOverride, + selectedDatasets, maxDatasetSize, + hasMaxDatasetSizeOverride, maxConcurrency, maxRetries, includeBaseline: isBaselineForbidden ? false : baselineChecked, @@ -544,22 +848,51 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari }), [ baselineChecked, - datasetOverride, dynamicParameters, isBaselineForbidden, labels, maxConcurrency, maxDatasetSize, + hasMaxDatasetSizeOverride, maxRetries, scenario, scenarioParamValues, + selectedDatasets, targetName, techniques, ], ) const estimateRequest = useMemo( - () => requestResult.ok ? buildEstimateRequest(requestResult.request) : null, - [requestResult], + () => { + if (!requestResult.ok || maxAttemptsInputRejected || maxDatasetSizeInputRejected) { + return null + } + const request = buildEstimateRequest(requestResult.request) + if (!usesAdaptiveTechniqueSelection || !request.scenario_params) { + return request + } + const scenarioParams = { ...request.scenario_params } + const configuredMaximum = scenarioParams[MAX_ATTEMPTS_PARAMETER_NAME] + if (knownAdaptiveCandidateMaximum === null || knownAdaptiveCandidateMaximum === 0) { + delete scenarioParams[MAX_ATTEMPTS_PARAMETER_NAME] + } else if ( + typeof configuredMaximum === 'number' + && configuredMaximum > knownAdaptiveCandidateMaximum + ) { + scenarioParams[MAX_ATTEMPTS_PARAMETER_NAME] = knownAdaptiveCandidateMaximum + } + return { + ...request, + scenario_params: Object.keys(scenarioParams).length > 0 ? scenarioParams : undefined, + } + }, + [ + knownAdaptiveCandidateMaximum, + maxAttemptsInputRejected, + maxDatasetSizeInputRejected, + requestResult, + usesAdaptiveTechniqueSelection, + ], ) const estimateRequestKey = useMemo( () => estimateRequest === null @@ -569,12 +902,36 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari ) useEffect(() => { - if (estimateRequest === null || estimateRequestKey === null) { - return + let cancelled = false + datasetsApi + .listDatasets() + .then((response) => { + if (cancelled) return + setAvailableDatasets([...new Set([ + ...scenario.default_datasets, + ...response.items.map((item) => item.name), + ])]) + setDatasetCatalogStatus('success') + setDatasetCatalogError(null) + }) + .catch((err: unknown) => { + if (cancelled) return + setAvailableDatasets([...scenario.default_datasets]) + setDatasetCatalogStatus('error') + setDatasetCatalogError(toApiError(err).detail) + }) + return () => { + cancelled = true } + }, [scenario.default_datasets]) + useEffect(() => { const requestSequence = estimateSequenceRef.current + 1 estimateSequenceRef.current = requestSequence + if (estimateRequest === null || estimateRequestKey === null) { + return + } + const controller = new AbortController() const debounceTimer = window.setTimeout(() => { @@ -588,14 +945,60 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari return } const result = mapScenarioRunEstimate(response, 'request') + const adaptiveDetails = + result.status === 'available' || result.status === 'conditional' + ? result.estimate.adaptiveDetails + : null + if (usesAdaptiveTechniqueSelection && adaptiveDetails) { + const maximum = adaptiveDetails.candidateTechniqueCount + const hadResolvedAdaptiveMetadata = hasResolvedAdaptiveMetadataRef.current + hasResolvedAdaptiveMetadataRef.current = true + setAdaptiveCandidateMetadata({ scopeKey: adaptiveCandidateScopeKey, maximum }) + const rawValue = scenarioParamValues[MAX_ATTEMPTS_PARAMETER_NAME] + const parsedValue = typeof rawValue === 'string' && rawValue.trim() !== '' + ? Number(rawValue) + : null + const configuredValue = parsedValue ?? adaptiveDetails.maxAttemptsPerObjective + const isDefaultReduction = parsedValue === null + if ( + maximum > 0 + && Number.isSafeInteger(configuredValue) + && configuredValue > maximum + ) { + setScenarioParamValues((current) => ({ + ...current, + [MAX_ATTEMPTS_PARAMETER_NAME]: String(maximum), + })) + setAdaptiveLimitNotice( + isDefaultReduction + ? { + scopeKey: adaptiveCandidateScopeKey, + message: `The scenario default of ${configuredValue.toLocaleString()} is reduced to ${ + maximum.toLocaleString() + } because ${adaptiveSelectionDisplayName} provides ${maximum.toLocaleString()} compatible ${ + maximum === 1 ? 'technique' : 'techniques' + } for this target.`, + validationState: 'none', + } + : hadResolvedAdaptiveMetadata + ? { + scopeKey: adaptiveCandidateScopeKey, + message: `Reduced to ${maximum.toLocaleString()} because ${ + adaptiveSelectionDisplayName + } provides ${maximum.toLocaleString()} compatible ${ + maximum === 1 ? 'technique' : 'techniques' + } for this target.`, + validationState: 'warning', + } + : null, + ) + } + } setEstimateRequestState({ status: 'resolved', requestKey: estimateRequestKey, result, }) - if (result.status === 'available' || result.status === 'conditional') { - setLastGoodEstimate(result) - } }) .catch((err: unknown) => { if ( @@ -604,10 +1007,11 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari ) { return } + const mappedError = mapEstimateError(err) setEstimateRequestState({ status: 'error', requestKey: estimateRequestKey, - error: toApiError(err).detail, + ...mappedError, }) }) }, ESTIMATE_DEBOUNCE_MS) @@ -616,76 +1020,226 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari window.clearTimeout(debounceTimer) controller.abort() } - }, [estimateRequest, estimateRequestKey, scenario.scenario_name]) + }, [ + adaptiveCandidateScopeKey, + adaptiveSelectionDisplayName, + estimateRequest, + estimateRequestKey, + scenario.scenario_name, + scenarioParamValues, + usesAdaptiveTechniqueSelection, + ]) + + const currentResolvedEstimate = estimateRequestState?.requestKey === estimateRequestKey + && estimateRequestState.status === 'resolved' + ? estimateRequestState.result + : null + const currentResolvedRunEstimate = currentResolvedEstimate + && (currentResolvedEstimate.status === 'available' || currentResolvedEstimate.status === 'conditional') + ? currentResolvedEstimate.estimate + : null + const currentResolvedAdaptiveDetails = currentResolvedRunEstimate + ? currentResolvedRunEstimate.adaptiveDetails + : null + const adaptiveCandidateMaximum = currentResolvedAdaptiveDetails?.candidateTechniqueCount + ?? knownAdaptiveCandidateMaximum + const adaptiveCandidateAvailability = adaptiveCandidateMaximum === null + ? undefined + : `${adaptiveSelectionDisplayName} provides ${adaptiveCandidateMaximum.toLocaleString()} compatible ${ + adaptiveCandidateMaximum === 1 ? 'technique' : 'techniques' + } for this target.` + const maxAttemptsParameter = dynamicParameters.find( + (parameter) => parameter.name === MAX_ATTEMPTS_PARAMETER_NAME, + ) + const maxAttemptsDefault = Number(maxAttemptsParameter?.default ?? 3) + const adaptiveDefaultIsReduced = usesAdaptiveTechniqueSelection + && adaptiveCandidateMaximum !== null + && adaptiveCandidateMaximum > 0 + && Number.isSafeInteger(maxAttemptsDefault) + && maxAttemptsDefault > adaptiveCandidateMaximum + const adaptiveDefaultHint = adaptiveDefaultIsReduced + ? `Blank restores the bounded default of ${adaptiveCandidateMaximum.toLocaleString()} techniques per objective for this target.` + : MAX_ATTEMPTS_DEFAULT_HINT + const maxAttemptsRawValue = scenarioParamValues[MAX_ATTEMPTS_PARAMETER_NAME] + const maxAttemptsNumericValue = typeof maxAttemptsRawValue === 'string' + && maxAttemptsRawValue.trim() !== '' + ? Number(maxAttemptsRawValue) + : null + const maxAttemptsExceedsCandidateMaximum = adaptiveCandidateMaximum !== null + && maxAttemptsNumericValue !== null + && maxAttemptsNumericValue > adaptiveCandidateMaximum + const adaptiveMetadataUnavailable = usesAdaptiveTechniqueSelection + && adaptiveCandidateMaximum === null + const noAdaptiveCandidatesError = usesAdaptiveTechniqueSelection && adaptiveCandidateMaximum === 0 + ? 'No compatible techniques are available for this target. Choose a different technique set or target.' + : undefined + const maxAttemptsClientError = usesAdaptiveTechniqueSelection + ? maxAttemptsInputRejected + ? MAX_ATTEMPTS_VALIDATION_MESSAGE + : maxAttemptsValidationError(scenarioParamValues[MAX_ATTEMPTS_PARAMETER_NAME]) + : undefined + const currentEstimateError = estimateRequestState?.requestKey === estimateRequestKey + && estimateRequestState.status === 'error' + ? estimateRequestState + : null + const maxAttemptsFieldError = maxAttemptsClientError + ?? noAdaptiveCandidatesError + ?? currentEstimateError?.maxAttemptsError + ?? launchMaxAttemptsError + ?? undefined + const maxDatasetSizeFieldError = maxDatasetSizeInputRejected + ? MAX_DATASET_SIZE_VALIDATION_MESSAGE + : maxDatasetSizeValidationError(maxDatasetSize) let estimateState: ScenarioRunEstimateState - if (!requestResult.ok) { + if (maxAttemptsFieldError || maxDatasetSizeFieldError) { + estimateState = { + status: 'unavailable', + scope: 'request', + label: CORRECT_HIGHLIGHTED_SETTING_MESSAGE, + } + } else if (!requestResult.ok) { estimateState = { status: 'unavailable', scope: 'request', label: 'Complete the required configuration to request an estimate.', note: requestResult.error, } - } else if ( - estimateRequestState?.requestKey === estimateRequestKey - && estimateRequestState.status === 'resolved' - ) { - estimateState = estimateRequestState.result - } else if ( - estimateRequestState?.requestKey === estimateRequestKey - && estimateRequestState.status === 'error' - ) { - estimateState = lastGoodEstimate - ? { - status: 'stale', - estimate: lastGoodEstimate.estimate, - label: 'Showing the last successful estimate.', - error: estimateRequestState.error, - } - : { - status: 'unavailable', - scope: 'request', - label: 'The backend estimate could not be refreshed.', - note: estimateRequestState.error, - } - } else if (lastGoodEstimate) { + } else if (currentResolvedEstimate) { + estimateState = currentResolvedEstimate + } else if (currentEstimateError) { estimateState = { - status: 'refreshing', - estimate: lastGoodEstimate.estimate, - label: 'Updating for the current configuration…', + status: 'unavailable', + scope: 'request', + label: currentEstimateError.summary, + note: currentEstimateError.note, } } else { estimateState = { status: 'loading', scope: 'request' } } + const estimateRequestBlocked = estimateRequestState?.requestKey === estimateRequestKey + && estimateRequestState.status === 'error' - const handlePresetChange = (preset: string): void => { - setTechniqueSelection({ mode: 'preset', preset }) + const handleTechniqueModeChange = (value: string): void => { + if (value === CUSTOM_TECHNIQUE_SET_VALUE) { + if (!customTechniquesInitializedRef.current) { + const members = techniqueSelection.mode === 'preset' + ? techniqueSetMembers(scenario, techniqueSelection.preset) + : [] + const concreteSet = new Set(concrete) + setCustomTechniques(members.filter((member) => concreteSet.has(member))) + customTechniquesInitializedRef.current = true + } + setTechniqueSelection({ mode: 'custom' }) + } else { + setTechniqueSelection({ mode: 'preset', preset: value }) + } setValidationError(null) } const handleConcreteChange = (name: string, checked: boolean): void => { - setTechniqueSelection((current) => { + setCustomTechniques((current) => { if (checked) { - if (current.mode === 'preset') { - return { mode: 'custom', techniques: [name] } - } - return current.techniques.includes(name) + return current.includes(name) ? current - : { mode: 'custom', techniques: [...current.techniques, name] } - } - if (current.mode === 'preset') { - return current + : [...current, name] } - return { - mode: 'custom', - techniques: current.techniques.filter((technique) => technique !== name), + return current.filter((technique) => technique !== name) + }) + setValidationError(null) + } + + const handleDatasetChange = (name: string, checked: boolean): void => { + setSelectedDatasets((current) => { + if (checked) { + return current.includes(name) ? current : [...current, name] } + return current.filter((dataset) => dataset !== name) }) setValidationError(null) } const updateScenarioParam = (name: string, value: ParameterFormValue): void => { setScenarioParamValues((current) => ({ ...current, [name]: value })) + if (name === MAX_ATTEMPTS_PARAMETER_NAME) { + setMaxAttemptsInputRejected(false) + setAdaptiveLimitNotice(null) + setLaunchMaxAttemptsError(null) + setApiError(null) + } + setValidationError(null) + } + + const rejectScenarioParamInput = ( + name: string, + reason: RejectedNumberInputReason, + retainedValue: string, + ): void => { + if (name !== MAX_ATTEMPTS_PARAMETER_NAME) { + return + } + + if (reason === 'above-max' && adaptiveCandidateMaximum !== null) { + setScenarioParamValues((current) => ({ + ...current, + [name]: String(adaptiveCandidateMaximum), + })) + setMaxAttemptsInputRejected(false) + setAdaptiveLimitNotice({ + scopeKey: adaptiveCandidateScopeKey, + message: `Maximum reached: ${adaptiveCandidateAvailability}`, + validationState: 'warning', + }) + setLaunchMaxAttemptsError(null) + setApiError(null) + setValidationError(null) + return + } + setScenarioParamValues((current) => ({ ...current, [name]: retainedValue })) + setMaxAttemptsInputRejected(true) + setAdaptiveLimitNotice(null) + setLaunchMaxAttemptsError(null) + setApiError(null) + setValidationError(null) + } + + const updateMaxDatasetSize = (_name: string, value: ParameterFormValue): void => { + const nextValue = typeof value === 'string' ? value : '' + const parsed = Number(nextValue.trim()) + const limit = scenario.dataset_size_limit + const matchesRepresentableDefault = nextValue.trim() !== '' + && limit.default_count !== null + && limit.default_scope === limit.override_scope + && parsed === limit.default_count + setMaxDatasetSize(nextValue) + setHasMaxDatasetSizeOverride(nextValue.trim() !== '' && !matchesRepresentableDefault) + setMaxDatasetSizeInputRejected(false) + setValidationError(null) + setApiError(null) + } + + const rejectMaxDatasetSizeInput = ( + _name: string, + _reason: RejectedNumberInputReason, + retainedValue: string, + ): void => { + setMaxDatasetSize(retainedValue) + setMaxDatasetSizeInputRejected(true) + setValidationError(null) + setApiError(null) + } + + const restoreDefaultDatasetSize = (): void => { + const limit = scenario.dataset_size_limit + setMaxDatasetSize( + limit.default_count !== null && limit.default_scope === limit.override_scope + ? String(limit.default_count) + : '', + ) + setHasMaxDatasetSizeOverride(false) + setMaxDatasetSizeInputRejected(false) + setValidationError(null) + setApiError(null) } const handleSubmit = async (): Promise => { @@ -694,6 +1248,14 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari } setApiError(null) + if ( + adaptiveMetadataUnavailable + || adaptiveCandidateMaximum === 0 + || maxAttemptsExceedsCandidateMaximum + ) { + setValidationError('Wait for the compatible technique limit to update.') + return + } if (!requestResult.ok) { setValidationError(requestResult.error) return @@ -708,8 +1270,14 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari navigate(`/scenario-history/${encodeURIComponent(summary.scenario_result_id)}`, { state: { scenarioName: scenario.scenario_name }, }) - } catch (err) { - setApiError(toApiError(err).detail) + } catch (err: unknown) { + const mappedError = mapEstimateError(err) + if (mappedError.maxAttemptsError) { + setLaunchMaxAttemptsError(mappedError.maxAttemptsError) + setApiError(null) + } else { + setApiError(mappedError.note ?? mappedError.summary) + } } finally { isSubmittingRef.current = false setSubmitting(false) @@ -722,17 +1290,47 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari } const techniqueSelectionInvalid = - techniqueSelection.mode === 'custom' && techniqueSelection.techniques.length === 0 - const previewDatasets = parseDatasetNames(datasetOverride) - const effectiveDatasets = previewDatasets.length > 0 ? previewDatasets : scenario.default_datasets + techniqueSelection.mode === 'custom' && customTechniques.length === 0 + const datasetSelectionInvalid = scenario.default_datasets.length > 0 && selectedDatasets.length === 0 + const datasetsAreDefaults = sameStringSet(selectedDatasets, scenario.default_datasets) + const datasetSizePreview = formatDatasetSizePreview( + scenario.dataset_size_limit, + maxDatasetSize, + hasMaxDatasetSizeOverride, + ) const presetMembers = techniqueSelection.mode === 'preset' - ? ( - scenario.aggregate_technique_expansions[techniqueSelection.preset] - ?? (techniqueSelection.preset === scenario.default_technique - ? scenario.default_techniques - : []) - ) + ? techniqueSetMembers(scenario, techniqueSelection.preset) : [] + const atAdaptiveCandidateMaximum = adaptiveCandidateMaximum !== null + && adaptiveCandidateMaximum > 0 + && maxAttemptsNumericValue === adaptiveCandidateMaximum + const scopedAdaptiveLimitNotice = adaptiveLimitNotice?.scopeKey === adaptiveCandidateScopeKey + ? adaptiveLimitNotice + : null + const currentAdaptiveLimitNotice = scopedAdaptiveLimitNotice?.message + ?? (atAdaptiveCandidateMaximum && adaptiveCandidateAvailability + ? `Maximum reached: ${adaptiveCandidateAvailability}` + : undefined) + const currentAdaptiveLimitValidationState = scopedAdaptiveLimitNotice?.validationState + ?? (atAdaptiveCandidateMaximum && adaptiveCandidateAvailability ? 'warning' : 'none') + const currentBaselineCount = currentResolvedRunEstimate?.components + .filter((component) => component.isBaseline) + .reduce((sum, component) => sum + component.count, 0) + const baselineHint = isBaselineForbidden + ? 'This scenario does not support sending objectives directly without an attack technique, so a direct comparison cannot be included.' + : baselineChecked && currentBaselineCount + ? `Adds ${currentBaselineCount.toLocaleString()} direct ${ + currentBaselineCount === 1 ? 'baseline attack' : 'baseline attacks' + } for the current objectives.` + : 'Also send each selected objective directly, without an attack technique. This provides a comparison point for measuring whether the selected techniques improve results and adds one planned attack per objective.' + const adaptiveCapFeedback = currentResolvedAdaptiveDetails + ? formatAdaptiveCapFeedback({ + selectedCandidateCount: currentResolvedAdaptiveDetails.selectedCandidateTechniqueCount, + compatibleCandidateCount: currentResolvedAdaptiveDetails.candidateTechniqueCount, + limit: currentResolvedAdaptiveDetails.maxAttemptsPerObjective, + effectiveMaximum: currentResolvedAdaptiveDetails.techniquesPerObjectiveUpperBound, + }) + : undefined return (
{scenario.scenario_name} + + {scenario.scenario_type} · v{scenario.scenario_version} + - Selecting a preset replaces any custom list. Selecting the first individual technique - switches to a custom list and clears the preset. + Choose a predefined set, or choose Custom to select techniques individually. -
- {presets.length > 0 ? ( - - handlePresetChange(data.value)} - aria-label="Aggregate preset" - > - {presets.map((name) => ( - - ))} - - - ) : ( + {usesAdaptiveTechniqueSelection && ( + <> - No aggregate presets are registered for this scenario. + Core, Extra, Light, Multi-turn, and Single-turn reflect tags on PyRIT's registered + techniques. All is generated from the catalog; Recommended is curated for this scenario. - )} + + + Adaptive uses these as a candidate pool. It tracks one progress step per compatible objective. + Adaptive tries no more than the configured maximum or the compatible candidate count, whichever + is smaller, and stops after the first success. Adding techniques changes the candidate pool, not + the number of progress steps; compatibility can still change how many objectives can run. + + + + )} +
+ + handleTechniqueModeChange(data.value)} + aria-label="Technique set" + > + {presets.map((name) => ( + + ))} + + + {techniqueSelection.mode === 'preset' && ( -
- Backend-resolved preset members +
+ Included techniques {presetMembers.length > 0 ? (
{presetMembers.map((name) => ( @@ -841,41 +1464,40 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari
) : ( - No concrete members were supplied for this preset. + No concrete members were supplied for this technique set. )}
)} - - {concrete.length > 0 ? ( -
- {concrete.map((name) => ( - handleConcreteChange(name, data.checked === true)} - data-testid={`technique-${name}`} - /> - ))} -
- ) : ( - - No concrete techniques are registered for custom selection. - - )} - + {techniqueSelection.mode === 'custom' && ( + + {concrete.length > 0 ? ( +
+ {concrete.map((name) => ( + handleConcreteChange(name, data.checked === true)} + data-testid={`technique-${name}`} + /> + ))} +
+ ) : ( + + No concrete techniques are registered for custom selection. + + )} + + )}
@@ -883,21 +1505,16 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari Baseline - + setBaselineChecked(data.checked === true)} data-testid="baseline-checkbox" /> - {isBaselineForbidden && ( - - This scenario forbids a baseline comparison run. - - )} {dynamicParameters.length > 0 && ( @@ -911,8 +1528,57 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari key={parameter.name} parameter={parameter} value={scenarioParamValues[parameter.name]} - disabled={submitting} + disabled={ + submitting + || ( + usesAdaptiveTechniqueSelection + && parameter.name === MAX_ATTEMPTS_PARAMETER_NAME + && (adaptiveMetadataUnavailable || adaptiveCandidateMaximum === 0) + ) + } onChange={updateScenarioParam} + displayLabel={usesAdaptiveTechniqueSelection + && parameter.name === MAX_ATTEMPTS_PARAMETER_NAME + ? MAX_ATTEMPTS_DISPLAY_LABEL + : undefined} + displayHint={usesAdaptiveTechniqueSelection + && parameter.name === MAX_ATTEMPTS_PARAMETER_NAME + ? `${adaptiveDefaultHint} ${MAX_ATTEMPTS_BEHAVIOR_HINT}${adaptiveCapFeedback + ? ` ${adaptiveCapFeedback}` + : ''}` + : undefined} + validationState={usesAdaptiveTechniqueSelection + && parameter.name === MAX_ATTEMPTS_PARAMETER_NAME + ? maxAttemptsFieldError + ? 'error' + : currentAdaptiveLimitNotice + ? currentAdaptiveLimitValidationState + : 'none' + : 'none'} + validationMessage={usesAdaptiveTechniqueSelection + && parameter.name === MAX_ATTEMPTS_PARAMETER_NAME + ? maxAttemptsFieldError ?? currentAdaptiveLimitNotice + : undefined} + numberMin={usesAdaptiveTechniqueSelection + && parameter.name === MAX_ATTEMPTS_PARAMETER_NAME + ? 1 + : undefined} + numberMax={usesAdaptiveTechniqueSelection + && parameter.name === MAX_ATTEMPTS_PARAMETER_NAME + && adaptiveCandidateMaximum !== null + && adaptiveCandidateMaximum > 0 + ? adaptiveCandidateMaximum + : undefined} + numberStep={usesAdaptiveTechniqueSelection + && parameter.name === MAX_ATTEMPTS_PARAMETER_NAME + ? 1 + : undefined} + numberWholeOnly={usesAdaptiveTechniqueSelection + && parameter.name === MAX_ATTEMPTS_PARAMETER_NAME} + onRejectedNumberInput={usesAdaptiveTechniqueSelection + && parameter.name === MAX_ATTEMPTS_PARAMETER_NAME + ? rejectScenarioParamInput + : undefined} testIdPrefix="scenario-param" /> ))} @@ -920,35 +1586,63 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari )} +
+ + Datasets + + + Choose the registered datasets that provide objectives for this run. + + { + setSelectedDatasets([...scenario.default_datasets]) + setValidationError(null) + }} + /> +
+ Advanced options
- - setDatasetOverride(data.value)} - placeholder={scenario.default_datasets.join(', ') || undefined} - data-testid="dataset-override-input" - /> - - - + {scenario.dataset_size_limit.override_scope !== 'unsupported' + && (hasMaxDatasetSizeOverride || maxDatasetSize.trim() === '') && ( + + )} - + Run preview - Review the exact configuration sent to the backend. + Review the exact configuration used for this run.
@@ -994,16 +1693,18 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari
{techniqueSelection.mode === 'preset' ? (
- Preset: {techniqueSelection.preset} + + Technique set: {techniqueSetDisplayName(scenario, techniqueSelection.preset)} + {presetMembers.length > 0 && ( Resolves to {presetMembers.join(', ')} )}
- ) : techniqueSelection.techniques.length > 0 ? ( + ) : customTechniques.length > 0 ? (
- {techniqueSelection.techniques.map((name) => ( + {customTechniques.map((name) => ( {name} ))}
@@ -1017,11 +1718,12 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari
- {effectiveDatasets.length > 0 ? effectiveDatasets.join(', ') : 'No datasets declared'} + {selectedDatasets.length > 0 ? selectedDatasets.join(', ') : 'No datasets selected'} - {previewDatasets.length > 0 ? 'Custom override' : 'Scenario defaults'} - {maxDatasetSize.trim() ? ` · capped at ${maxDatasetSize.trim()} each` : ''} + {datasetsAreDefaults ? 'Scenario defaults' : 'Selected datasets'} + {' · '} + {datasetSizePreview}
@@ -1033,7 +1735,7 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari
{dynamicParameters.map((parameter) => (
-
{parameter.name}
+
{parameterDisplayLabel(parameter, usesAdaptiveTechniqueSelection)}
{formatParameterPreview(scenarioParamValues[parameter.name])}
))} @@ -1047,15 +1749,15 @@ function ScenarioLaunchForm({ scenario, targets, activeTarget, labels }: Scenari
Baseline
{isBaselineForbidden - ? 'Excluded by scenario policy' + ? 'Not included — this scenario does not support direct comparison' : baselineChecked - ? 'Included' + ? 'Included — direct objective without an attack technique' : 'Not included'}
- Backend-owned size + Planned run size {submitting ? 'Launching...' : 'Launch scenario'} diff --git a/frontend/src/components/Scenarios/ScenarioFlow.test.tsx b/frontend/src/components/Scenarios/ScenarioFlow.test.tsx index a4fec85ae8..fc662e73c0 100644 --- a/frontend/src/components/Scenarios/ScenarioFlow.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioFlow.test.tsx @@ -4,7 +4,7 @@ import { FluentProvider, webLightTheme } from '@fluentui/react-components' import { MemoryRouter, Route, Routes, useLocation } from 'react-router' import { useScenarioRunProgress } from '@/hooks/useScenarioRunProgress' -import { scenariosApi, targetsApi } from '@/services/api' +import { datasetsApi, scenariosApi, targetsApi } from '@/services/api' import type { RegisteredScenario, ScenarioDefaultRunSizeEstimate, @@ -31,6 +31,9 @@ jest.mock('@/services/api', () => ({ targetsApi: { listTargets: jest.fn(), }, + datasetsApi: { + listDatasets: jest.fn(), + }, })) const mockUseScenarioRunProgress = useScenarioRunProgress as jest.Mock @@ -39,6 +42,7 @@ const mockGetScenario = scenariosApi.getScenario as jest.Mock const mockListCatalog = scenariosApi.listCatalog as jest.Mock const mockStartRun = scenariosApi.startRun as jest.Mock const mockListTargets = targetsApi.listTargets as jest.Mock +const mockListDatasets = datasetsApi.listDatasets as jest.Mock const SCENARIO_NAME = 'foundry.red_team_agent' const RUN_ID = '123e4567-e89b-12d3-a456-426614174000' @@ -57,6 +61,11 @@ const SCENARIO: RegisteredScenario = { }, all_techniques: ['crescendo'], default_datasets: ['harmbench'], + dataset_size_limit: { + default_scope: 'none', + default_count: null, + override_scope: 'per_dataset', + }, default_dataset_summaries: [], baseline_policy: 'enabled', include_baseline_by_default: true, @@ -65,8 +74,12 @@ const SCENARIO: RegisteredScenario = { version: 1, status: 'exact', total_attack_count: 2, + minimum_attack_count: null, + maximum_attack_count: null, + condition: null, components: [], datasets: [], + adaptive_details: null, note: null, retries_included: false, }, @@ -84,6 +97,9 @@ const ESTIMATE: ScenarioDefaultRunSizeEstimate = { version: 1, status: 'exact', total_attack_count: 2, + minimum_attack_count: null, + maximum_attack_count: null, + condition: null, components: [{ label: 'Configured attacks', count: 2, @@ -92,6 +108,7 @@ const ESTIMATE: ScenarioDefaultRunSizeEstimate = { note: null, }], datasets: [], + adaptive_details: null, note: null, retries_included: false, } @@ -162,6 +179,7 @@ describe('Scenario catalog-to-run integration', () => { items: [TARGET], pagination: { limit: 200, has_more: false }, }) + mockListDatasets.mockResolvedValue({ items: [{ name: 'harmbench' }] }) mockEstimateRun.mockResolvedValue(ESTIMATE) mockStartRun.mockResolvedValue({ scenario_result_id: RUN_ID }) mockUseScenarioRunProgress.mockReturnValue({ @@ -171,12 +189,13 @@ describe('Scenario catalog-to-run integration', () => { }) }) - it('carries one configured request from catalog detail through estimate, launch, and run hydration', async () => { + it('carries one configured request from catalog through estimate, launch, and run hydration', async () => { const user = userEvent.setup() renderFlow() - await user.click(await screen.findByRole('link', { name: SCENARIO_NAME })) + await user.click(await screen.findByRole('button', { name: 'Configure run' })) expect(await screen.findByRole('heading', { level: 1, name: SCENARIO_NAME })).toBeInTheDocument() + expect(screen.getByText('RedTeamAgentScenario · v1')).toBeInTheDocument() const expectedEstimateRequest = { target_name: TARGET.target_registry_name, @@ -189,7 +208,7 @@ describe('Scenario catalog-to-run integration', () => { expect.any(AbortSignal), )) expect(within(screen.getByRole('complementary', { name: 'Run preview' })) - .getByText('2 planned attacks')).toBeInTheDocument() + .getByRole('group', { name: '2 planned attacks.' })).toBeInTheDocument() await user.click(screen.getByTestId('launch-scenario-btn')) diff --git a/frontend/src/components/Scenarios/ScenarioRunEstimate.styles.ts b/frontend/src/components/Scenarios/ScenarioRunEstimate.styles.ts index 1edd185b6a..0fcc38296e 100644 --- a/frontend/src/components/Scenarios/ScenarioRunEstimate.styles.ts +++ b/frontend/src/components/Scenarios/ScenarioRunEstimate.styles.ts @@ -27,112 +27,74 @@ export const useScenarioRunEstimateStyles = makeStyles({ gap: tokens.spacingVerticalM, minWidth: 0, }, - detailGroup: { + calculationSection: { display: 'flex', flexDirection: 'column', - gap: tokens.spacingVerticalXS, - minWidth: 0, - }, - componentList: { - display: 'grid', gap: tokens.spacingVerticalS, - margin: 0, - padding: 0, - listStyleType: 'none', + minWidth: 0, }, - component: { + equation: { display: 'flex', - flexDirection: 'column', - gap: tokens.spacingVerticalXXS, - paddingLeft: tokens.spacingHorizontalS, - borderLeft: `${tokens.strokeWidthThick} solid ${tokens.colorNeutralStroke2}`, + alignItems: 'stretch', + flexWrap: 'wrap', + gap: `${tokens.spacingVerticalXS} ${tokens.spacingHorizontalXS}`, minWidth: 0, - overflowWrap: 'anywhere', }, - componentHeader: { - display: 'flex', + operand: { + display: 'inline-flex', alignItems: 'baseline', - justifyContent: 'space-between', - gap: tokens.spacingHorizontalS, - }, - componentCount: { - display: 'flex', - alignItems: 'center', - gap: tokens.spacingHorizontalXS, - flexShrink: 0, - fontVariantNumeric: 'tabular-nums', - }, - factorList: { - display: 'flex', flexWrap: 'wrap', - gap: `${tokens.spacingVerticalXXS} ${tokens.spacingHorizontalS}`, - margin: 0, - padding: 0, - listStyleType: 'none', - color: tokens.colorNeutralForeground2, - }, - datasetList: { - display: 'grid', - gap: tokens.spacingVerticalS, - }, - dataset: { - display: 'flex', - flexDirection: 'column', - gap: tokens.spacingVerticalXXS, + gap: tokens.spacingHorizontalXXS, + minWidth: 0, padding: `${tokens.spacingVerticalXS} ${tokens.spacingHorizontalS}`, + color: tokens.colorNeutralForeground1, backgroundColor: tokens.colorNeutralBackground3, - borderRadius: tokens.borderRadiusSmall, - minWidth: 0, + border: `${tokens.strokeWidthThin} solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusMedium, overflowWrap: 'anywhere', }, - datasetHeader: { - display: 'flex', - alignItems: 'center', + resultOperand: { + display: 'inline-flex', + alignItems: 'baseline', flexWrap: 'wrap', - gap: tokens.spacingHorizontalXS, - }, - countList: { - display: 'grid', - gap: tokens.spacingVerticalXXS, - margin: 0, + gap: tokens.spacingHorizontalXXS, + minWidth: 0, + padding: `${tokens.spacingVerticalXS} ${tokens.spacingHorizontalS}`, + color: tokens.colorBrandForeground2, + backgroundColor: tokens.colorBrandBackground2, + border: `${tokens.strokeWidthThin} solid ${tokens.colorBrandStroke1}`, + borderRadius: tokens.borderRadiusMedium, + overflowWrap: 'anywhere', }, - countRow: { - display: 'grid', - gridTemplateColumns: 'minmax(0, 1fr) auto', - gap: tokens.spacingHorizontalS, + operandValue: { fontVariantNumeric: 'tabular-nums', - '& dd': { - margin: 0, - fontWeight: tokens.fontWeightSemibold, - }, }, - capGroup: { - display: 'flex', - flexDirection: 'column', - gap: tokens.spacingVerticalXXS, + operandDetail: { + flexBasis: '100%', + color: tokens.colorNeutralForeground3, }, - capList: { - display: 'grid', - gap: tokens.spacingVerticalXXS, - margin: 0, - paddingLeft: tokens.spacingHorizontalL, + operator: { + display: 'inline-flex', + alignItems: 'center', + minHeight: '2rem', + color: tokens.colorNeutralForeground2, + fontSize: tokens.fontSizeBase400, }, - formula: { - display: 'block', - padding: `${tokens.spacingVerticalXS} ${tokens.spacingHorizontalS}`, - overflowWrap: 'anywhere', - fontFamily: tokens.fontFamilyMonospace, - fontSize: tokens.fontSizeBase200, - backgroundColor: tokens.colorNeutralBackground3, - borderRadius: tokens.borderRadiusSmall, + calculationContext: { + maxWidth: '72ch', + color: tokens.colorNeutralForeground2, + }, + sources: { + display: 'grid', + gap: tokens.spacingVerticalXS, + paddingTop: tokens.spacingVerticalXS, + borderTop: `${tokens.strokeWidthThin} solid ${tokens.colorNeutralStroke2}`, }, - staleNotice: { + source: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalXXS, - padding: `${tokens.spacingVerticalXS} ${tokens.spacingHorizontalS}`, - color: tokens.colorPaletteDarkOrangeForeground1, - backgroundColor: tokens.colorPaletteDarkOrangeBackground1, - borderRadius: tokens.borderRadiusSmall, + minWidth: 0, + overflowWrap: 'anywhere', }, }) diff --git a/frontend/src/components/Scenarios/ScenarioRunEstimate.test.tsx b/frontend/src/components/Scenarios/ScenarioRunEstimate.test.tsx index aa69a1f15f..4ecf198111 100644 --- a/frontend/src/components/Scenarios/ScenarioRunEstimate.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioRunEstimate.test.tsx @@ -1,6 +1,6 @@ import type { ReactNode } from 'react' -import { render, screen } from '@testing-library/react' +import { render, screen, within } from '@testing-library/react' import { FluentProvider, webLightTheme } from '@fluentui/react-components' import type { ScenarioDefaultRunSizeEstimate, ScenarioRunEstimateState } from '@/types' @@ -15,146 +15,714 @@ function TestWrapper({ children }: { children: ReactNode }) { return {children} } -const EXACT_ESTIMATE: ScenarioDefaultRunSizeEstimate = { - version: 1, - status: 'exact', - total_attack_count: 8, - components: [ - { - label: 'Prompt sending', - count: 8, - factors: [ - { label: 'selected seed groups', count: 4 }, - { label: 'jailbreak templates', count: 2 }, - { label: 'techniques', count: 1 }, - { label: 'attempts', count: 1 }, - ], - is_baseline: false, - note: 'One planned attack per selected objective and template.', - }, - { - label: 'Baseline attack', - // Deliberately differs from the authoritative total when added to the - // first component so this test detects accidental client-side summing. - count: 2, - factors: [], - is_baseline: true, - note: 'Fixture component used to guard the authoritative total.', +function makeEstimate( + overrides: Partial = {}, +): ScenarioDefaultRunSizeEstimate { + return { + version: 1, + status: 'exact', + total_attack_count: 16, + minimum_attack_count: null, + maximum_attack_count: null, + condition: null, + components: [ + { + label: 'Default technique sweep', + count: 16, + factors: [ + { label: 'selected logical seed groups', count: 4 }, + { label: 'default concrete techniques', count: 4 }, + ], + is_baseline: false, + note: null, + }, + ], + datasets: [ + { + name: 'harmbench', + kind: 'dataset', + logical_seed_group_count: 400, + selected_seed_group_count: 4, + configured_caps: [], + selection_note: 'The default selection uses 4 of 400 available objectives.', + }, + ], + adaptive_details: null, + note: null, + retries_included: false, + ...overrides, + } +} + +function renderDetails(estimate: ScenarioDefaultRunSizeEstimate): void { + render( + + + , + ) +} + +describe('ScenarioRunEstimate', () => { + it('renders an exact technique-by-objective equation with a complete accessible sentence', () => { + renderDetails(makeEstimate()) + + const equation = screen.getByRole('group', { + name: '4 techniques multiplied by 4 objectives equals 16 planned attacks.', + }) + expect(within(equation).getAllByText('4')).toHaveLength(2) + expect(within(equation).getByText('techniques')).toBeInTheDocument() + expect(within(equation).getByText('objectives')).toBeInTheDocument() + expect(within(equation).getByText('16')).toBeInTheDocument() + expect(within(equation).getByText('planned attacks')).toBeInTheDocument() + expect(screen.getByText('4 objectives from harmbench · 400 available')).toBeInTheDocument() + expect(screen.getByRole('heading', { name: 'Run calculation' })).toBeInTheDocument() + }) + + it.each([1, 4, 5])( + 'renders a universal per-dataset cap of %i once before the objective-source rows', + (capCount) => { + renderDetails(makeEstimate({ + datasets: [ + { + name: 'airt_hate', + kind: 'dataset', + logical_seed_group_count: 4, + selected_seed_group_count: 4, + configured_caps: [{ + label: 'per-dataset cap', + count: capCount, + configured_on: 'dataset', + dataset_name: 'airt_hate', + }], + selection_note: null, + }, + { + name: 'airt_leakage', + kind: 'dataset', + logical_seed_group_count: 9, + selected_seed_group_count: 5, + configured_caps: [{ + label: 'per-dataset cap', + count: capCount, + configured_on: 'dataset', + dataset_name: 'airt_leakage', + }], + selection_note: null, + }, + ], + })) + + const capText = `Per-dataset cap: ${capCount} ${capCount === 1 ? 'objective' : 'objectives'}` + expect(screen.getAllByText(capText)).toHaveLength(1) + expect(screen.getByText('4 objectives from airt_hate')).toBeInTheDocument() + expect(screen.getByText('5 objectives from airt_leakage · 9 available')).toBeInTheDocument() + const sources = screen.getByRole('group', { name: 'Objective sources' }) + const sourceText = sources.textContent ?? '' + expect(sourceText.indexOf(capText)).toBeLessThan(sourceText.indexOf('4 objectives from airt_hate')) }, - ], - datasets: [ - { - name: 'harmbench', - kind: 'dataset', - logical_seed_group_count: 4, - selected_seed_group_count: 4, - configured_caps: [ - { - label: 'Jailbreak templates', + ) + + it('keeps differing dataset caps with their affected objective-source rows', () => { + renderDetails(makeEstimate({ + datasets: [ + { + name: 'dataset_alpha', + kind: 'dataset', + logical_seed_group_count: 8, + selected_seed_group_count: 4, + configured_caps: [{ + label: 'per-dataset cap', + count: 4, + configured_on: 'dataset', + dataset_name: 'dataset_alpha', + }], + selection_note: null, + }, + { + name: 'dataset_beta', + kind: 'dataset', + logical_seed_group_count: 10, + selected_seed_group_count: 5, + configured_caps: [{ + label: 'per-dataset cap', + count: 5, + configured_on: 'dataset', + dataset_name: 'dataset_beta', + }], + selection_note: null, + }, + ], + })) + + const alpha = screen.getByRole('group', { name: 'Objective source: dataset_alpha' }) + const beta = screen.getByRole('group', { name: 'Objective source: dataset_beta' }) + expect(within(alpha).getByText('Per-dataset cap: 4 objectives')).toBeInTheDocument() + expect(within(beta).getByText('Per-dataset cap: 5 objectives')).toBeInTheDocument() + expect(screen.getAllByText(/Per-dataset cap:/)).toHaveLength(2) + }) + + it('keeps a single-dataset cap attached to its objective-source row', () => { + renderDetails(makeEstimate({ + datasets: [{ + name: 'harmbench', + kind: 'dataset', + logical_seed_group_count: 8, + selected_seed_group_count: 4, + configured_caps: [{ + label: 'per-dataset cap', + count: 4, + configured_on: 'dataset', + dataset_name: 'harmbench', + }], + selection_note: null, + }], + })) + + const source = screen.getByRole('group', { name: 'Objective source: harmbench' }) + expect(within(source).getByText('Per-dataset cap: 4 objectives')).toBeInTheDocument() + }) + + it('renders a global cap once while preserving differing per-dataset caps on rows', () => { + renderDetails(makeEstimate({ + datasets: [ + { + name: 'dataset_alpha', + kind: 'dataset', + logical_seed_group_count: 8, + selected_seed_group_count: 3, + configured_caps: [ + { + label: 'per-dataset cap', + count: 3, + configured_on: 'dataset', + dataset_name: 'dataset_alpha', + }, + { + label: 'combined compound cap', + count: 10, + configured_on: 'compound', + dataset_name: null, + }, + ], + selection_note: null, + }, + { + name: 'dataset_beta', + kind: 'dataset', + logical_seed_group_count: 8, + selected_seed_group_count: 4, + configured_caps: [ + { + label: 'per-dataset cap', + count: 4, + configured_on: 'dataset', + dataset_name: 'dataset_beta', + }, + { + label: 'combined compound cap', + count: 10, + configured_on: 'compound', + dataset_name: null, + }, + ], + selection_note: null, + }, + ], + })) + + expect(screen.getAllByText('Combined compound cap: 10')).toHaveLength(1) + expect(within(screen.getByRole('group', { + name: 'Objective source: dataset_alpha', + })).getByText('Per-dataset cap: 3 objectives')).toBeInTheDocument() + expect(within(screen.getByRole('group', { + name: 'Objective source: dataset_beta', + })).getByText('Per-dataset cap: 4 objectives')).toBeInTheDocument() + }) + + it('keeps a configuration cap on only the rows where it applies', () => { + renderDetails(makeEstimate({ + datasets: [ + { + name: 'dataset_alpha', + kind: 'dataset', + logical_seed_group_count: 8, + selected_seed_group_count: 4, + configured_caps: [{ + label: 'shared configuration cap', + count: 6, + configured_on: 'configuration', + dataset_name: null, + }], + selection_note: null, + }, + { + name: 'dataset_beta', + kind: 'dataset', + logical_seed_group_count: 8, + selected_seed_group_count: 4, + configured_caps: [{ + label: 'shared configuration cap', + count: 6, + configured_on: 'configuration', + dataset_name: null, + }], + selection_note: null, + }, + { + name: 'dataset_gamma', + kind: 'dataset', + logical_seed_group_count: 8, + selected_seed_group_count: 8, + configured_caps: [], + selection_note: null, + }, + ], + })) + + expect(within(screen.getByRole('group', { + name: 'Objective source: dataset_alpha', + })).getByText('Shared configuration cap: 6')).toBeInTheDocument() + expect(within(screen.getByRole('group', { + name: 'Objective source: dataset_beta', + })).getByText('Shared configuration cap: 6')).toBeInTheDocument() + expect(within(screen.getByRole('group', { + name: 'Objective source: dataset_gamma', + })).queryByText(/Shared configuration cap/)).not.toBeInTheDocument() + expect(screen.getAllByText('Shared configuration cap: 6')).toHaveLength(2) + }) + + it('renders no cap summary for uncapped datasets and preserves the selection note', () => { + renderDetails(makeEstimate({ + datasets: [{ + name: 'harmbench', + kind: 'dataset', + logical_seed_group_count: 8, + selected_seed_group_count: 4, + configured_caps: [], + selection_note: 'Four compatible objectives remain after filtering.', + }], + })) + + expect(screen.queryByText(/cap:/i)).not.toBeInTheDocument() + expect(screen.getByText('4 objectives from harmbench · 8 available')).toBeInTheDocument() + expect(screen.getByText('Four compatible objectives remain after filtering.')).toBeInTheDocument() + }) + + it('renders heterogeneous compatibility as truthful per-technique additive terms', () => { + renderDetails(makeEstimate({ + total_attack_count: 6, + components: [ + { + label: 'technique_alpha', + count: 4, + factors: [ + { label: 'selected concrete techniques', count: 1 }, + { label: 'compatible logical seed groups', count: 4 }, + ], + is_baseline: false, + note: null, + }, + { + label: 'technique_beta', count: 2, - configured_on: 'configuration', - dataset_name: null, + factors: [ + { label: 'selected concrete techniques', count: 1 }, + { label: 'compatible logical seed groups', count: 2 }, + ], + is_baseline: false, + note: null, }, ], - selection_note: 'Four compatible objective groups selected.', - }, - ], - note: 'The backend total is authoritative.', - retries_included: false, -} + })) -describe('ScenarioRunEstimate', () => { - it('renders the authoritative total, ordered factors, dataset counts, caps, and notes', () => { - const state = mapScenarioRunEstimate(EXACT_ESTIMATE, 'request') + const equation = screen.getByTestId('run-calculation') + expect(within(equation).getByText('objectives · Technique alpha')).toBeInTheDocument() + expect(within(equation).getByText('objectives · Technique beta')).toBeInTheDocument() + expect(equation).toHaveTextContent('4objectives · Technique alpha+2objectives · Technique beta=6planned attacks') + }) + it('uses parentheses to make baseline precedence explicit', () => { + renderDetails(makeEstimate({ + total_attack_count: 20, + components: [ + ...makeEstimate().components, + { + label: 'Baseline', + count: 4, + factors: [{ label: 'selected logical seed groups', count: 4 }], + is_baseline: true, + note: null, + }, + ], + })) + + const equation = screen.getByTestId('run-calculation') + expect(equation).toHaveTextContent('(4techniques×4objectives)+4direct baseline attacks=20planned attacks') + expect(screen.getByRole('group', { + name: '( 4 techniques multiplied by 4 objectives ) plus 4 direct baseline attacks equals 20 planned attacks.', + })).toBeInTheDocument() + }) + + it('keeps guaranteed and target-conditional terms in one bounded equation', () => { + renderDetails(makeEstimate({ + status: 'conditional', + total_attack_count: null, + minimum_attack_count: 12, + maximum_attack_count: 20, + condition: 'target_capabilities', + components: [ + { + label: 'Baseline', + count: 4, + factors: [{ label: 'objectives', count: 4 }], + is_baseline: true, + note: null, + }, + { + label: 'Inline jailbreak delivery', + count: 8, + factors: [ + { label: 'objectives', count: 4 }, + { label: 'jailbreak templates', count: 2 }, + ], + is_baseline: false, + note: null, + }, + { + label: 'Native system-prompt jailbreak delivery', + count: 8, + factors: [ + { label: 'objectives', count: 4 }, + { label: 'jailbreak templates', count: 2 }, + ], + is_baseline: false, + condition: 'target_capabilities', + note: null, + }, + ], + })) + + const equation = screen.getByTestId('run-calculation') + expect(equation).toHaveTextContent('4objectives · Inline jailbreak delivery') + expect(equation).toHaveTextContent('4objectives · Native system-prompt jailbreak delivery · if supported') + expect(equation).toHaveTextContent('4direct baseline attacks') + expect(equation).toHaveTextContent('12–20planned attacks') + }) + + it('shows adaptive progress objectives and the bounded underlying attempt work', () => { + const estimate = makeEstimate({ + status: 'conditional', + total_attack_count: null, + components: [], + adaptive_details: { + objective_count: 21, + selected_candidate_technique_count: 2, + candidate_technique_count: 2, + max_attempts_per_objective: 3, + techniques_per_objective_upper_bound: 2, + technique_attempt_count_upper_bound: 42, + stop_on_first_success: true, + compatibility_may_reduce_attempts: true, + }, + }) + const state = mapScenarioRunEstimate(estimate, 'request') render( + , ) - expect(screen.getByText('8 planned attacks')).toBeInTheDocument() - expect(screen.queryByText('10 planned attacks')).not.toBeInTheDocument() - expect(screen.getByText('Prompt sending')).toBeInTheDocument() - expect(screen.getByText('Baseline attack')).toBeInTheDocument() - expect(screen.getByText('Baseline')).toBeInTheDocument() - expect(screen.getByText('× 4 selected seed groups')).toBeInTheDocument() - expect(screen.getByText('× 2 jailbreak templates')).toBeInTheDocument() - expect(screen.getByText('harmbench')).toBeInTheDocument() - expect(screen.getByText('Jailbreak templates: 2 (configuration)')).toBeInTheDocument() - expect(screen.getByText('Four compatible objective groups selected.')).toBeInTheDocument() + expect(screen.getByText('21 objectives · up to 42 technique attempts')).toBeInTheDocument() + expect(screen.getByRole('group', { + name: '21 objectives multiplied by up to 2 techniques per objective, the smaller of 2 selected candidates and limit 3, equals up to 42 technique attempts.', + })).toBeInTheDocument() + expect(screen.getByText('2 selected candidates · limit 3')).toBeInTheDocument() + expect(screen.getByRole('group', { + name: 'Direct baseline comparison is not included: up to 21 Adaptive attacks. Planned total is confirmed at launch.', + })).toBeInTheDocument() + expect(screen.queryByText('Exact total')).not.toBeInTheDocument() expect(screen.getByText( - 'Prompt sending: 4 selected seed groups × 2 jailbreak templates × 1 techniques × 1 attempts = 8 + Baseline attack: 2; backend total = 8', + 'Technique-attempt totals exclude multi-turn target exchanges and retries. Adaptive stops each objective after the first successful technique. Compatibility may reduce how many candidates each objective can try.', )).toBeInTheDocument() - expect(screen.getByText('The backend total is authoritative.')).toBeInTheDocument() - expect(screen.getByText('Retries are not included. Estimate schema v1.')).toBeInTheDocument() }) - it('supports loading, conditional null totals, unavailable, and stale states', () => { - const loading: ScenarioRunEstimateState = { status: 'loading', scope: 'request' } - const { rerender } = render( + it('shows baseline-aware planned attacks before unchanged Adaptive work', () => { + const estimate = makeEstimate({ + status: 'conditional', + total_attack_count: null, + minimum_attack_count: 21, + maximum_attack_count: 42, + components: [ + { + label: 'Baseline', + count: 21, + factors: [{ label: 'objectives', count: 21 }], + is_baseline: true, + note: null, + }, + { + label: 'Adaptive objectives', + count: 21, + factors: [{ label: 'compatible objectives', count: 21 }], + is_baseline: false, + note: null, + }, + ], + adaptive_details: { + objective_count: 21, + selected_candidate_technique_count: 2, + candidate_technique_count: 2, + max_attempts_per_objective: 3, + techniques_per_objective_upper_bound: 2, + technique_attempt_count_upper_bound: 42, + stop_on_first_success: true, + compatibility_may_reduce_attempts: true, + }, + }) + const state = mapScenarioRunEstimate(estimate, 'request') + render( - + + , ) - expect(screen.getByText('Loading backend run estimate...')).toBeInTheDocument() - const conditional = mapScenarioRunEstimate({ - ...EXACT_ESTIMATE, + expect(screen.getByText('21–42 planned attacks · up to 42 technique attempts')).toBeInTheDocument() + const plannedEquation = screen.getByRole('group', { + name: 'Direct baseline comparison is included: 21 direct baseline attacks plus up to 21 Adaptive attacks equals 21–42 planned attacks.', + }) + expect(plannedEquation).toHaveTextContent( + '21direct baseline attacks+up to 21Adaptive attacks=21–42planned attacks', + ) + expect(screen.getByRole('heading', { name: 'Planned attacks' })).toBeInTheDocument() + expect(screen.getByRole('heading', { name: 'Adaptive work' })).toBeInTheDocument() + const adaptiveWork = screen.getByTestId('adaptive-work-calculation') + expect(adaptiveWork).toHaveTextContent('21objectives×up to 2techniques per objective') + expect(adaptiveWork).toHaveTextContent('=up to 42technique attempts') + expect(screen.queryByText(/Attempt ceiling:/)).not.toBeInTheDocument() + expect(screen.queryByText(/Progress tracks/)).not.toBeInTheDocument() + expect(screen.queryByText(/objective envelope|logical seed groups|selected seed groups/i)).not.toBeInTheDocument() + }) + + it('removes the baseline term while keeping Adaptive work unchanged', () => { + renderDetails(makeEstimate({ + status: 'conditional', + total_attack_count: null, + minimum_attack_count: null, + maximum_attack_count: 21, + components: [ + { + label: 'Adaptive objectives', + count: 21, + factors: [{ label: 'compatible objectives', count: 21 }], + is_baseline: false, + note: null, + }, + ], + adaptive_details: { + objective_count: 21, + selected_candidate_technique_count: 14, + candidate_technique_count: 14, + max_attempts_per_objective: 14, + techniques_per_objective_upper_bound: 14, + technique_attempt_count_upper_bound: 294, + stop_on_first_success: true, + compatibility_may_reduce_attempts: true, + }, + })) + + const plannedEquation = screen.getByRole('group', { + name: 'Direct baseline comparison is not included: up to 21 Adaptive attacks equals up to 21 planned attacks.', + }) + expect(plannedEquation).toHaveTextContent('up to 21Adaptive attacks=up to 21planned attacks') + expect(within(plannedEquation).queryByText(/baseline attack/)).not.toBeInTheDocument() + const adaptiveWork = screen.getByTestId('adaptive-work-calculation') + expect(adaptiveWork).toHaveTextContent('21objectives×up to 14techniques per objective') + expect(adaptiveWork).toHaveTextContent('=up to 294technique attempts') + }) + + it('renders exact Adaptive planned values without inventing a range', () => { + renderDetails(makeEstimate({ + status: 'exact', + total_attack_count: 42, + minimum_attack_count: null, + maximum_attack_count: null, + components: [ + { + label: 'Baseline', + count: 21, + factors: [{ label: 'objectives', count: 21 }], + is_baseline: true, + note: null, + }, + { + label: 'Adaptive objectives', + count: 21, + factors: [{ label: 'objectives', count: 21 }], + is_baseline: false, + note: null, + }, + ], + adaptive_details: { + objective_count: 21, + selected_candidate_technique_count: 14, + candidate_technique_count: 14, + max_attempts_per_objective: 14, + techniques_per_objective_upper_bound: 14, + technique_attempt_count_upper_bound: 294, + stop_on_first_success: true, + compatibility_may_reduce_attempts: false, + }, + })) + + expect(screen.getByRole('group', { + name: 'Direct baseline comparison is included: 21 direct baseline attacks plus 21 Adaptive attacks equals 42 planned attacks.', + })).toBeInTheDocument() + expect(screen.queryByText('21–42')).not.toBeInTheDocument() + }) + + it('preserves a nonzero Adaptive planned range when no baseline is included', () => { + renderDetails(makeEstimate({ status: 'conditional', total_attack_count: null, + minimum_attack_count: 5, + maximum_attack_count: 21, components: [], - datasets: [], - note: null, - }, 'default') - rerender( - - - , + adaptive_details: { + objective_count: 21, + selected_candidate_technique_count: 2, + candidate_technique_count: 2, + max_attempts_per_objective: 2, + techniques_per_objective_upper_bound: 2, + technique_attempt_count_upper_bound: 42, + stop_on_first_success: true, + compatibility_may_reduce_attempts: true, + }, + })) + + expect(screen.getByRole('group', { + name: 'Direct baseline comparison is not included: 5–21 Adaptive attacks equals 5–21 planned attacks.', + })).toBeInTheDocument() + }) + + it('uses the configured max when it is lower than the adaptive candidate pool', () => { + renderDetails(makeEstimate({ + status: 'conditional', + total_attack_count: null, + components: [], + adaptive_details: { + objective_count: 21, + selected_candidate_technique_count: 14, + candidate_technique_count: 5, + max_attempts_per_objective: 3, + techniques_per_objective_upper_bound: 3, + technique_attempt_count_upper_bound: 63, + stop_on_first_success: true, + compatibility_may_reduce_attempts: true, + }, + })) + + expect(screen.getByText('5 compatible candidates from 14 selected · limit 3')).toBeInTheDocument() + expect(screen.getByText('up to 63')).toBeInTheDocument() + }) + + it('uses the candidate pool when it is lower than the adaptive max', () => { + renderDetails(makeEstimate({ + status: 'conditional', + total_attack_count: null, + components: [], + adaptive_details: { + objective_count: 21, + selected_candidate_technique_count: 2, + candidate_technique_count: 2, + max_attempts_per_objective: 5, + techniques_per_objective_upper_bound: 2, + technique_attempt_count_upper_bound: 42, + stop_on_first_success: true, + compatibility_may_reduce_attempts: true, + }, + })) + + expect(screen.getByText('techniques per objective')).toBeInTheDocument() + expect(screen.getByText('2 selected candidates · limit 5')).toBeInTheDocument() + expect(screen.getByText('up to 42')).toBeInTheDocument() + }) + + it('adapts legacy version-one payloads without the selected candidate count', () => { + const estimate = mapScenarioRunEstimate(makeEstimate({ + status: 'conditional', + total_attack_count: null, + components: [], + adaptive_details: { + objective_count: 21, + candidate_technique_count: 2, + max_attempts_per_objective: 3, + techniques_per_objective_upper_bound: 2, + technique_attempt_count_upper_bound: 42, + stop_on_first_success: true, + compatibility_may_reduce_attempts: true, + }, + }), 'request') + + expect(estimate).toMatchObject({ + status: 'conditional', + estimate: { + adaptiveDetails: { + selectedCandidateTechniqueCount: 2, + }, + }, + }) + }) + + it('preserves loading, unavailable, and unknown conditional states', () => { + const loading: ScenarioRunEstimateState = { status: 'loading', scope: 'request' } + const { rerender } = render( + , ) - expect(screen.getByText('Conditional estimate')).toBeInTheDocument() - expect(screen.getByText('Total depends on configuration')).toBeInTheDocument() - expect(screen.getByText('Default configuration')).toBeInTheDocument() - expect(screen.getByText( - 'No additive components supplied; backend total is conditional', - )).toBeInTheDocument() + expect(screen.getByText('Calculating planned attacks...')).toBeInTheDocument() - const unavailable = mapScenarioRunEstimate({ - ...EXACT_ESTIMATE, + const unavailable = mapScenarioRunEstimate(makeEstimate({ status: 'unavailable', total_attack_count: null, components: [], datasets: [], note: 'Target capability is not available.', - }, 'request') + }), 'request') + rerender() + expect(screen.getByText('Estimate unavailable')).toBeInTheDocument() + expect(screen.getByText('Configured run size unavailable')).toBeInTheDocument() + rerender( - - + , ) - expect(screen.getAllByText('Estimate unavailable')).toHaveLength(2) - expect(screen.getByText('Configured run size unavailable')).toBeInTheDocument() - expect(screen.getByText('Target capability is not available.')).toBeInTheDocument() - - const exact = mapScenarioRunEstimate(EXACT_ESTIMATE, 'request') - if (exact.status !== 'available') { - throw new Error('Expected exact estimate to map to an available state.') - } - const stale: ScenarioRunEstimateState = { - status: 'stale', - estimate: exact.estimate, - label: 'Showing the last successful estimate.', - error: 'Preview service timed out.', - } - rerender( + expect(screen.getByText('Exact total')).toBeInTheDocument() + expect(screen.getByText('unavailable')).toBeInTheDocument() + }) + + it('does not render implementation terminology in the shared estimate surfaces', () => { + const state = mapScenarioRunEstimate(makeEstimate(), 'request') + render( - + + , ) - expect(screen.getByText('Previous estimate')).toBeInTheDocument() - expect(screen.getByText('8 planned attacks')).toBeInTheDocument() - expect(screen.getByText('Showing the last successful estimate.')).toBeInTheDocument() - expect(screen.getByText('Preview service timed out.')).toBeInTheDocument() + + expect(screen.queryByText(/logical seed groups/i)).not.toBeInTheDocument() + expect(screen.queryByText(/selected seed groups/i)).not.toBeInTheDocument() + expect(screen.queryByText(/planned components/i)).not.toBeInTheDocument() + expect(screen.queryByText(/objective envelopes/i)).not.toBeInTheDocument() + expect(screen.queryByText(/how this count is calculated/i)).not.toBeInTheDocument() }) }) diff --git a/frontend/src/components/Scenarios/ScenarioRunEstimate.tsx b/frontend/src/components/Scenarios/ScenarioRunEstimate.tsx index 50fb74931a..cd56c635c4 100644 --- a/frontend/src/components/Scenarios/ScenarioRunEstimate.tsx +++ b/frontend/src/components/Scenarios/ScenarioRunEstimate.tsx @@ -3,9 +3,16 @@ import { Badge, Spinner, Text } from '@fluentui/react-components' import type { ScenarioRunEstimate, ScenarioRunEstimateComponent, + ScenarioRunEstimateDatasetCap, + ScenarioRunEstimateFactor, ScenarioRunEstimateState, } from '@/types' +import { + formatAdaptiveCapAccessibleRule, + formatAdaptiveCapMetadata, +} from './scenarioAdaptiveCap' +import { normalizeDatasetCaps } from './scenarioDatasetCaps' import { useScenarioRunEstimateStyles } from './ScenarioRunEstimate.styles' interface ScenarioRunEstimateSummaryProps { @@ -17,12 +24,34 @@ interface ScenarioRunEstimateDetailsProps { idPrefix?: string } +interface CalculationOperand { + id: string + value: string + label: string + detail?: string + result?: boolean +} + +interface CalculationOperator { + id: string + symbol: '(' | ')' | '×' | '+' | '=' +} + +type CalculationPart = + | { kind: 'operand'; operand: CalculationOperand } + | { kind: 'operator'; operator: CalculationOperator } + +interface RunCalculation { + parts: CalculationPart[] + accessibleLabel: string + summary?: string + context?: string +} + function stateEstimate(state: ScenarioRunEstimateState): ScenarioRunEstimate | undefined { switch (state.status) { case 'available': case 'conditional': - case 'refreshing': - case 'stale': return state.estimate default: return undefined @@ -36,18 +65,13 @@ function scopeLabel(state: ScenarioRunEstimateState): string { return scope === 'default' ? 'Default configuration' : 'Current configuration' } -function statusLabel(state: ScenarioRunEstimateState): string { +function statusLabel(state: ScenarioRunEstimateState): string | null { switch (state.status) { case 'loading': return 'Loading estimate' case 'available': - return 'Backend estimate' case 'conditional': - return 'Conditional estimate' - case 'refreshing': - return 'Updating estimate' - case 'stale': - return 'Previous estimate' + return null case 'unavailable': return 'Estimate unavailable' } @@ -56,16 +80,18 @@ function statusLabel(state: ScenarioRunEstimateState): string { function statusColor(state: ScenarioRunEstimateState): 'brand' | 'warning' | 'subtle' { switch (state.status) { case 'available': - case 'refreshing': return 'brand' case 'conditional': - case 'stale': return 'warning' default: return 'subtle' } } +function formatCount(value: number): string { + return value.toLocaleString() +} + function formatEstimateValue(value: number): string { return value.toLocaleString() } @@ -89,70 +115,355 @@ function formatPlannedAttackSummary(estimate: ScenarioRunEstimate): string { if (estimate.minimum != null) { return `At least ${countLabel(estimate.minimum, 'planned attack', 'planned attacks')}` } - return 'Total depends on configuration' + return estimate.scope === 'default' + ? 'Select targets to calculate' + : 'Run size is confirmed at launch.' +} + +function baselineCount(estimate: ScenarioRunEstimate): number { + return estimate.components + .filter((component) => component.isBaseline) + .reduce((sum, component) => sum + component.count, 0) +} + +function formatEstimateSummary(estimate: ScenarioRunEstimate): string { + if (estimate.adaptiveDetails) { + const { objectiveCount, techniqueAttemptCountUpperBound } = estimate.adaptiveDetails + const attemptSummary = `up to ${countLabel( + techniqueAttemptCountUpperBound, + 'technique attempt', + 'technique attempts', + )}` + const hasPlannedAttackBound = estimate.total !== null + || estimate.minimum != null + || estimate.maximum != null + return hasPlannedAttackBound + ? `${formatPlannedAttackSummary(estimate)} · ${attemptSummary}` + : `${countLabel(objectiveCount, 'objective', 'objectives')} · ${attemptSummary}` + } + return formatPlannedAttackSummary(estimate) +} + +function operand(id: string, value: string, label: string, result = false, detail?: string): CalculationPart { + return { kind: 'operand', operand: { id, value, label, detail, result } } +} + +function operator(id: string, symbol: CalculationOperator['symbol']): CalculationPart { + return { kind: 'operator', operator: { id, symbol } } } -function formatProgressUnitSummary(estimate: ScenarioRunEstimate): string { +function humanizeLabel(label: string): string { + const words = label.replace(/_/g, ' ').trim() + return words.length > 0 ? `${words[0].toUpperCase()}${words.slice(1)}` : label +} + +function formatDatasetCap(cap: ScenarioRunEstimateDatasetCap): string { + const count = cap.configuredOn === 'dataset' + ? countLabel(cap.count, 'objective', 'objectives') + : formatCount(cap.count) + return `${humanizeLabel(cap.label)}: ${count}` +} + +function semanticFactorLabel(factor: ScenarioRunEstimateFactor): string { + const label = factor.label.toLowerCase() + if (label.includes('seed group') || label === 'objectives') { + return factor.count === 1 ? 'objective' : 'objectives' + } + if (label.includes('technique')) { + return factor.count === 1 ? 'technique' : 'techniques' + } + if (factor.count === 1 && label.endsWith('s')) { + return label.slice(0, -1) + } + return label +} + +function factorPriority(factor: ScenarioRunEstimateFactor): number { + const label = semanticFactorLabel(factor) + if (label === 'technique' || label === 'techniques') return 0 + if (label === 'objective' || label === 'objectives') return 1 + return 2 +} + +function objectiveFactor(component: ScenarioRunEstimateComponent): ScenarioRunEstimateFactor | undefined { + return component.factors.find((factor) => { + const label = semanticFactorLabel(factor) + return label === 'objective' || label === 'objectives' + }) +} + +function resultOperand(estimate: ScenarioRunEstimate): CalculationOperand { if (estimate.total !== null) { - return countLabel(estimate.total, 'progress unit', 'progress units') + return { + id: 'result', + value: formatCount(estimate.total), + label: estimate.total === 1 ? 'planned attack' : 'planned attacks', + result: true, + } } if (estimate.minimum != null && estimate.maximum != null) { - return estimate.minimum === estimate.maximum - ? countLabel(estimate.minimum, 'progress unit', 'progress units') - : `${formatEstimateValue(estimate.minimum)}–${formatEstimateValue(estimate.maximum)} progress units` + return { + id: 'result', + value: estimate.minimum === estimate.maximum + ? formatCount(estimate.minimum) + : `${formatCount(estimate.minimum)}–${formatCount(estimate.maximum)}`, + label: estimate.minimum === 1 && estimate.maximum === 1 ? 'planned attack' : 'planned attacks', + result: true, + } } if (estimate.maximum != null) { - return `Up to ${countLabel(estimate.maximum, 'progress unit', 'progress units')}` + return { + id: 'result', + value: `up to ${formatCount(estimate.maximum)}`, + label: estimate.maximum === 1 ? 'planned attack' : 'planned attacks', + result: true, + } } if (estimate.minimum != null) { - return `At least ${countLabel(estimate.minimum, 'progress unit', 'progress units')}` + return { + id: 'result', + value: `at least ${formatCount(estimate.minimum)}`, + label: estimate.minimum === 1 ? 'planned attack' : 'planned attacks', + result: true, + } } - return 'Progress units are confirmed at launch.' + return { id: 'result', value: 'Exact total', label: 'unavailable', result: true } } -function baselineCount(estimate: ScenarioRunEstimate): number { - return estimate.components - .filter((component) => component.isBaseline) - .reduce((sum, component) => sum + component.count, 0) -} - -function formatEstimateSummary(estimate: ScenarioRunEstimate): string { - if (!estimate.adaptiveDetails) { - return formatPlannedAttackSummary(estimate) - } - const attackAttemptUpperBound = estimate.adaptiveDetails.techniqueAttemptCountUpperBound - + baselineCount(estimate) - const attemptSummary = `up to ${countLabel( - attackAttemptUpperBound, - 'attack attempt', - 'attack attempts', - )}` - const hasPlannedAttackBound = estimate.total !== null +function adaptivePlannedCalculation(estimate: ScenarioRunEstimate): RunCalculation { + const details = estimate.adaptiveDetails + if (!details) { + throw new Error('Adaptive planned calculation requires adaptive details.') + } + const directBaselineCount = baselineCount(estimate) + const hasExactTotal = estimate.total !== null + || ( + estimate.minimum != null + && estimate.maximum != null + && estimate.minimum === estimate.maximum + ) + const hasPlannedTotal = estimate.total !== null || estimate.minimum != null || estimate.maximum != null - return hasPlannedAttackBound - ? `${attemptSummary} · ${formatProgressUnitSummary(estimate)}` - : `${countLabel(estimate.adaptiveDetails.objectiveCount, 'objective', 'objectives')} · ${attemptSummary}` + const adaptiveAttackCount = hasExactTotal + ? Math.max((estimate.total ?? estimate.maximum ?? 0) - directBaselineCount, 0) + : estimate.maximum != null + ? Math.max(estimate.maximum - directBaselineCount, 0) + : estimate.minimum != null + ? Math.max(estimate.minimum - directBaselineCount, 0) + : details.objectiveCount + const hasAdaptiveRange = directBaselineCount === 0 + && estimate.minimum != null + && estimate.minimum > 0 + && estimate.maximum != null + && estimate.minimum !== estimate.maximum + const adaptiveValue = hasExactTotal + ? formatCount(adaptiveAttackCount) + : hasAdaptiveRange + ? `${formatCount(estimate.minimum ?? 0)}–${formatCount(estimate.maximum ?? 0)}` + : estimate.maximum != null || estimate.minimum == null + ? `up to ${formatCount(adaptiveAttackCount)}` + : `at least ${formatCount(adaptiveAttackCount)}` + const adaptiveLabel = adaptiveAttackCount === 1 ? 'Adaptive attack' : 'Adaptive attacks' + const result = resultOperand(estimate) + const parts: CalculationPart[] = [] + if (directBaselineCount > 0) { + parts.push(operand( + 'baseline', + formatCount(directBaselineCount), + directBaselineCount === 1 ? 'direct baseline attack' : 'direct baseline attacks', + )) + parts.push(operator('baseline-plus', '+')) + } + parts.push(operand('adaptive-attacks', adaptiveValue, adaptiveLabel)) + if (hasPlannedTotal) { + parts.push(operator('planned-equals', '=')) + parts.push({ kind: 'operand', operand: result }) + } + + const adaptivePhrase = `${adaptiveValue} ${adaptiveLabel}` + const resultPhrase = `${result.value} ${result.label}` + const plannedResultPhrase = hasPlannedTotal + ? ` equals ${resultPhrase}.` + : '. Planned total is confirmed at launch.' + const accessibleLabel = directBaselineCount > 0 + ? `Direct baseline comparison is included: ${countLabel( + directBaselineCount, + 'direct baseline attack', + 'direct baseline attacks', + )} plus ${adaptivePhrase}${plannedResultPhrase}` + : `Direct baseline comparison is not included: ${adaptivePhrase}${plannedResultPhrase}` + return { parts, accessibleLabel } +} + +function adaptiveWorkCalculation(estimate: ScenarioRunEstimate): RunCalculation { + const details = estimate.adaptiveDetails + if (!details) { + throw new Error('Adaptive work calculation requires adaptive details.') + } + const objectiveLabel = details.objectiveCount === 1 ? 'objective' : 'objectives' + const techniqueLabel = details.techniquesPerObjectiveUpperBound === 1 + ? 'technique per objective' + : 'techniques per objective' + const attemptLabel = details.techniqueAttemptCountUpperBound === 1 + ? 'technique attempt' + : 'technique attempts' + const capProvenance = { + selectedCandidateCount: details.selectedCandidateTechniqueCount, + compatibleCandidateCount: details.candidateTechniqueCount, + limit: details.maxAttemptsPerObjective, + effectiveMaximum: details.techniquesPerObjectiveUpperBound, + } + const effectiveCapRule = formatAdaptiveCapMetadata(capProvenance) + const accessibleCapRule = formatAdaptiveCapAccessibleRule(capProvenance) + return { + parts: [ + operand('adaptive-objectives', formatCount(details.objectiveCount), objectiveLabel), + operator('adaptive-multiply', '×'), + operand( + 'adaptive-techniques', + `up to ${formatCount(details.techniquesPerObjectiveUpperBound)}`, + techniqueLabel, + false, + effectiveCapRule, + ), + operator('adaptive-equals', '='), + operand( + 'adaptive-result', + `up to ${formatCount(details.techniqueAttemptCountUpperBound)}`, + attemptLabel, + true, + ), + ], + accessibleLabel: `${countLabel(details.objectiveCount, 'objective', 'objectives')} multiplied by up to ${ + countLabel(details.techniquesPerObjectiveUpperBound, 'technique per objective', 'techniques per objective') + }, ${accessibleCapRule}, equals up to ${ + countLabel(details.techniqueAttemptCountUpperBound, 'technique attempt', 'technique attempts') + }.`, + } +} + +function adaptiveWorkContext(estimate: ScenarioRunEstimate): string { + const details = estimate.adaptiveDetails + if (!details) { + throw new Error('Adaptive work context requires adaptive details.') + } + const compatibilityContext = details.compatibilityMayReduceAttempts + ? ' Compatibility may reduce how many candidates each objective can try.' + : '' + return `Technique-attempt totals exclude multi-turn target exchanges and retries. Adaptive stops each objective after the first successful technique.${compatibilityContext}` } -function formatComponentFormula(component: ScenarioRunEstimateComponent): string { - if (component.factors.length === 0) { - return `${component.label}: ${formatEstimateValue(component.count)}` +function homogeneousTechniqueCalculation( + components: ScenarioRunEstimateComponent[], +): CalculationPart[] | null { + if (components.length < 2 || components.some((component) => component.condition !== null)) { + return null + } + const objectiveCounts = components.map((component) => objectiveFactor(component)?.count) + if (objectiveCounts.some((count) => count === undefined)) { + return null } - const factors = component.factors - .map((factor) => `${formatEstimateValue(factor.count)} ${factor.label}`) - .join(' × ') - return `${component.label}: ${factors} = ${formatEstimateValue(component.count)}` + const firstCount = objectiveCounts[0] + if (!objectiveCounts.every((count) => count === firstCount)) { + return null + } + return [ + operand( + 'technique-count', + formatCount(components.length), + components.length === 1 ? 'technique' : 'techniques', + ), + operator('technique-multiply', '×'), + operand( + 'objective-count', + formatCount(firstCount ?? 0), + firstCount === 1 ? 'objective' : 'objectives', + ), + ] } -function formatBackendFormula(estimate: ScenarioRunEstimate): string { - const components = estimate.components.length > 0 - ? estimate.components.map(formatComponentFormula).join(' + ') - : 'No additive components supplied' - const total = estimate.total === null - ? 'backend total is conditional' - : `backend total = ${formatEstimateValue(estimate.total)}` - return `${components}; ${total}` +function componentTerms(components: ScenarioRunEstimateComponent[]): CalculationPart[] { + const homogeneous = homogeneousTechniqueCalculation(components) + if (homogeneous) { + return homogeneous + } + if (components.length === 1 && components[0].condition === null && components[0].factors.length > 0) { + return [...components[0].factors] + .sort((left, right) => factorPriority(left) - factorPriority(right)) + .flatMap((factor, index) => [ + ...(index > 0 ? [operator(`factor-${index}-multiply`, '×')] : []), + operand(`factor-${factor.id}`, formatCount(factor.count), semanticFactorLabel(factor)), + ]) + } + return components.flatMap((component, index) => { + const objectiveCount = objectiveFactor(component)?.count + const value = formatCount(objectiveCount ?? component.count) + const unit = objectiveCount === undefined + ? component.count === 1 ? 'planned attack' : 'planned attacks' + : objectiveCount === 1 ? 'objective' : 'objectives' + const condition = component.condition ? ' · if supported' : '' + return [ + ...(index > 0 ? [operator(`component-${index}-plus`, '+')] : []), + operand( + `component-${component.id}`, + value, + `${unit} · ${humanizeLabel(component.label)}${condition}`, + ), + ] + }) +} + +function ordinaryCalculation(estimate: ScenarioRunEstimate): RunCalculation { + const baselineCount = estimate.components + .filter((component) => component.isBaseline) + .reduce((sum, component) => sum + component.count, 0) + const attackComponents = estimate.components.filter((component) => !component.isBaseline) + const resultOnly = baselineCount === 0 + && attackComponents.length === 1 + && attackComponents[0].factors.length === 0 + && attackComponents[0].condition === null + const attackParts = resultOnly ? [] : componentTerms(attackComponents) + const hasMultiplication = attackParts.some( + (part) => part.kind === 'operator' && part.operator.symbol === '×', + ) + const parts: CalculationPart[] = [] + if (baselineCount > 0 && hasMultiplication) { + parts.push(operator('attack-open', '(')) + } + parts.push(...attackParts) + if (baselineCount > 0 && hasMultiplication) { + parts.push(operator('attack-close', ')')) + } + if (baselineCount > 0) { + if (attackParts.length > 0) { + parts.push(operator('baseline-plus', '+')) + } + parts.push(operand( + 'baseline', + formatCount(baselineCount), + baselineCount === 1 ? 'direct baseline attack' : 'direct baseline attacks', + )) + } + const result = resultOperand(estimate) + if (parts.length > 0) { + parts.push(operator('total-equals', '=')) + } + parts.push({ kind: 'operand', operand: result }) + + const visibleExpression = parts.map((part) => part.kind === 'operator' + ? part.operator.symbol + : `${part.operand.value} ${part.operand.label}`).join(' ') + return { + parts, + accessibleLabel: `${visibleExpression + .replace(/×/g, 'multiplied by') + .replace(/\+/g, 'plus') + .replace(/=/g, 'equals')}.`, + context: estimate.total === null && estimate.minimum == null && estimate.maximum == null + ? formatEstimateSummary(estimate) + : undefined, + } } export function ScenarioRunEstimateSummary({ state }: ScenarioRunEstimateSummaryProps) { @@ -169,125 +480,109 @@ export function ScenarioRunEstimateSummary({ state }: ScenarioRunEstimateSummary )}
- {scopeLabel(state)} ) } -function EstimateComponents({ - estimate, +function RunCalculationView({ + calculation, + heading, idPrefix, + testId, }: { - estimate: ScenarioRunEstimate + calculation: RunCalculation + heading: string idPrefix: string + testId: string }) { const styles = useScenarioRunEstimateStyles() - const headingId = `${idPrefix}-components` + const headingId = `${idPrefix}-calculation` return ( -
- - Planned components - - {estimate.components.length === 0 ? ( - - No additive components supplied by the backend. +
+ {heading} +
+ {calculation.parts.map((part) => part.kind === 'operator' ? ( + + ) : ( + + ))} +
+ {calculation.summary && ( + + {calculation.summary} - ) : ( -
    - {estimate.components.map((component) => ( -
  1. -
    - {component.label} -
    - {component.isBaseline && ( - Baseline - )} - {formatEstimateValue(component.count)} -
    -
    - {component.factors.length > 0 && ( -
      - {component.factors.map((factor) => ( -
    • - - × {formatEstimateValue(factor.count)} {factor.label} - -
      • - ))} -
    - )} - {component.note && ( - {component.note} - )} -
    2. - ))} -
+ )} + {calculation.context && ( + {calculation.context} )}
) } -function EstimateDatasets({ - estimate, - idPrefix, -}: { - estimate: ScenarioRunEstimate - idPrefix: string -}) { +function EstimateSources({ estimate }: { estimate: ScenarioRunEstimate }) { const styles = useScenarioRunEstimateStyles() - const headingId = `${idPrefix}-datasets` + if (estimate.datasets.length === 0) { + return null + } + const { commonCaps, residualCapsByDatasetId } = normalizeDatasetCaps(estimate.datasets) return ( -
- - Dataset populations - - {estimate.datasets.length === 0 ? ( - - No dataset population details supplied by the backend. +
+ {commonCaps.length > 0 && ( + + {commonCaps.map(formatDatasetCap).join(' · ')} - ) : ( -
- {estimate.datasets.map((dataset) => ( -
-
- {dataset.name} - {dataset.kind} -
-
-
-
Logical seed groups
-
{formatEstimateValue(dataset.logicalSeedGroupCount)}
-
-
-
Selected seed groups
-
{formatEstimateValue(dataset.selectedSeedGroupCount)}
-
-
- {dataset.configuredCaps.length > 0 && ( -
- Configured caps -
    - {dataset.configuredCaps.map((cap) => ( -
  • - - {cap.label}: {formatEstimateValue(cap.count)} - {' '}({cap.configuredOn}{cap.datasetName ? `: ${cap.datasetName}` : ''}) - -
    • - ))} -
-
- )} - {dataset.selectionNote && ( - {dataset.selectionNote} - )} -
- ))} -
)} -
+ {estimate.datasets.map((dataset) => { + const residualCaps = residualCapsByDatasetId.get(dataset.id) ?? [] + return ( +
+ + {countLabel(dataset.selectedSeedGroupCount, 'objective', 'objectives')} from {dataset.name} + {dataset.logicalSeedGroupCount !== dataset.selectedSeedGroupCount + ? ` · ${formatCount(dataset.logicalSeedGroupCount)} available` + : ''} + + {dataset.selectionNote && ( + {dataset.selectionNote} + )} + {residualCaps.length > 0 && ( + + {residualCaps.map(formatDatasetCap).join(' · ')} + + )} +
+ ) + })} + ) } @@ -300,7 +595,7 @@ export function ScenarioRunEstimateDetails({ if (state.status === 'loading') { return (
- + {scopeLabel(state)}
) @@ -317,38 +612,38 @@ export function ScenarioRunEstimateDetails({ } const { estimate } = state + const hasAdaptiveDetails = estimate.adaptiveDetails !== null return (
- - {state.status === 'refreshing' && ( - {state.label} - )} - {state.status === 'stale' && ( -
- {state.label} - {state.error} -
+ {hasAdaptiveDetails ? ( + <> + + + + ) : ( + )} - - -
- - Backend formula - - {formatBackendFormula(estimate)} -
-
- - Estimate notes - - - {estimate.note ?? 'No additional note supplied by the backend.'} - - - Retries are {estimate.retriesIncluded ? 'included' : 'not included'}. - {' '}Estimate schema v{estimate.version}. - -
+ + + {hasAdaptiveDetails + ? adaptiveWorkContext(estimate) + : `Retries are ${estimate.retriesIncluded ? 'included' : 'not included'}.`} +
) } diff --git a/frontend/src/components/Scenarios/scenarioAdaptiveCap.test.ts b/frontend/src/components/Scenarios/scenarioAdaptiveCap.test.ts new file mode 100644 index 0000000000..1879453e53 --- /dev/null +++ b/frontend/src/components/Scenarios/scenarioAdaptiveCap.test.ts @@ -0,0 +1,42 @@ +import { + formatAdaptiveCapAccessibleRule, + formatAdaptiveCapFeedback, + formatAdaptiveCapMetadata, +} from './scenarioAdaptiveCap' + +describe('scenarioAdaptiveCap', () => { + it.each([ + [1, 1, '1 selected candidate · limit 3'], + [4, 4, '4 selected candidates · limit 3'], + [2, 1, '1 compatible candidate from 2 selected · limit 3'], + [4, 2, '2 compatible candidates from 4 selected · limit 3'], + ])( + 'formats metadata for %i selected and %i compatible candidates', + (selectedCandidateCount, compatibleCandidateCount, expected) => { + expect(formatAdaptiveCapMetadata({ + selectedCandidateCount, + compatibleCandidateCount, + limit: 3, + effectiveMaximum: 2, + })).toBe(expected) + }, + ) + + it('formats feedback with the effective maximum', () => { + expect(formatAdaptiveCapFeedback({ + selectedCandidateCount: 4, + compatibleCandidateCount: 2, + limit: 3, + effectiveMaximum: 2, + })).toBe('2 compatible candidates from 4 selected · limit 3 · effective maximum 2.') + }) + + it('formats the accessible minimum rule', () => { + expect(formatAdaptiveCapAccessibleRule({ + selectedCandidateCount: 1, + compatibleCandidateCount: 1, + limit: 3, + effectiveMaximum: 1, + })).toBe('the smaller of 1 selected candidate and limit 3') + }) +}) diff --git a/frontend/src/components/Scenarios/scenarioAdaptiveCap.ts b/frontend/src/components/Scenarios/scenarioAdaptiveCap.ts new file mode 100644 index 0000000000..6e7f4a3b26 --- /dev/null +++ b/frontend/src/components/Scenarios/scenarioAdaptiveCap.ts @@ -0,0 +1,33 @@ +interface AdaptiveCapProvenance { + selectedCandidateCount: number + compatibleCandidateCount: number + limit: number + effectiveMaximum: number +} + +function candidateContext({ + selectedCandidateCount, + compatibleCandidateCount, +}: Pick): string { + const compatibleLabel = compatibleCandidateCount === 1 ? 'compatible candidate' : 'compatible candidates' + const selectedLabel = selectedCandidateCount === 1 ? 'selected candidate' : 'selected candidates' + return compatibleCandidateCount < selectedCandidateCount + ? `${compatibleCandidateCount.toLocaleString()} ${compatibleLabel} from ${ + selectedCandidateCount.toLocaleString() + } selected` + : `${selectedCandidateCount.toLocaleString()} ${selectedLabel}` +} + +export function formatAdaptiveCapMetadata(provenance: AdaptiveCapProvenance): string { + return `${candidateContext(provenance)} · limit ${provenance.limit.toLocaleString()}` +} + +export function formatAdaptiveCapFeedback(provenance: AdaptiveCapProvenance): string { + return `${formatAdaptiveCapMetadata(provenance)} · effective maximum ${ + provenance.effectiveMaximum.toLocaleString() + }.` +} + +export function formatAdaptiveCapAccessibleRule(provenance: AdaptiveCapProvenance): string { + return `the smaller of ${candidateContext(provenance)} and limit ${provenance.limit.toLocaleString()}` +} diff --git a/frontend/src/components/Scenarios/scenarioDatasetCaps.test.ts b/frontend/src/components/Scenarios/scenarioDatasetCaps.test.ts new file mode 100644 index 0000000000..b3c0e2d9a9 --- /dev/null +++ b/frontend/src/components/Scenarios/scenarioDatasetCaps.test.ts @@ -0,0 +1,70 @@ +import type { ScenarioRunEstimateDataset, ScenarioRunEstimateDatasetCap } from '@/types' + +import { normalizeDatasetCaps } from './scenarioDatasetCaps' + +const SHARED_CAP: ScenarioRunEstimateDatasetCap = { + id: 'shared-cap', + label: 'combined cap', + count: 10, + configuredOn: 'compound', + datasetName: null, +} + +function makeDataset( + name: string, + configuredCaps: ScenarioRunEstimateDatasetCap[], +): ScenarioRunEstimateDataset { + return { + id: name, + name, + kind: 'dataset', + logicalSeedGroupCount: 20, + selectedSeedGroupCount: 5, + configuredCaps, + selectionNote: null, + } +} + +describe('scenarioDatasetCaps', () => { + it('returns empty normalized collections without datasets', () => { + const normalized = normalizeDatasetCaps([]) + + expect(normalized.commonCaps).toEqual([]) + expect(normalized.residualCapsByDatasetId.size).toBe(0) + }) + + it('lifts compound caps and preserves single-dataset row caps', () => { + const rowCap: ScenarioRunEstimateDatasetCap = { + ...SHARED_CAP, + id: 'row-cap', + configuredOn: 'dataset', + datasetName: 'alpha', + } + const dataset = makeDataset('alpha', [SHARED_CAP, rowCap]) + const normalized = normalizeDatasetCaps([dataset]) + + expect(normalized.commonCaps).toEqual([SHARED_CAP]) + expect(normalized.residualCapsByDatasetId.get('alpha')).toEqual([rowCap]) + }) + + it('preserves cap multiplicity while separating universal and residual row caps', () => { + const rowCap: ScenarioRunEstimateDatasetCap = { + id: 'row-cap', + label: 'per-dataset cap', + count: 5, + configuredOn: 'dataset', + datasetName: null, + } + const datasets = [ + makeDataset('alpha', [SHARED_CAP, rowCap, { ...rowCap, id: 'row-cap-duplicate' }]), + makeDataset('beta', [{ ...SHARED_CAP, id: 'shared-cap-beta' }, rowCap]), + ] + const normalized = normalizeDatasetCaps(datasets) + + expect(normalized.commonCaps).toEqual([SHARED_CAP, rowCap]) + expect(normalized.residualCapsByDatasetId.get('alpha')).toEqual([ + { ...rowCap, id: 'row-cap-duplicate' }, + ]) + expect(normalized.residualCapsByDatasetId.get('beta')).toEqual([]) + }) +}) diff --git a/frontend/src/components/Scenarios/scenarioDatasetCaps.ts b/frontend/src/components/Scenarios/scenarioDatasetCaps.ts new file mode 100644 index 0000000000..9c1b86c6f1 --- /dev/null +++ b/frontend/src/components/Scenarios/scenarioDatasetCaps.ts @@ -0,0 +1,87 @@ +import type { + ScenarioRunEstimateDataset, + ScenarioRunEstimateDatasetCap, +} from '@/types' + +interface NormalizedDatasetCaps { + readonly commonCaps: ScenarioRunEstimateDatasetCap[] + readonly residualCapsByDatasetId: ReadonlyMap +} + +function semanticCapKey(cap: ScenarioRunEstimateDatasetCap): string { + return JSON.stringify([cap.label, cap.count, cap.configuredOn]) +} + +function capOccurrences(caps: ScenarioRunEstimateDatasetCap[]): Map { + const occurrences = new Map() + for (const cap of caps) { + const key = semanticCapKey(cap) + occurrences.set(key, (occurrences.get(key) ?? 0) + 1) + } + return occurrences +} + +export function normalizeDatasetCaps(datasets: ScenarioRunEstimateDataset[]): NormalizedDatasetCaps { + const commonCaps: ScenarioRunEstimateDatasetCap[] = [] + const seenListLevelCaps = new Set() + + for (const dataset of datasets) { + for (const cap of dataset.configuredCaps) { + if (cap.configuredOn !== 'compound') { + continue + } + const key = semanticCapKey(cap) + if (!seenListLevelCaps.has(key)) { + seenListLevelCaps.add(key) + commonCaps.push(cap) + } + } + } + + const universalRowCapOccurrences = datasets.length > 1 + ? capOccurrences(datasets[0].configuredCaps.filter((cap) => cap.configuredOn !== 'compound')) + : new Map() + for (const dataset of datasets.slice(1)) { + const datasetOccurrences = capOccurrences( + dataset.configuredCaps.filter((cap) => cap.configuredOn !== 'compound'), + ) + for (const [key, count] of universalRowCapOccurrences) { + universalRowCapOccurrences.set(key, Math.min(count, datasetOccurrences.get(key) ?? 0)) + } + } + + const emittedRowCapOccurrences = new Map() + if (datasets.length > 0) { + for (const cap of datasets[0].configuredCaps) { + if (cap.configuredOn === 'compound') { + continue + } + const key = semanticCapKey(cap) + const emitted = emittedRowCapOccurrences.get(key) ?? 0 + if (emitted < (universalRowCapOccurrences.get(key) ?? 0)) { + commonCaps.push(cap) + emittedRowCapOccurrences.set(key, emitted + 1) + } + } + } + + const residualCapsByDatasetId = new Map() + for (const dataset of datasets) { + const consumedRowCapOccurrences = new Map() + const residualCaps = dataset.configuredCaps.filter((cap) => { + if (cap.configuredOn === 'compound') { + return false + } + const key = semanticCapKey(cap) + const consumed = consumedRowCapOccurrences.get(key) ?? 0 + if (consumed < (universalRowCapOccurrences.get(key) ?? 0)) { + consumedRowCapOccurrences.set(key, consumed + 1) + return false + } + return true + }) + residualCapsByDatasetId.set(dataset.id, residualCaps) + } + + return { commonCaps, residualCapsByDatasetId } +} diff --git a/frontend/src/components/Scenarios/scenarioTechniqueSets.test.ts b/frontend/src/components/Scenarios/scenarioTechniqueSets.test.ts new file mode 100644 index 0000000000..19b244fa61 --- /dev/null +++ b/frontend/src/components/Scenarios/scenarioTechniqueSets.test.ts @@ -0,0 +1,74 @@ +import type { RegisteredScenario } from '@/types' + +import { + techniqueSetDisplayName, + techniqueSetMembers, + techniqueSetName, + techniqueSetOptionLabel, +} from './scenarioTechniqueSets' + +function makeScenario(overrides: Partial = {}): RegisteredScenario { + return { + scenario_name: 'test.scenario', + scenario_type: 'TestScenario', + scenario_version: 1, + description: 'Test scenario.', + description_markdown: 'Test scenario.', + default_technique: 'default', + default_techniques: ['crescendo'], + aggregate_techniques: ['default', 'quick_set'], + aggregate_technique_expansions: { + quick_set: ['crescendo', 'crescendo', 'pair'], + }, + all_techniques: ['crescendo', 'pair'], + default_datasets: [], + dataset_size_limit: { + default_scope: 'none', + default_count: null, + override_scope: 'unsupported', + }, + default_dataset_summaries: [], + baseline_policy: 'forbidden', + include_baseline_by_default: false, + supported_parameters: [], + default_run_size: { + version: 1, + status: 'unavailable', + total_attack_count: null, + minimum_attack_count: null, + maximum_attack_count: null, + condition: null, + components: [], + datasets: [], + adaptive_details: null, + note: null, + retries_included: false, + }, + ...overrides, + } +} + +describe('scenarioTechniqueSets', () => { + it('formats known, custom, and empty technique-set names', () => { + expect(techniqueSetName('default')).toBe('Recommended') + expect(techniqueSetName('custom_red_team')).toBe('Custom red team') + expect(techniqueSetName('')).toBe('') + }) + + it('expands named sets, removes duplicates, and falls back to default members', () => { + const scenario = makeScenario() + + expect(techniqueSetMembers(scenario, 'quick_set')).toEqual(['crescendo', 'pair']) + expect(techniqueSetMembers(scenario, 'default')).toEqual(['crescendo']) + expect(techniqueSetMembers(scenario, 'unknown_set')).toEqual([]) + }) + + it('labels default and custom sets with singular and plural member counts', () => { + const scenario = makeScenario() + + expect(techniqueSetDisplayName(scenario, 'default')).toBe('Recommended (default)') + expect(techniqueSetDisplayName(scenario, 'quick_set')).toBe('Quick set') + expect(techniqueSetOptionLabel(scenario, 'default')).toBe('Recommended (default) — 1 technique') + expect(techniqueSetOptionLabel(scenario, 'quick_set')).toBe('Quick set (2 techniques)') + }) +}) diff --git a/frontend/src/services/api.test.ts b/frontend/src/services/api.test.ts index 3e5f935102..9721b6cec9 100644 --- a/frontend/src/services/api.test.ts +++ b/frontend/src/services/api.test.ts @@ -18,6 +18,7 @@ import { versionApi, targetsApi, attacksApi, + datasetsApi, scenariosApi, } from "./api"; @@ -469,6 +470,22 @@ describe("api service", () => { }); }); + describe("datasetsApi", () => { + it("lists registered datasets", async () => { + const mockResponse = { + data: { + items: [{ name: "harmbench" }, { name: "xstest" }], + }, + }; + (apiClient.get as jest.Mock).mockResolvedValueOnce(mockResponse); + + const result = await datasetsApi.listDatasets(); + + expect(apiClient.get).toHaveBeenCalledWith("/datasets"); + expect(result).toEqual(mockResponse.data); + }); + }); + describe("scenariosApi", () => { it("lists the scenario catalog with default params", async () => { const mockResponse = { @@ -541,8 +558,12 @@ describe("api service", () => { version: 1, status: "exact", total_attack_count: 8, + minimum_attack_count: null, + maximum_attack_count: null, + condition: null, components: [], datasets: [], + adaptive_details: null, note: null, retries_included: false, }, @@ -573,6 +594,55 @@ describe("api service", () => { expect(result.total_attack_count).toBe(8); }); + it("preserves Adaptive conditional work metadata in the initial estimate response", async () => { + const mockResponse = { + data: { + version: 1, + status: "conditional", + total_attack_count: null, + minimum_attack_count: null, + maximum_attack_count: null, + condition: null, + components: [], + datasets: [], + adaptive_details: { + objective_count: 21, + selected_candidate_technique_count: 2, + candidate_technique_count: 2, + max_attempts_per_objective: 3, + techniques_per_objective_upper_bound: 2, + technique_attempt_count_upper_bound: 42, + stop_on_first_success: true, + compatibility_may_reduce_attempts: true, + }, + note: null, + retries_included: false, + }, + }; + (apiClient.post as jest.Mock).mockResolvedValueOnce(mockResponse); + const controller = new AbortController(); + const request = { + target_name: "target-a", + techniques: ["default"], + include_baseline: true, + scenario_params: { max_attempts_per_objective: 3 }, + }; + + const result = await scenariosApi.estimateRun( + "adaptive.text_adaptive", + request, + controller.signal + ); + + expect(apiClient.post).toHaveBeenCalledWith( + "/scenarios/catalog/adaptive.text_adaptive/estimate", + request, + { signal: controller.signal } + ); + expect(result.adaptive_details).toEqual(mockResponse.data.adaptive_details); + expect(result.total_attack_count).toBeNull(); + }); + it("posts the exact RunScenarioRequest payload to start a run", async () => { const mockResponse = { data: { diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 75580637d0..3fd03c50db 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -28,6 +28,7 @@ import type { CreateConversationRequest, CreateConversationResponse, ChangeMainConversationResponse, + DatasetListResponse, ListRegisteredScenariosResponse, RegisteredScenario, RunScenarioRequest, @@ -351,6 +352,13 @@ export const labelsApi = { }, } +export const datasetsApi = { + listDatasets: async (): Promise => { + const response = await apiClient.get('/datasets') + return response.data + }, +} + export const scenariosApi = { /** * Lists one page of the scenario catalog. Callers that need the full diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 82d9bfe4bf..761cc63ff8 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -379,6 +379,16 @@ export interface ChangeMainConversationResponse { conversation_id: string } +// --- Datasets --- + +export interface DatasetInfo { + name: string +} + +export interface DatasetListResponse { + items: DatasetInfo[] +} + // --- Scenarios --- export interface RegisteredScenario { @@ -393,6 +403,7 @@ export interface RegisteredScenario { aggregate_technique_expansions: Record all_techniques: string[] default_datasets: string[] + dataset_size_limit: ScenarioDatasetSizeLimit default_dataset_summaries: ScenarioDatasetSummary[] baseline_policy: 'enabled' | 'disabled' | 'forbidden' include_baseline_by_default: boolean @@ -465,6 +476,12 @@ export interface ScenarioDatasetSummary { selection_note: string | null } +export interface ScenarioDatasetSizeLimit { + default_scope: 'none' | 'per_dataset' | 'combined' | 'heterogeneous' + default_count: number | null + override_scope: 'per_dataset' | 'combined' | 'unsupported' +} + export interface ScenarioDefaultRunSizeEstimate { version: 1 status: ScenarioRunSizeEstimateStatus @@ -569,17 +586,6 @@ export type ScenarioRunEstimateState = status: 'loading' scope: 'default' | 'request' } - | { - status: 'refreshing' - estimate: ScenarioRunEstimate - label: string - } - | { - status: 'stale' - estimate: ScenarioRunEstimate - label: string - error: string - } | ScenarioRunEstimateResult export type ScenarioRunEstimator = ( diff --git a/pyrit/analytics/technique_analysis.py b/pyrit/analytics/technique_analysis.py index b892946113..a804a510ac 100644 --- a/pyrit/analytics/technique_analysis.py +++ b/pyrit/analytics/technique_analysis.py @@ -5,6 +5,7 @@ from __future__ import annotations +from collections import Counter, defaultdict from typing import TYPE_CHECKING from pyrit.analytics.result_analysis import AttackStats, _compute_stats @@ -12,11 +13,33 @@ from pyrit.models import AttackOutcome if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Iterable, Mapping, Sequence from pyrit.memory.memory_interface import MemoryInterface +def _compute_grouped_outcome_stats(grouped_outcomes: Iterable[tuple[str, AttackOutcome]]) -> dict[str, AttackStats]: + """ + Aggregate keyed outcomes into attack statistics. + + Returns: + dict[str, AttackStats]: Statistics keyed by the caller's grouping value. + """ + counts: dict[str, Counter[AttackOutcome]] = defaultdict(Counter) + for key, outcome in grouped_outcomes: + counts[key][outcome] += 1 + + return { + key: _compute_stats( + successes=outcomes[AttackOutcome.SUCCESS], + failures=outcomes[AttackOutcome.FAILURE], + undetermined=outcomes[AttackOutcome.UNDETERMINED], + errors=outcomes[AttackOutcome.ERROR], + ) + for key, outcomes in counts.items() + } + + def compute_technique_stats( *, technique_eval_hashes: Sequence[str], @@ -61,24 +84,88 @@ def compute_technique_stats( ) requested = set(technique_eval_hashes) - counts: dict[str, tuple[int, int, int, int]] = {} + grouped_outcomes: list[tuple[str, AttackOutcome]] = [] for result in results: identifier = result.atomic_attack_identifier eval_hash = identifier.eval_hash if identifier is not None else None if eval_hash is None or eval_hash not in requested: continue + grouped_outcomes.append((eval_hash, result.outcome)) + + return _compute_grouped_outcome_stats(grouped_outcomes) + + +def compute_labeled_technique_stats( + *, + technique_identifiers: Sequence[str], + label_name: str, + technique_eval_hashes_by_identifier: Mapping[str, str] | None = None, + scenario_result_id: str | None = None, + targeted_harm_categories: Sequence[str] | None = None, + memory: MemoryInterface | None = None, +) -> dict[str, AttackStats]: + """ + Compute per-technique statistics from identity labels and eval-hash history. + + Args: + technique_identifiers (Sequence[str]): Stable technique identifiers to + aggregate. Returned dict is keyed by these identifiers. + label_name (str): Result-label key containing the technique identifier. + technique_eval_hashes_by_identifier (Mapping[str, str] | None): + Optional mapping from requested selector identifiers to the full + ``AttackTechnique`` eval hashes persisted by normal scenarios. + Matching labeled and eval-hash rows are merged by result ID so a + row visible through both paths is counted once. + scenario_result_id (str | None): Restrict to a single scenario run. + Defaults to ``None`` (aggregate across all runs). + targeted_harm_categories (Sequence[str] | None): Restrict to results + whose attack targeted these harm categories. Defaults to ``None``. + memory (MemoryInterface | None): Memory backend to query. Defaults to + ``CentralMemory.get_memory_instance()``. - s, f, u, e = counts.get(eval_hash, (0, 0, 0, 0)) - if result.outcome == AttackOutcome.SUCCESS: - counts[eval_hash] = (s + 1, f, u, e) - elif result.outcome == AttackOutcome.FAILURE: - counts[eval_hash] = (s, f + 1, u, e) - elif result.outcome == AttackOutcome.ERROR: - counts[eval_hash] = (s, f, u, e + 1) + Returns: + dict[str, AttackStats]: Stats per requested technique identifier. + Identifiers with no historical results are omitted. + """ + if not technique_identifiers: + return {} + + if memory is None: + memory = CentralMemory.get_memory_instance() + labeled_results = memory.get_attack_results( + labels={label_name: list(technique_identifiers)}, + scenario_result_id=scenario_result_id, + targeted_harm_categories=targeted_harm_categories, + ) + eval_results = ( + memory.get_attack_results( + atomic_attack_eval_hashes=sorted(set(technique_eval_hashes_by_identifier.values())), + scenario_result_id=scenario_result_id, + targeted_harm_categories=targeted_harm_categories, + ) + if technique_eval_hashes_by_identifier + else [] + ) + + requested = set(technique_identifiers) + identifiers_by_eval_hash: dict[str, list[str]] = {} + for technique_identifier, eval_hash in (technique_eval_hashes_by_identifier or {}).items(): + if technique_identifier in requested: + identifiers_by_eval_hash.setdefault(eval_hash, []).append(technique_identifier) + + unique_results = {result.attack_result_id: result for result in [*labeled_results, *eval_results]} + grouped_outcomes: list[tuple[str, AttackOutcome]] = [] + for result in unique_results.values(): + labeled_identifier = result.labels.get(label_name) + if labeled_identifier in requested: + matching_identifiers = [labeled_identifier] else: - counts[eval_hash] = (s, f, u + 1, e) + result_identifier = result.atomic_attack_identifier + result_eval_hash = result_identifier.eval_hash if result_identifier is not None else None + matching_identifiers = identifiers_by_eval_hash.get(result_eval_hash or "", []) + if not matching_identifiers: + continue - return { - eval_hash: _compute_stats(successes=s, failures=f, undetermined=u, errors=e) - for eval_hash, (s, f, u, e) in counts.items() - } + grouped_outcomes.extend((technique_identifier, result.outcome) for technique_identifier in matching_identifiers) + + return _compute_grouped_outcome_stats(grouped_outcomes) diff --git a/pyrit/backend/services/scenario_run_service.py b/pyrit/backend/services/scenario_run_service.py index d407d49a10..9cafe49293 100644 --- a/pyrit/backend/services/scenario_run_service.py +++ b/pyrit/backend/services/scenario_run_service.py @@ -76,6 +76,7 @@ TargetRegistry, ) from pyrit.scenario import Scenario +from pyrit.scenario.core.dataset_configuration import CompoundDatasetAttackConfiguration if TYPE_CHECKING: from pyrit.converter import Converter @@ -952,7 +953,27 @@ def resolve_scenario_configuration( if dataset_names or max_dataset_size is not None or filters: default_config = introspection_instance._default_dataset_config - if dataset_names: + if isinstance(default_config, CompoundDatasetAttackConfiguration): + names_changed = dataset_names is not None and dataset_names != default_config.dataset_names + if names_changed: + try: + resolved["dataset_config"] = default_config.with_dataset_names( + dataset_names=dataset_names, + max_dataset_size=max_dataset_size, + filters=filters or None, + ) + except TypeError as exc: + raise ValueError( + f"Scenario '{scenario_name}' does not support overriding datasets through " + f"its {type(default_config).__name__} configuration: {exc}" + ) from exc + else: + if max_dataset_size is not None: + default_config.update_child_max_dataset_size(max_dataset_size=max_dataset_size) + if filters: + default_config.update_filters(filters=filters) + resolved["dataset_config"] = default_config + elif dataset_names: # Construct a fresh instance of the scenario's own dataset-config # class so subclass-specific behavior is preserved. default_config_class = type(default_config) diff --git a/pyrit/backend/services/scenario_service.py b/pyrit/backend/services/scenario_service.py index e6fcbfccd6..7da10cca86 100644 --- a/pyrit/backend/services/scenario_service.py +++ b/pyrit/backend/services/scenario_service.py @@ -14,7 +14,7 @@ from pyrit.backend.models.common import PaginationInfo from pyrit.backend.models.scenarios import ListRegisteredScenariosResponse from pyrit.backend.services.scenario_run_service import ScenarioRunService -from pyrit.models.catalog.scenario import ( +from pyrit.models.catalog import ( RegisteredScenario, ScenarioDefaultRunSizeEstimate, ScenarioRunSizeEstimateRequest, @@ -24,7 +24,9 @@ logger = logging.getLogger(__name__) _ESTIMATE_CACHE_SIZE = 128 -_ESTIMATE_CONCURRENCY = 1 +_ESTIMATE_CONCURRENCY = 4 +_CONFIGURED_ESTIMATE_CONCURRENCY = 4 +_DEFAULT_ESTIMATE_TIMEOUT_SECONDS = 3.0 _ESTIMATE_INFLIGHT_SIZE = 256 _UNAVAILABLE_CACHE_TTL_SECONDS = 30.0 _EstimateCacheKey = tuple[str, int] @@ -62,6 +64,7 @@ def _metadata_to_registered_scenario( }, all_techniques=list(metadata.all_techniques), default_datasets=list(metadata.default_datasets), + dataset_size_limit=metadata.dataset_size_limit, default_dataset_summaries=estimate.datasets, supported_parameters=list(metadata.supported_parameters), baseline_policy=metadata.baseline_policy, @@ -80,6 +83,7 @@ def __init__(self) -> None: self._estimate_tasks: OrderedDict[_EstimateCacheKey, _EstimateTask] = OrderedDict() self._estimate_task_lock = asyncio.Lock() self._estimate_semaphore = asyncio.Semaphore(_ESTIMATE_CONCURRENCY) + self._configured_estimate_semaphore = asyncio.Semaphore(_CONFIGURED_ESTIMATE_CONCURRENCY) async def list_scenarios_async( self, @@ -154,11 +158,7 @@ async def estimate_scenario_run_size_async( if metadata is None: return None - semaphore = getattr(self, "_estimate_semaphore", None) - if semaphore is None: - semaphore = asyncio.Semaphore(_ESTIMATE_CONCURRENCY) - self._estimate_semaphore = semaphore - async with semaphore: + async with self._configured_estimate_semaphore: return await self._estimate_configured_run_size_async( scenario_name=scenario_name, request=request, @@ -169,30 +169,19 @@ async def _get_default_run_size_estimate_async( ) -> ScenarioDefaultRunSizeEstimate: """Return a cached, cancellation-safe scenario-owned estimate.""" cache_key = (metadata.registry_name, metadata.scenario_version) - cache = getattr(self, "_estimate_cache", None) - if cache is None: - cache = OrderedDict() - self._estimate_cache = cache while True: cached = self._read_estimate_cache(cache_key=cache_key) if cached is not None: return cached - task_lock = getattr(self, "_estimate_task_lock", None) - if task_lock is None: - task_lock = asyncio.Lock() - self._estimate_task_lock = task_lock wait_for_capacity: _EstimateTask | None = None task: _EstimateTask | None = None - async with task_lock: + async with self._estimate_task_lock: cached = self._read_estimate_cache(cache_key=cache_key) if cached is not None: return cached - tasks = getattr(self, "_estimate_tasks", None) - if tasks is None: - tasks = OrderedDict() - self._estimate_tasks = tasks + tasks = self._estimate_tasks for completed_key in [key for key, candidate in tasks.items() if candidate.done()]: del tasks[completed_key] task = tasks.get(cache_key) @@ -243,19 +232,26 @@ async def _compute_default_run_size_estimate_async( Returns: ScenarioDefaultRunSizeEstimate: Scenario-owned estimate. """ - semaphore = getattr(self, "_estimate_semaphore", None) - if semaphore is None: - semaphore = asyncio.Semaphore(_ESTIMATE_CONCURRENCY) - self._estimate_semaphore = semaphore - async with semaphore: - try: - scenario = await asyncio.to_thread(self._registry.create_instance, scenario_name) - estimate = await scenario.get_default_run_size_estimate_async() - except Exception as exc: - logger.warning("Default-run estimate failed for scenario '%s': %s", scenario_name, exc) - estimate = ScenarioDefaultRunSizeEstimate.unavailable( - note=f"The scenario could not resolve its default inputs for estimation ({type(exc).__name__})." + try: + async with self._estimate_semaphore: + estimate = await asyncio.wait_for( + self._run_default_estimate_async(scenario_name=scenario_name), + timeout=_DEFAULT_ESTIMATE_TIMEOUT_SECONDS, ) + except TimeoutError: + logger.warning( + "Default-run estimate timed out for scenario '%s' after %.1f seconds", + scenario_name, + _DEFAULT_ESTIMATE_TIMEOUT_SECONDS, + ) + estimate = ScenarioDefaultRunSizeEstimate.unavailable( + note="The default estimate timed out; open the scenario to calculate the configured run size." + ) + except Exception as exc: + logger.warning("Default-run estimate failed for scenario '%s': %s", scenario_name, exc) + estimate = ScenarioDefaultRunSizeEstimate.unavailable( + note=f"The scenario could not resolve its default inputs for estimation ({type(exc).__name__})." + ) expires_at = ( monotonic() + _UNAVAILABLE_CACHE_TTL_SECONDS @@ -269,6 +265,16 @@ async def _compute_default_run_size_estimate_async( cache.popitem(last=False) return estimate + async def _run_default_estimate_async(self, *, scenario_name: str) -> ScenarioDefaultRunSizeEstimate: + """ + Run one default estimate. + + Returns: + ScenarioDefaultRunSizeEstimate: The authoritative scenario estimate. + """ + scenario = await asyncio.to_thread(self._registry.create_instance, scenario_name) + return await scenario.get_default_run_size_estimate_async() + def _clear_estimate_task(self, *, task: _EstimateTask, cache_key: _EstimateCacheKey) -> None: """Remove a completed single-flight task without disturbing a replacement.""" tasks = self._estimate_tasks diff --git a/pyrit/models/__init__.py b/pyrit/models/__init__.py index 4441401427..213f15f8c0 100644 --- a/pyrit/models/__init__.py +++ b/pyrit/models/__init__.py @@ -18,11 +18,14 @@ from pyrit.models.additional_initializer import AdditionalInitializer from pyrit.models.catalog import ( + ScenarioAdaptiveRunSizeDetails, ScenarioDatasetSizeCap, + ScenarioDatasetSizeLimit, ScenarioDatasetSummary, ScenarioDefaultRunSizeEstimate, ScenarioRunSizeComponent, ScenarioRunSizeEstimate, + ScenarioRunSizeEstimateCondition, ScenarioRunSizeEstimateRequest, ScenarioRunSizeEstimateStatus, ScenarioRunSizeFactor, @@ -227,12 +230,15 @@ "ScorerEvaluationIdentifier", "ScorerIdentifier", "ScenarioIdentifier", + "ScenarioAdaptiveRunSizeDetails", "ScenarioDatasetSizeCap", + "ScenarioDatasetSizeLimit", "ScenarioDatasetSummary", "ScenarioDefaultRunSizeEstimate", "ScenarioRunSizeEstimate", "ScenarioResult", "ScenarioRunSizeComponent", + "ScenarioRunSizeEstimateCondition", "ScenarioRunSizeEstimateRequest", "ScenarioRunSizeEstimateStatus", "ScenarioRunSizeFactor", diff --git a/pyrit/models/catalog/__init__.py b/pyrit/models/catalog/__init__.py index 692e3a54d5..f69f147f9b 100644 --- a/pyrit/models/catalog/__init__.py +++ b/pyrit/models/catalog/__init__.py @@ -21,15 +21,21 @@ AttackRetrySummary, RegisteredScenario, RunScenarioRequest, + ScenarioAdaptiveRunSizeDetails, ScenarioDatasetSizeCap, + ScenarioDatasetSizeLimit, ScenarioDatasetSummary, ScenarioDefaultRunSizeEstimate, + ScenarioOverloadSummary, + ScenarioRunHeader, ScenarioRunSizeComponent, ScenarioRunSizeEstimate, + ScenarioRunSizeEstimateCondition, ScenarioRunSizeEstimateRequest, ScenarioRunSizeEstimateStatus, ScenarioRunSizeFactor, ScenarioRunSummary, + ScenarioTargetSummary, ) from pyrit.models.catalog.target import ( TargetInstance, @@ -41,14 +47,20 @@ "RegisteredInitializer", "RegisteredScenario", "RunScenarioRequest", + "ScenarioAdaptiveRunSizeDetails", "ScenarioDatasetSizeCap", + "ScenarioDatasetSizeLimit", "ScenarioDatasetSummary", "ScenarioDefaultRunSizeEstimate", + "ScenarioOverloadSummary", "ScenarioRunSizeEstimate", "ScenarioRunSizeComponent", + "ScenarioRunSizeEstimateCondition", "ScenarioRunSizeEstimateRequest", "ScenarioRunSizeEstimateStatus", "ScenarioRunSizeFactor", + "ScenarioRunHeader", "ScenarioRunSummary", + "ScenarioTargetSummary", "TargetInstance", ] diff --git a/pyrit/models/catalog/scenario.py b/pyrit/models/catalog/scenario.py index ee19a54ee5..cd10c1b8b0 100644 --- a/pyrit/models/catalog/scenario.py +++ b/pyrit/models/catalog/scenario.py @@ -67,6 +67,13 @@ class ScenarioRunSizeEstimateStatus(str, Enum): Unavailable = "unavailable" +class ScenarioRunSizeEstimateCondition(str, Enum): + """Reason an estimate remains conditional until launch.""" + + TargetCapabilities = "target_capabilities" + LaunchConfiguration = "launch_configuration" + + class ScenarioRunSizeFactor(BaseModel): """One labeled multiplicative factor in a run-size component.""" @@ -81,6 +88,7 @@ class ScenarioRunSizeComponent(BaseModel): count: int = Field(..., ge=0) factors: list[ScenarioRunSizeFactor] = Field(default_factory=list) is_baseline: bool = False + condition: ScenarioRunSizeEstimateCondition | None = None note: str | None = None @model_validator(mode="after") @@ -103,6 +111,65 @@ def validate_factor_product(self) -> "ScenarioRunSizeComponent": return self +class ScenarioAdaptiveRunSizeDetails(BaseModel): + """Structured work bounds for an adaptive scenario estimate.""" + + objective_count: int = Field(..., ge=0) + selected_candidate_technique_count: int = Field(..., ge=1) + candidate_technique_count: int = Field(..., ge=1) + max_attempts_per_objective: int = Field(..., ge=1) + techniques_per_objective_upper_bound: int = Field(..., ge=1) + technique_attempt_count_upper_bound: int = Field(..., ge=0) + stop_on_first_success: Literal[True] = True + compatibility_may_reduce_attempts: Literal[True] = True + + @model_validator(mode="before") + @classmethod + def default_selected_candidate_count(cls, data: Any) -> Any: + """ + Preserve version-1 payload compatibility when the selected count is absent. + + Returns: + Any: Input data with the selected count defaulted to the compatible count. + """ + if ( + isinstance(data, dict) + and "selected_candidate_technique_count" not in data + and "candidate_technique_count" in data + ): + return { + **data, + "selected_candidate_technique_count": data["candidate_technique_count"], + } + return data + + @model_validator(mode="after") + def validate_attempt_bounds(self) -> "ScenarioAdaptiveRunSizeDetails": + """ + Ensure the serialized adaptive attempt bounds match the configured pool. + + Returns: + ScenarioAdaptiveRunSizeDetails: The validated details. + + Raises: + ValueError: If either derived upper bound is inconsistent. + """ + if self.candidate_technique_count > self.selected_candidate_technique_count: + raise ValueError("candidate_technique_count cannot exceed selected_candidate_technique_count") + expected_per_objective = min(self.candidate_technique_count, self.max_attempts_per_objective) + if self.techniques_per_objective_upper_bound != expected_per_objective: + raise ValueError( + "techniques_per_objective_upper_bound must equal " + "min(candidate_technique_count, max_attempts_per_objective)" + ) + expected_total = self.objective_count * expected_per_objective + if self.technique_attempt_count_upper_bound != expected_total: + raise ValueError( + "technique_attempt_count_upper_bound must equal objective_count * techniques_per_objective_upper_bound" + ) + return self + + class ScenarioDatasetSizeCap(BaseModel): """One configured cap affecting a dataset or compound population.""" @@ -127,6 +194,30 @@ class ScenarioDatasetSummary(BaseModel): selection_note: str | None = None +class ScenarioDatasetSizeLimit(BaseModel): + """Structured default and override semantics for a scenario's dataset-size limit.""" + + default_scope: Literal["none", "per_dataset", "combined", "heterogeneous"] = "none" + default_count: int | None = Field(default=None, ge=1) + override_scope: Literal["per_dataset", "combined", "unsupported"] = "per_dataset" + + @model_validator(mode="after") + def validate_default_count(self) -> "ScenarioDatasetSizeLimit": + """ + Require a count exactly when the default has one representable scope. + + Returns: + ScenarioDatasetSizeLimit: The validated limit metadata. + + Raises: + ValueError: If the count does not match the declared default scope. + """ + has_representable_default = self.default_scope in {"per_dataset", "combined"} + if has_representable_default != (self.default_count is not None): + raise ValueError("default_count must be set exactly for per_dataset or combined defaults") + return self + + class ScenarioDefaultRunSizeEstimate(BaseModel): """ Structured estimate of default planned scenario execution units. @@ -142,8 +233,12 @@ class ScenarioDefaultRunSizeEstimate(BaseModel): ge=0, validation_alias=AliasChoices("total_attack_count", "total"), ) + minimum_attack_count: int | None = Field(default=None, ge=0) + maximum_attack_count: int | None = Field(default=None, ge=0) + condition: ScenarioRunSizeEstimateCondition | None = None components: list[ScenarioRunSizeComponent] = Field(default_factory=list) datasets: list[ScenarioDatasetSummary] = Field(default_factory=list) + adaptive_details: ScenarioAdaptiveRunSizeDetails | None = None note: str | None = Field(default=None, validation_alias=AliasChoices("note", "caveat")) retries_included: Literal[False] = False @@ -191,14 +286,30 @@ def validate_total(self) -> "ScenarioDefaultRunSizeEstimate": Raises: ValueError: If an exact estimate omits or misstates its total. """ - if self.status is ScenarioRunSizeEstimateStatus.Exact: - if self.total_attack_count is None: - raise ValueError("Exact default-run estimates require total_attack_count") - component_total = sum(component.count for component in self.components) - if component_total != self.total_attack_count: - raise ValueError( - f"Exact default-run estimate components total {component_total}, not {self.total_attack_count}" - ) + if ( + self.minimum_attack_count is not None + and self.maximum_attack_count is not None + and self.minimum_attack_count > self.maximum_attack_count + ): + raise ValueError("minimum_attack_count must be less than or equal to maximum_attack_count") + + if self.status is not ScenarioRunSizeEstimateStatus.Exact: + return self + + if self.total_attack_count is None: + raise ValueError("Exact default-run estimates require total_attack_count") + for field_name, bound in ( + ("minimum_attack_count", self.minimum_attack_count), + ("maximum_attack_count", self.maximum_attack_count), + ): + if bound is not None and bound != self.total_attack_count: + raise ValueError(f"Exact default-run estimates require {field_name} to equal total_attack_count") + + component_total = sum(component.count for component in self.components) + if component_total != self.total_attack_count: + raise ValueError( + f"Exact default-run estimate components total {component_total}, not {self.total_attack_count}" + ) return self @classmethod @@ -246,6 +357,10 @@ class RegisteredScenario(BaseModel): ) all_techniques: list[str] = Field(..., description="All available concrete technique names") default_datasets: list[str] = Field(..., description="Default dataset names used by the scenario") + dataset_size_limit: ScenarioDatasetSizeLimit = Field( + default_factory=ScenarioDatasetSizeLimit, + description="Structured scenario-default and explicit-override dataset-size limit semantics", + ) default_dataset_summaries: list[ScenarioDatasetSummary] = Field( default_factory=list, description="Logical and effectively selected attack-group counts for the default configuration", @@ -452,4 +567,31 @@ class ScenarioTargetSummary(BaseModel): identifier_hash: str | None = Field(None, description="Canonical target identifier hash") +class ScenarioRunHeader(BaseModel): + """Stable scenario run fields shared with lightweight progress responses.""" + + scenario_result_id: str = Field(..., description="UUID of the ScenarioResult in memory") + scenario_name: str = Field(..., description="Registry key of the scenario being run") + scenario_registry_name: str | None = Field(None, description="Requested scenario registry key when available") + scenario_version: int = Field(0, ge=0, description="Version of the scenario") + status: ScenarioRunState = Field(..., description="Current run status") + created_at: datetime = Field(..., description="When the run was created") + techniques_used: list[str] = Field(default_factory=list, description="Technique names that were executed") + labels: dict[str, str] = Field(default_factory=dict, description="Labels attached to this run") + completed_at: datetime | None = Field(None, description="When the scenario finished") + pyrit_version: str | None = Field(None, description="PyRIT version that created the run") + target: ScenarioTargetSummary | None = Field(None, description="Safe objective-target identity") + datasets_used: list[str] = Field(default_factory=list, description="Resolved datasets selected for the run") + scenario_parameters: dict[str, Any] = Field( + default_factory=dict, + description="Safe resolved scenario parameters; sensitive fields are removed", + ) + queue_position: int | None = Field(None, ge=1, description="Current 1-based waiting position") + active_scenario_result_id: str | None = Field(None, description="Currently executing scenario result ID") + overload_summaries: list[ScenarioOverloadSummary] = Field( + default_factory=list, + description="Bounded recent HTTP 429 and 5xx retry evidence grouped by component role", + ) + + ScenarioRunSummary.model_rebuild() diff --git a/pyrit/registry/components/scenario_registry.py b/pyrit/registry/components/scenario_registry.py index 5be7fea7dd..b03fda50ea 100644 --- a/pyrit/registry/components/scenario_registry.py +++ b/pyrit/registry/components/scenario_registry.py @@ -18,7 +18,7 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal -from pyrit.models import ScenarioDefaultRunSizeEstimate, class_name_to_snake_case +from pyrit.models import ScenarioDatasetSizeLimit, ScenarioDefaultRunSizeEstimate, class_name_to_snake_case from pyrit.models.identifiers.scenario_identifier import ScenarioIdentifier from pyrit.registry.registry import ParamBagRegistry from pyrit.registry.registry_metadata import RegistryMetadata @@ -63,6 +63,9 @@ class ScenarioMetadata(RegistryMetadata): # Default dataset names used by this scenario. default_datasets: tuple[str, ...] = field(kw_only=True) + # Structured default and override semantics for the dataset-size control. + dataset_size_limit: ScenarioDatasetSizeLimit = field(kw_only=True, default_factory=ScenarioDatasetSizeLimit) + # Scenario-declared custom parameters. supported_parameters: tuple[Parameter, ...] = field(kw_only=True, default=()) @@ -175,7 +178,12 @@ def _build_metadata(self, name: str, cls: type[Scenario]) -> ScenarioMetadata: ) for aggregate in technique_class.get_aggregate_techniques() ) - default_datasets = tuple(instance._default_dataset_config.dataset_names) + default_dataset_config = instance._default_dataset_config + default_datasets = tuple(default_dataset_config.dataset_names) + dataset_size_limit = self._build_dataset_size_limit( + default_dataset_config=default_dataset_config, + override_scope=instance.get_dataset_size_limit_override_scope(), + ) return ScenarioMetadata( class_name=cls.__name__, @@ -190,11 +198,46 @@ def _build_metadata(self, name: str, cls: type[Scenario]) -> ScenarioMetadata: aggregate_techniques=aggregate_techniques, aggregate_technique_expansions=aggregate_technique_expansions, default_datasets=default_datasets, + dataset_size_limit=dataset_size_limit, supported_parameters=supported_parameters, baseline_policy=instance.BASELINE_ATTACK_POLICY.value, include_baseline_by_default=instance.BASELINE_ATTACK_POLICY.value == "enabled", ) + @staticmethod + def _build_dataset_size_limit( + *, + default_dataset_config: Any, + override_scope: Literal["per_dataset", "combined", "unsupported"], + ) -> ScenarioDatasetSizeLimit: + """ + Normalize a scenario dataset configuration into form-level limit semantics. + + Returns: + ScenarioDatasetSizeLimit: The structured default and override scopes. + """ + dataset_names = tuple(default_dataset_config.dataset_names) + caps_by_dataset = default_dataset_config.size_caps_by_dataset() + configured_caps = [caps for caps in caps_by_dataset.values() if caps] + if not configured_caps: + return ScenarioDatasetSizeLimit(default_scope="none", override_scope=override_scope) + + expected_source_count = max(1, len(set(dataset_names))) + if all(len(caps) == 1 for caps in configured_caps) and len(configured_caps) == expected_source_count: + cap_signatures = {(caps[0][1], caps[0][2]) for caps in configured_caps} + if len(cap_signatures) == 1: + count, configured_on = next(iter(cap_signatures)) + default_scope: Literal["per_dataset", "combined"] = ( + "per_dataset" if configured_on == "dataset" else "combined" + ) + return ScenarioDatasetSizeLimit( + default_scope=default_scope, + default_count=count, + override_scope=override_scope, + ) + + return ScenarioDatasetSizeLimit(default_scope="heterogeneous", override_scope=override_scope) + async def create_and_estimate_async( self, *, diff --git a/pyrit/scenario/core/dataset_configuration.py b/pyrit/scenario/core/dataset_configuration.py index 4f43d5fdc6..4e8261b001 100644 --- a/pyrit/scenario/core/dataset_configuration.py +++ b/pyrit/scenario/core/dataset_configuration.py @@ -338,9 +338,10 @@ def __init__( self._dataset_names = list(dataset_names) if dataset_names is not None else None self.max_dataset_size = max_dataset_size self._filters: dict[str, list[str]] = dict(filters or {}) + self._custom_validators = list(validators) if validators else [] self._validators: list[Callable[[ResolvedDataset], None]] = [ *self._default_validators(), - *(list(validators) if validators else []), + *self._custom_validators, ] self._auto_fetch = auto_fetch @@ -742,6 +743,28 @@ async def get_attack_groups_by_dataset_async( raise DatasetConstraintError(f"Resolved attack-group dataset is empty (datasets: {names}).") return result + async def resolve_attack_groups_for_estimate_async( + self, + ) -> tuple[dict[str, list[AttackSeedGroup]], dict[str, list[AttackSeedGroup]]]: + """ + Resolve full and sampled attack groups with one dataset fetch. + + Returns: + tuple: Full groups and effectively selected groups, both keyed by dataset. + + Raises: + DatasetConstraintError: If the resolved or sampled attack-group population is empty. + """ + groups_by_dataset, resolved = await self._build_groups_by_dataset_async() + self.validate(resolved) + selected = { + name: groups for name, groups in self._sample_groups_by_dataset(groups_by_dataset).items() if groups + } + if not groups_by_dataset or not selected: + names = ", ".join(self._dataset_names) if self._dataset_names else "" + raise DatasetConstraintError(f"Resolved attack-group dataset is empty (datasets: {names}).") + return groups_by_dataset, selected + def _sample_groups_by_dataset( self, groups_by_dataset: dict[str, list[AttackSeedGroup]] ) -> dict[str, list[AttackSeedGroup]]: @@ -849,6 +872,75 @@ def per_dataset( ] ) + def with_dataset_names( + self, + *, + dataset_names: Sequence[str], + max_dataset_size: int | None = None, + filters: dict[str, list[str]] | None = None, + ) -> CompoundDatasetAttackConfiguration: + """ + Rebuild a homogeneous per-dataset compound for an explicit name selection. + + This preserves the scenario's per-dataset cap, auto-fetch policy, and shared + filters when every child is a plain single-dataset attack configuration. + Heterogeneous compounds must provide their own scenario-specific override + path rather than silently losing child shaping behavior. + + Args: + dataset_names (Sequence[str]): Selected dataset names in request order. + max_dataset_size (int | None): Optional replacement per-dataset cap. + filters (dict[str, list[str]] | None): Filters merged over the shared defaults. + + Returns: + CompoundDatasetAttackConfiguration: A fresh compound for the selected datasets. + + Raises: + TypeError: If the compound has heterogeneous or shaped child configurations. + ValueError: If ``dataset_names`` is empty or contains duplicates. + """ + if len(set(dataset_names)) != len(dataset_names): + raise ValueError("dataset-name overrides cannot contain duplicates") + if any( + type(child) is not DatasetAttackConfiguration or len(child.dataset_names) != 1 + for child in self._configurations + ): + raise TypeError( + "dataset-name overrides require homogeneous single-dataset DatasetAttackConfiguration children" + ) + + child_caps = {child.max_dataset_size for child in self._configurations} + child_auto_fetch = {child._auto_fetch for child in self._configurations} + child_filters = { + tuple(sorted((key, tuple(values)) for key, values in child.filters.items())) + for child in self._configurations + } + template_child = self._configurations[0] + child_validators_match = all( + child._custom_validators == template_child._custom_validators for child in self._configurations[1:] + ) + if len(child_caps) != 1 or len(child_auto_fetch) != 1 or len(child_filters) != 1 or not child_validators_match: + raise TypeError( + "dataset-name overrides require children with shared caps, filters, validators, and auto-fetch policy" + ) + + inherited_filters = {key: list(values) for key, values in next(iter(child_filters))} + inherited_filters.update(filters or {}) + per_dataset_cap = max_dataset_size if max_dataset_size is not None else next(iter(child_caps)) + rebuilt = type(self).per_dataset( + dataset_names=dataset_names, + max_dataset_size=per_dataset_cap, + auto_fetch=next(iter(child_auto_fetch)), + filters=inherited_filters or None, + ) + for child in rebuilt._configurations: + child._custom_validators = list(template_child._custom_validators) + child._validators = list(template_child._validators) + rebuilt.max_dataset_size = self.max_dataset_size + rebuilt._custom_validators = list(self._custom_validators) + rebuilt._validators = list(self._validators) + return rebuilt + @property def dataset_names(self) -> list[str]: """ @@ -911,6 +1003,21 @@ def update_filters(self, *, filters: dict[str, list[str]]) -> None: for child in self._configurations: child.update_filters(filters=filters) + def update_child_max_dataset_size(self, *, max_dataset_size: int) -> None: + """ + Apply the same independent sampling cap to every child configuration. + + Args: + max_dataset_size (int): Positive per-child logical-group cap. + + Raises: + ValueError: If ``max_dataset_size`` is less than one. + """ + if max_dataset_size < 1: + raise ValueError("'max_dataset_size' must be a positive integer (>= 1).") + for child in self._configurations: + child.max_dataset_size = max_dataset_size + async def get_attack_seed_groups_async(self, *, apply_sampling: bool = True) -> list[AttackSeedGroup]: """ Concatenate every child's flat result, then validate and apply the global cap. @@ -961,6 +1068,27 @@ async def get_attack_groups_by_dataset_async( self.validate(self._resolved_from_groups([group for groups in merged.values() for group in groups])) return self._sample_groups_by_dataset(merged) if apply_sampling else merged + async def resolve_attack_groups_for_estimate_async( + self, + ) -> tuple[dict[str, list[AttackSeedGroup]], dict[str, list[AttackSeedGroup]]]: + """ + Resolve every child's full and sampled populations with one fetch per child. + + Returns: + tuple: Full groups and effectively selected groups, both keyed by dataset. + """ + full_merged: dict[str, list[AttackSeedGroup]] = {} + selected_merged: dict[str, list[AttackSeedGroup]] = {} + for child in self._configurations: + full_groups, selected_groups = await child.resolve_attack_groups_for_estimate_async() + for name, groups in full_groups.items(): + full_merged.setdefault(name, []).extend(groups) + for name, groups in selected_groups.items(): + selected_merged.setdefault(name, []).extend(groups) + self.validate(self._resolved_from_groups([group for groups in full_merged.values() for group in groups])) + selected = {name: groups for name, groups in self._sample_groups_by_dataset(selected_merged).items() if groups} + return full_merged, selected + def _resolved_from_groups(self, groups: list[AttackSeedGroup]) -> ResolvedDataset: """ Build a ResolvedDataset over the combined groups for compound-level validation. diff --git a/pyrit/scenario/core/scenario.py b/pyrit/scenario/core/scenario.py index 0f94633de7..116a19e1e0 100644 --- a/pyrit/scenario/core/scenario.py +++ b/pyrit/scenario/core/scenario.py @@ -733,9 +733,12 @@ async def _resolve_dataset_groups_for_estimate_async( configured_dataset = self._dataset_config with read_only_dataset_resolution(): self._dataset_config = configured_dataset - full_groups = await self._resolve_seed_groups_by_dataset_async(apply_sampling=False) - self._dataset_config = configured_dataset - selected_groups = await self._resolve_seed_groups_by_dataset_async(apply_sampling=True) + if type(self)._resolve_seed_groups_by_dataset_async is Scenario._resolve_seed_groups_by_dataset_async: + full_groups, selected_groups = await configured_dataset.resolve_attack_groups_for_estimate_async() + else: + full_groups = await self._resolve_seed_groups_by_dataset_async(apply_sampling=False) + self._dataset_config = configured_dataset + selected_groups = await self._resolve_seed_groups_by_dataset_async(apply_sampling=True) configured_caps = self._dataset_config.size_caps_by_dataset() datasets: list[ScenarioDatasetSummary] = [] diff --git a/pyrit/scenario/scenarios/adaptive/__init__.py b/pyrit/scenario/scenarios/adaptive/__init__.py index 4be199024c..21c175682e 100644 --- a/pyrit/scenario/scenarios/adaptive/__init__.py +++ b/pyrit/scenario/scenarios/adaptive/__init__.py @@ -6,15 +6,21 @@ from pyrit.scenario.scenarios.adaptive.adaptive_scenario import AdaptiveScenario from pyrit.scenario.scenarios.adaptive.dispatcher import ( ADAPTIVE_ATTEMPT_LABEL, + ADAPTIVE_TECHNIQUE_ID_LABEL, + ADAPTIVE_TECHNIQUE_NAME_LABEL, AdaptiveTechniqueDispatcher, TechniqueBundle, ) from pyrit.scenario.scenarios.adaptive.selectors import EpsilonGreedyTechniqueSelector, SelectorScope, TechniqueSelector +from pyrit.scenario.scenarios.adaptive.technique_identity import AdaptiveTechniqueIdentifier from pyrit.scenario.scenarios.adaptive.text_adaptive import TextAdaptive __all__ = [ "ADAPTIVE_ATTEMPT_LABEL", + "ADAPTIVE_TECHNIQUE_ID_LABEL", + "ADAPTIVE_TECHNIQUE_NAME_LABEL", "AdaptiveScenario", + "AdaptiveTechniqueIdentifier", "AdaptiveTechniqueDispatcher", "EpsilonGreedyTechniqueSelector", "SelectorScope", diff --git a/pyrit/scenario/scenarios/adaptive/adaptive_scenario.py b/pyrit/scenario/scenarios/adaptive/adaptive_scenario.py index 8cb393b483..ff5ffec332 100644 --- a/pyrit/scenario/scenarios/adaptive/adaptive_scenario.py +++ b/pyrit/scenario/scenarios/adaptive/adaptive_scenario.py @@ -23,12 +23,14 @@ from pyrit.common.utils import to_sha256 from pyrit.executor.attack import AttackScoringConfig from pyrit.models import ( + AtomicAttackEvaluationIdentifier, + AtomicAttackIdentifier, + ScenarioAdaptiveRunSizeDetails, ScenarioDefaultRunSizeEstimate, ScenarioRunSizeComponent, ScenarioRunSizeEstimateStatus, ScenarioRunSizeFactor, ) -from pyrit.models.identifiers import compute_inner_attack_eval_hash from pyrit.scenario.core.atomic_attack import AtomicAttack from pyrit.scenario.core.attack_technique import AttackTechnique from pyrit.scenario.core.matrix_atomic_attack_builder import build_baseline_atomic_attack @@ -36,6 +38,7 @@ from pyrit.scenario.core.scenario_target_defaults import get_default_adversarial_target from pyrit.scenario.scenarios.adaptive.dispatcher import AdaptiveTechniqueDispatcher, TechniqueBundle from pyrit.scenario.scenarios.adaptive.selectors import EpsilonGreedyTechniqueSelector, TechniqueSelector +from pyrit.scenario.scenarios.adaptive.technique_identity import AdaptiveTechniqueIdentifier if TYPE_CHECKING: from pyrit.models import AttackSeedGroup @@ -209,16 +212,24 @@ async def _estimate_run_size_async(self) -> ScenarioDefaultRunSizeEstimate: Returns: ScenarioDefaultRunSizeEstimate: The adaptive outer-envelope estimate. + + Raises: + ValueError: If ``max_attempts_per_objective`` is less than one. """ selected_groups, datasets = await self._resolve_dataset_groups_for_estimate_async() selected_count = sum(len(groups) for groups in selected_groups.values()) max_attempts = int(self.params.get("max_attempts_per_objective", 3)) + if max_attempts < 1: + raise ValueError(f"max_attempts_per_objective must be >= 1, got {max_attempts}") + selected_candidate_count = len(self._scenario_techniques) + selected_attempt_bound = min(selected_candidate_count, max_attempts) + baseline_count = selected_count if self._include_baseline else 0 baseline_components = ( [ ScenarioRunSizeComponent( label="Baseline", count=selected_count, - factors=[ScenarioRunSizeFactor(label="selected logical seed groups", count=selected_count)], + factors=[ScenarioRunSizeFactor(label="objectives", count=selected_count)], is_baseline=True, ) ] @@ -229,24 +240,35 @@ async def _estimate_run_size_async(self) -> ScenarioDefaultRunSizeEstimate: components = [ *baseline_components, ScenarioRunSizeComponent( - label="Adaptive attack-envelope candidates", + label="Adaptive objectives", count=selected_count, - factors=[ScenarioRunSizeFactor(label="selected logical seed groups", count=selected_count)], + factors=[ScenarioRunSizeFactor(label="objectives", count=selected_count)], ), ] return ScenarioDefaultRunSizeEstimate( status=ScenarioRunSizeEstimateStatus.Conditional, + minimum_attack_count=baseline_count, + maximum_attack_count=baseline_count + selected_count, components=components, datasets=datasets, + adaptive_details=ScenarioAdaptiveRunSizeDetails( + objective_count=selected_count, + selected_candidate_technique_count=selected_candidate_count, + candidate_technique_count=selected_candidate_count, + max_attempts_per_objective=max_attempts, + techniques_per_objective_upper_bound=selected_attempt_bound, + technique_attempt_count_upper_bound=selected_count * selected_attempt_bound, + ), note=( - "The authoritative total depends on which selected techniques are compatible with the " - f"configured objective target and each seed group. Up to {max_attempts} inner attempts per " - "envelope and retries are excluded." + "The planned-attack total depends on which selected techniques are compatible with the " + f"configured objective target. Up to {selected_attempt_bound} selected technique attempts " + "may run per adaptive objective; retries are excluded." ), ) assert self._objective_target is not None techniques = self._build_techniques_dict(objective_target=self._objective_target) + candidate_count = len(techniques) dispatcher = AdaptiveTechniqueDispatcher( objective_target=self._objective_target, techniques=techniques, @@ -264,9 +286,9 @@ async def _estimate_run_size_async(self) -> ScenarioDefaultRunSizeEstimate: components = [ *baseline_components, ScenarioRunSizeComponent( - label="Adaptive attack envelopes", + label="Adaptive objectives", count=compatible_group_count, - factors=[ScenarioRunSizeFactor(label="compatible logical seed groups", count=compatible_group_count)], + factors=[ScenarioRunSizeFactor(label="compatible objectives", count=compatible_group_count)], ), ] status = ( @@ -274,22 +296,42 @@ async def _estimate_run_size_async(self) -> ScenarioDefaultRunSizeEstimate: if self._estimate_has_binding_size_cap else ScenarioRunSizeEstimateStatus.Exact ) + adaptive_objective_bound = ( + selected_count if status is ScenarioRunSizeEstimateStatus.Conditional else compatible_group_count + ) total_attack_count = ( None if status is ScenarioRunSizeEstimateStatus.Conditional else sum(component.count for component in components) ) + minimum_attack_count = ( + baseline_count if status is ScenarioRunSizeEstimateStatus.Conditional and baseline_count > 0 else None + ) + maximum_attack_count = ( + baseline_count + selected_count if status is ScenarioRunSizeEstimateStatus.Conditional else None + ) + technique_attempt_bound = min(candidate_count, max_attempts) note = ( - f"Each planned unit is one persisted adaptive envelope. Up to {max_attempts} selected technique " - "attempts may run inside that unit; inner attempts and retries are excluded." + f"Each compatible adaptive objective is one planned attack. Up to {technique_attempt_bound} selected " + "technique attempts may run for that objective; retries are excluded." ) if status is ScenarioRunSizeEstimateStatus.Conditional: note += " A binding randomized dataset cap may select a different compatibility mix at launch." return ScenarioDefaultRunSizeEstimate( status=status, total_attack_count=total_attack_count, + minimum_attack_count=minimum_attack_count, + maximum_attack_count=maximum_attack_count, components=components, datasets=datasets, + adaptive_details=ScenarioAdaptiveRunSizeDetails( + objective_count=adaptive_objective_bound, + selected_candidate_technique_count=selected_candidate_count, + candidate_technique_count=candidate_count, + max_attempts_per_objective=max_attempts, + techniques_per_objective_upper_bound=technique_attempt_bound, + technique_attempt_count_upper_bound=adaptive_objective_bound * technique_attempt_bound, + ), note=note, ) @@ -299,19 +341,17 @@ def _build_techniques_dict( objective_target: PromptTarget, ) -> dict[str, TechniqueBundle]: """ - Resolve selected techniques into a ``{eval_hash: TechniqueBundle}`` map. + Resolve selected techniques into a ``{adaptive_id: TechniqueBundle}`` map. Each bundle carries the inner attack technique along with the factory's ``seed_technique`` and ``adversarial_chat`` so the dispatcher can reproduce the static ``AtomicAttack`` execution path per attempt. - Technique keys are eval hashes derived from the inner attack technique's - identifier (run through ``AtomicAttackEvaluationIdentifier`` so seeds, - scorers, and operational target params are excluded). The same hash is - auto-stamped on every persisted ``AttackResultEntry.atomic_attack_identifier`` - by the executor, which lets the selector aggregate historical success - rates by behavioral configuration via - ``MemoryInterface.get_attack_results(atomic_attack_eval_hashes=...)``. + Technique keys join the canonical factory identifier hash with the full + ``AttackTechnique`` eval hash. Factory identity keeps distinct registered + configurations as separate arms even when they share an inner attack; + technique eval identity links those arms to normal-scenario history. + The dispatcher persists this joined identity on each child result. For factories whose attack class narrows ``attack_scoring_config`` to a specific subtype (e.g. ``TAPAttackScoringConfig`` for TAP), this method @@ -321,7 +361,7 @@ def _build_techniques_dict( are dropped with a warning so the rest of the pool continues to run. Returns: - dict[str, TechniqueBundle]: Mapping from technique eval hash to its + dict[str, TechniqueBundle]: Mapping from joined Adaptive identity to its bundle, in the order selected techniques were resolved. Raises: @@ -357,11 +397,17 @@ def _build_techniques_dict( skipped_incompatible[technique_name] = str(exc) logger.warning(f"Skipping technique '{technique_name}': {type(exc).__name__}: {exc}") continue - eval_hash = compute_inner_attack_eval_hash(attack=technique.attack) + technique_eval_hash = AtomicAttackEvaluationIdentifier( + AtomicAttackIdentifier.build(technique_identifier=technique.get_identifier()) + ).eval_hash + technique_identifier = AdaptiveTechniqueIdentifier( + factory_hash=factory.get_identifier().hash, + technique_eval_hash=technique_eval_hash, + ).serialize() adversarial_chat = factory.adversarial_chat if adversarial_chat is None and factory.uses_adversarial: adversarial_chat = get_default_adversarial_target() - techniques[eval_hash] = TechniqueBundle( + techniques[technique_identifier] = TechniqueBundle( attack=technique.attack, name=technique_name, seed_technique=technique.seed_technique, diff --git a/pyrit/scenario/scenarios/adaptive/dispatcher.py b/pyrit/scenario/scenarios/adaptive/dispatcher.py index ba4bc31f18..bafe0509b0 100644 --- a/pyrit/scenario/scenarios/adaptive/dispatcher.py +++ b/pyrit/scenario/scenarios/adaptive/dispatcher.py @@ -12,15 +12,10 @@ hands them to the scenario base for execution. The returned attack is a plain ``SequentialAttack`` with -``SequenceCompletionPolicy.FIRST_SUCCESS``. The per-attempt dispatch trail -(which technique ran, with what outcome, in what order) is not stamped onto -the envelope — every child ``AttackResult`` in -``SequentialAttackResult.child_attack_results`` already carries its own -``outcome`` and its own ``atomic_attack_identifier.eval_hash``. Callers that -want a human-readable technique label per child read it directly from the -child via ``child.get_attack_strategy_identifier().unique_name`` (the -executor auto-stamps ``class_name`` and ``unique_name`` on every persisted -row), so there is no separate ``{eval_hash: name}`` map to consult. +``SequenceCompletionPolicy.FIRST_SUCCESS``. Each child result carries the +stable registered-factory identity and friendly technique name in labels so +different configured techniques remain attributable even when their inner +attack implementations are execution-equivalent. """ from __future__ import annotations @@ -50,6 +45,12 @@ ADAPTIVE_ATTEMPT_LABEL: str = "_adaptive_attempt" """1-based attempt index within the per-objective loop.""" +ADAPTIVE_TECHNIQUE_ID_LABEL: str = "_adaptive_technique_id" +"""Joined registered-factory and behavioral-history identity for the selected arm.""" + +ADAPTIVE_TECHNIQUE_NAME_LABEL: str = "_adaptive_technique_name" +"""Registered technique name for human-readable result attribution.""" + @dataclass(frozen=True) class TechniqueBundle: @@ -58,16 +59,8 @@ class TechniqueBundle: Carries the inner attack strategy alongside the factory-supplied ``seed_technique`` (if any) and ``adversarial_chat`` (required when the - seed_technique contains a simulated-conversation config). ``name`` is the - factory-registration key; the dispatcher does not consume it, but it is - convenient for diagnostics and is preserved here so callers/tests can - cross-check which factory each bundle came from. - - Notebook/report code that wants a human-readable label for a persisted - child ``AttackResult`` should read it from the child itself via - ``child.get_attack_strategy_identifier()`` — the executor already stamps - ``class_name`` and ``unique_name`` on every row, so there is no need to - publish a separate ``{eval_hash: name}`` map. + seed technique contains a simulated-conversation config). ``name`` is the + factory-registration key and is persisted on every selected child result. """ attack: AttackStrategy[Any, AttackResult] @@ -110,7 +103,7 @@ def __init__( Args: objective_target (PromptTarget): The target inner attacks run against. techniques (dict[str, TechniqueBundle]): Mapping from - technique eval hash to its bundle. Must be non-empty. + joined Adaptive technique identity to its bundle. Must be non-empty. selector (TechniqueSelector): Stateless technique selector. objective_scorer (TrueFalseScorer | None): Scorer forwarded to inner attacks that generate simulated conversations. @@ -137,14 +130,14 @@ def __init__( def compatible_techniques(self, *, seed_group: AttackSeedGroup) -> list[str]: """ - Return technique hashes whose ``seed_technique`` is compatible with ``seed_group``. + Return technique identifiers whose ``seed_technique`` is compatible with ``seed_group``. Techniques with no ``seed_technique`` are universally compatible. Used by ``AdaptiveScenario`` to drop seed groups with no usable techniques before building atomic attacks. Returns: - list[str]: Technique eval hashes in declaration order. + list[str]: Joined Adaptive technique identities in declaration order. """ return [ name @@ -179,12 +172,8 @@ async def build_attack_async( technique map. Returns: - SequentialAttack: The ready-to-run attack. Each child's - identity is captured by its own - ``atomic_attack_identifier.eval_hash`` after execution; - callers wanting the friendly technique name read it - directly from the child via - ``child.get_attack_strategy_identifier().unique_name``. + SequentialAttack: The ready-to-run attack. Each child carries its + canonical factory identity and registered name in result labels. Raises: ValueError: If ``seed_group.objective`` is not initialized, @@ -202,7 +191,7 @@ async def build_attack_async( f"(objective={seed_group.objective.value!r})." ) - chosen_hashes = await self._selector.select_async( + chosen_identifiers = await self._selector.select_async( technique_identifiers=compatible, objective=seed_group.objective.value, num_top_techniques=self._max_attempts, @@ -210,7 +199,7 @@ async def build_attack_async( ) child_attacks: list[SequentialChildAttack] = [] - for attempt_idx, chosen in enumerate(chosen_hashes): + for attempt_idx, chosen in enumerate(chosen_identifiers): bundle = self._techniques[chosen] execution_group = ( seed_group.with_technique(technique=bundle.seed_technique) @@ -223,7 +212,11 @@ async def build_attack_async( seed_group=execution_group, adversarial_chat=bundle.adversarial_chat, objective_scorer=self._objective_scorer, - memory_labels={ADAPTIVE_ATTEMPT_LABEL: str(attempt_idx + 1)}, + memory_labels={ + ADAPTIVE_ATTEMPT_LABEL: str(attempt_idx + 1), + ADAPTIVE_TECHNIQUE_ID_LABEL: chosen, + ADAPTIVE_TECHNIQUE_NAME_LABEL: bundle.name, + }, ) ) diff --git a/pyrit/scenario/scenarios/adaptive/selectors/epsilon_greedy.py b/pyrit/scenario/scenarios/adaptive/selectors/epsilon_greedy.py index a415182733..3fd2b000e9 100644 --- a/pyrit/scenario/scenarios/adaptive/selectors/epsilon_greedy.py +++ b/pyrit/scenario/scenarios/adaptive/selectors/epsilon_greedy.py @@ -11,8 +11,10 @@ import struct from typing import TYPE_CHECKING -from pyrit.analytics.technique_analysis import compute_technique_stats +from pyrit.analytics.technique_analysis import compute_labeled_technique_stats +from pyrit.scenario.scenarios.adaptive.dispatcher import ADAPTIVE_TECHNIQUE_ID_LABEL from pyrit.scenario.scenarios.adaptive.selectors.technique_selector import SelectorScope +from pyrit.scenario.scenarios.adaptive.technique_identity import get_history_eval_hash if TYPE_CHECKING: from collections.abc import Sequence @@ -129,8 +131,13 @@ async def select_async( rng = _derive_rng(self._seed, decision_key) effective_run_id = scenario_result_id if self._scope.current_run_only else None - stats = compute_technique_stats( - technique_eval_hashes=technique_list, + stats = compute_labeled_technique_stats( + technique_identifiers=technique_list, + label_name=ADAPTIVE_TECHNIQUE_ID_LABEL, + technique_eval_hashes_by_identifier={ + technique_identifier: get_history_eval_hash(technique_identifier=technique_identifier) + for technique_identifier in technique_list + }, scenario_result_id=effective_run_id, targeted_harm_categories=self._scope.targeted_harm_categories, ) diff --git a/pyrit/scenario/scenarios/adaptive/selectors/technique_selector.py b/pyrit/scenario/scenarios/adaptive/selectors/technique_selector.py index 0161e4923b..4ae383df55 100644 --- a/pyrit/scenario/scenarios/adaptive/selectors/technique_selector.py +++ b/pyrit/scenario/scenarios/adaptive/selectors/technique_selector.py @@ -20,7 +20,7 @@ class SelectorScope: All fields default to "no restriction"; combine fields to narrow the scope (e.g. current run only, same harm category). Filter values flow - through ``compute_technique_stats`` to + through labeled technique statistics to ``MemoryInterface.get_attack_results``. The scope is held by the selector at construction time. The per-call @@ -28,10 +28,9 @@ class SelectorScope: to memory only when ``current_run_only`` is set; otherwise the selector queries across all runs. - Per-technique disambiguation uses ``atomic_attack_identifier.eval_hash`` - (auto-stamped on every persisted attack result), which already encodes - the attack class plus its behavior-relevant params. Class-based - narrowing is therefore unnecessary at this layer. + Per-technique disambiguation uses the canonical registered factory + identifier persisted on every Adaptive child result. This keeps distinct + registered configurations separate even when they share an attack class. """ current_run_only: bool = False @@ -86,8 +85,8 @@ async def select_async( Return techniques in priority order (try first, try second, …). Args: - technique_identifiers (Sequence[str]): Available technique eval - hashes. + technique_identifiers (Sequence[str]): Available stable Adaptive + technique identities. objective (str): The objective text for this selection. num_top_techniques (int): Max techniques to return. Defaults to 1. scenario_result_id (str | None): The current scenario run ID, @@ -96,7 +95,7 @@ async def select_async( ``current_run_only=True``. Returns: - Sequence[str]: Up to ``num_top_techniques`` technique eval hashes + Sequence[str]: Up to ``num_top_techniques`` technique identities in priority order. Fewer if not enough techniques are available. """ diff --git a/pyrit/scenario/scenarios/adaptive/technique_identity.py b/pyrit/scenario/scenarios/adaptive/technique_identity.py new file mode 100644 index 0000000000..950d7358b7 --- /dev/null +++ b/pyrit/scenario/scenarios/adaptive/technique_identity.py @@ -0,0 +1,72 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Stable identity carried by Adaptive selector arms.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import ClassVar + + +@dataclass(frozen=True) +class AdaptiveTechniqueIdentifier: + """ + Join registered factory identity with cross-scenario behavioral identity. + + The factory hash keeps separately registered configured techniques as + distinct selector arms. The technique eval hash links each arm to normal + scenario results persisted from the same ``AttackTechnique`` behavior. + """ + + factory_hash: str + technique_eval_hash: str + + _PREFIX: ClassVar[str] = "adaptive-v1" + _SEPARATOR: ClassVar[str] = ":" + + def serialize(self) -> str: + """ + Serialize the identifier for selector keys and persisted labels. + + Returns: + str: Versioned identifier containing both canonical hashes. + + Raises: + ValueError: If either hash is empty or contains the field separator. + """ + if self._SEPARATOR in self.factory_hash or self._SEPARATOR in self.technique_eval_hash: + raise ValueError("Adaptive technique identity hashes cannot contain ':'") + if not self.factory_hash or not self.technique_eval_hash: + raise ValueError("Adaptive technique identity hashes cannot be empty") + return self._SEPARATOR.join((self._PREFIX, self.factory_hash, self.technique_eval_hash)) + + @classmethod + def parse(cls, value: str) -> AdaptiveTechniqueIdentifier | None: + """ + Parse a serialized Adaptive identifier. + + Unknown selector identifiers remain valid for custom selectors and + legacy tests, so malformed or unversioned values return ``None``. + + Returns: + AdaptiveTechniqueIdentifier | None: Parsed identity when recognized. + """ + parts = value.split(cls._SEPARATOR) + if len(parts) != 3 or parts[0] != cls._PREFIX or not parts[1] or not parts[2]: + return None + return cls(factory_hash=parts[1], technique_eval_hash=parts[2]) + + +def get_history_eval_hash(*, technique_identifier: str) -> str: + """ + Return the normal-scenario eval hash associated with a selector arm. + + Unversioned identifiers fall back to themselves for backward compatibility + with custom selectors and callers that already pass eval hashes. + + Returns: + str: Behavioral eval hash used for historical result lookup. + """ + parsed = AdaptiveTechniqueIdentifier.parse(technique_identifier) + return parsed.technique_eval_hash if parsed is not None else technique_identifier diff --git a/pyrit/scenario/scenarios/adaptive/text_adaptive.py b/pyrit/scenario/scenarios/adaptive/text_adaptive.py index ff31ed29d8..87df590446 100644 --- a/pyrit/scenario/scenarios/adaptive/text_adaptive.py +++ b/pyrit/scenario/scenarios/adaptive/text_adaptive.py @@ -128,7 +128,10 @@ def additional_parameters(cls) -> list[Parameter]: return [ Parameter( name="max_attempts_per_objective", - description="Max techniques tried per objective. Defaults to 3.", + description=( + "Maximum different compatible techniques Adaptive may try for one objective, stopping after " + "the first success. This is separate from retries." + ), param_type=int, default=3, ), diff --git a/pyrit/scenario/scenarios/airt/jailbreak.py b/pyrit/scenario/scenarios/airt/jailbreak.py index 72169176f0..66aa61e71d 100644 --- a/pyrit/scenario/scenarios/airt/jailbreak.py +++ b/pyrit/scenario/scenarios/airt/jailbreak.py @@ -17,6 +17,7 @@ Parameter, ScenarioDefaultRunSizeEstimate, ScenarioRunSizeComponent, + ScenarioRunSizeEstimateCondition, ScenarioRunSizeEstimateStatus, ScenarioRunSizeFactor, ) @@ -137,9 +138,16 @@ def _build_jailbreak_technique() -> type[ScenarioTechnique]: type[ScenarioTechnique]: The dynamically generated technique enum class. """ registry = AttackTechniqueRegistry.get_registry_singleton() - registered = [ - factory for factory in registry.get_factories_or_raise().values() if _is_jailbreak_compatible_factory(factory) - ] + registry_factories = list(registry.get_factories_or_raise().values()) + excluded_without_converter_composition = sorted( + factory.name for factory in registry_factories if not factory.supports_request_converter_composition + ) + if excluded_without_converter_composition: + logger.warning( + "Jailbreak excluded attack technique factories that cannot compose the required request converter: %s", + ", ".join(excluded_without_converter_composition), + ) + registered = [factory for factory in registry_factories if _is_jailbreak_compatible_factory(factory)] factories = registered + list(_extra_default_factories().values()) return AttackTechniqueRegistry.build_technique_class_from_factories( # type: ignore[return-value, ty:invalid-return-type] class_name="JailbreakTechnique", @@ -409,6 +417,7 @@ async def _estimate_run_size_async(self) -> ScenarioDefaultRunSizeEstimate: ScenarioRunSizeFactor(label="jailbreak templates", count=template_count), ScenarioRunSizeFactor(label="attempts", count=attempt_count), ], + condition=ScenarioRunSizeEstimateCondition.TargetCapabilities, note=( "The selected objective target supports native system-prompt delivery." if system_delivery_supported is True @@ -450,6 +459,15 @@ async def _estimate_run_size_async(self) -> ScenarioDefaultRunSizeEstimate: return ScenarioDefaultRunSizeEstimate( status=status, total_attack_count=planned_count if status is ScenarioRunSizeEstimateStatus.Exact else None, + minimum_attack_count=( + target_agnostic_count if status is ScenarioRunSizeEstimateStatus.Conditional else None + ), + maximum_attack_count=planned_count if status is ScenarioRunSizeEstimateStatus.Conditional else None, + condition=( + ScenarioRunSizeEstimateCondition.TargetCapabilities + if status is ScenarioRunSizeEstimateStatus.Conditional + else None + ), components=components, datasets=datasets, note=f"{formula}{baseline_explanation}{capability_note}", diff --git a/pyrit/scenario/scenarios/airt/psychosocial.py b/pyrit/scenario/scenarios/airt/psychosocial.py index 43aa58ff73..fb0afeb4e8 100644 --- a/pyrit/scenario/scenarios/airt/psychosocial.py +++ b/pyrit/scenario/scenarios/airt/psychosocial.py @@ -6,7 +6,7 @@ import logging import pathlib from dataclasses import dataclass -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, ClassVar, Literal, cast from pyrit.common import apply_defaults from pyrit.common.path import DATASETS_PATH @@ -345,6 +345,9 @@ class Psychosocial(Scenario): """ VERSION: int = 3 + DATASET_SIZE_LIMIT_OVERRIDE_SCOPE: ClassVar[Literal["per_dataset", "combined", "unsupported"] | None] = ( + "per_dataset" + ) @classmethod def additional_parameters(cls) -> list[Parameter]: diff --git a/pyrit/scenario/scenarios/garak/web_injection.py b/pyrit/scenario/scenarios/garak/web_injection.py index d081b98b6b..35ae7190af 100644 --- a/pyrit/scenario/scenarios/garak/web_injection.py +++ b/pyrit/scenario/scenarios/garak/web_injection.py @@ -6,7 +6,7 @@ import asyncio import logging import random -from typing import TYPE_CHECKING, ClassVar, cast +from typing import TYPE_CHECKING, ClassVar, Literal, cast from pyrit.common import apply_defaults from pyrit.executor.attack.core.attack_config import AttackScoringConfig @@ -99,6 +99,9 @@ class WebInjection(Scenario): VERSION: int = 1 BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Enabled + DATASET_SIZE_LIMIT_OVERRIDE_SCOPE: ClassVar[Literal["per_dataset", "combined", "unsupported"] | None] = ( + "unsupported" + ) # Local ``.prompt`` datasets under datasets/seed_datasets/local/garak. DATASET_EXAMPLE_DOMAINS: ClassVar[str] = "garak_example_domains_xss" diff --git a/tests/unit/analytics/test_technique_analysis.py b/tests/unit/analytics/test_technique_analysis.py index 04b1d94890..c31a1cd0ea 100644 --- a/tests/unit/analytics/test_technique_analysis.py +++ b/tests/unit/analytics/test_technique_analysis.py @@ -2,15 +2,17 @@ # Licensed under the MIT license. from unittest.mock import MagicMock, patch +from uuid import uuid4 import pytest -from pyrit.analytics.technique_analysis import compute_technique_stats +from pyrit.analytics.technique_analysis import compute_labeled_technique_stats, compute_technique_stats from pyrit.models import AttackOutcome def _make_result(*, eval_hash: str | None, outcome: AttackOutcome) -> MagicMock: r = MagicMock() + r.attack_result_id = str(uuid4()) if eval_hash is None: r.atomic_attack_identifier = None else: @@ -18,9 +20,16 @@ def _make_result(*, eval_hash: str | None, outcome: AttackOutcome) -> MagicMock: identifier.eval_hash = eval_hash r.atomic_attack_identifier = identifier r.outcome = outcome + r.labels = {} return r +def _make_labeled_result(*, label_name: str, technique_identifier: str, outcome: AttackOutcome) -> MagicMock: + result = _make_result(eval_hash=None, outcome=outcome) + result.labels = {label_name: technique_identifier} + return result + + @pytest.fixture(autouse=True) def _patch_memory(): mock_memory = MagicMock() @@ -144,3 +153,77 @@ def test_injected_memory_bypasses_central_memory(self, _patch_memory): injected.get_attack_results.assert_called_once() _patch_memory.get_attack_results.assert_not_called() assert stats["a"].successes == 1 + + +class TestComputeLabeledTechniqueStats: + def test_counts_distinct_labeled_techniques(self, _patch_memory): + label_name = "_adaptive_technique_id" + _patch_memory.get_attack_results.return_value = [ + _make_labeled_result( + label_name=label_name, + technique_identifier="role-play-movie", + outcome=AttackOutcome.SUCCESS, + ), + _make_labeled_result( + label_name=label_name, + technique_identifier="role-play-video", + outcome=AttackOutcome.FAILURE, + ), + ] + + stats = compute_labeled_technique_stats( + technique_identifiers=["role-play-movie", "role-play-video"], + label_name=label_name, + ) + + assert stats["role-play-movie"].successes == 1 + assert stats["role-play-video"].failures == 1 + assert _patch_memory.get_attack_results.call_args.kwargs["labels"] == { + label_name: ["role-play-movie", "role-play-video"] + } + + def test_unlabeled_and_unrequested_results_are_ignored(self, _patch_memory): + label_name = "_adaptive_technique_id" + unlabeled = _make_result(eval_hash="shared", outcome=AttackOutcome.SUCCESS) + _patch_memory.get_attack_results.return_value = [ + unlabeled, + _make_labeled_result( + label_name=label_name, + technique_identifier="other", + outcome=AttackOutcome.SUCCESS, + ), + ] + + stats = compute_labeled_technique_stats( + technique_identifiers=["requested"], + label_name=label_name, + ) + + assert stats == {} + + def test_merges_labeled_and_normal_scenario_history_without_double_counting(self, _patch_memory): + label_name = "_adaptive_technique_id" + labeled = _make_labeled_result( + label_name=label_name, + technique_identifier="factory-arm", + outcome=AttackOutcome.SUCCESS, + ) + labeled.atomic_attack_identifier = MagicMock(eval_hash="inner-attack-hash") + normal = _make_result(eval_hash="full-technique-hash", outcome=AttackOutcome.FAILURE) + _patch_memory.get_attack_results.side_effect = [ + [labeled], + [labeled, normal], + ] + + stats = compute_labeled_technique_stats( + technique_identifiers=["factory-arm"], + label_name=label_name, + technique_eval_hashes_by_identifier={"factory-arm": "full-technique-hash"}, + ) + + assert stats["factory-arm"].successes == 1 + assert stats["factory-arm"].failures == 1 + assert stats["factory-arm"].total_decided == 2 + assert _patch_memory.get_attack_results.call_args_list[1].kwargs["atomic_attack_eval_hashes"] == [ + "full-technique-hash" + ] diff --git a/tests/unit/backend/test_scenario_run_service.py b/tests/unit/backend/test_scenario_run_service.py index 47569bccbd..6ee7b6a5f7 100644 --- a/tests/unit/backend/test_scenario_run_service.py +++ b/tests/unit/backend/test_scenario_run_service.py @@ -44,7 +44,11 @@ config_hash, ) from pyrit.models.catalog.scenario import RunScenarioRequest -from pyrit.scenario.core import DatasetAttackConfiguration, DatasetConfiguration +from pyrit.scenario.core import ( + CompoundDatasetAttackConfiguration, + DatasetAttackConfiguration, + DatasetConfiguration, +) from pyrit.scenario.core.scenario_technique import ScenarioTechnique from unit.mocks import make_scenario_result @@ -583,6 +587,81 @@ class _MarkerDatasetConfiguration(DatasetConfiguration): assert built_config.dataset_names == ["only_this"] assert built_config.max_dataset_size is None + async def test_start_run_dataset_names_rebuilds_homogeneous_compound(self, mock_all_registries) -> None: + """Compound per-dataset defaults support exact selected-name overrides.""" + default_config = CompoundDatasetAttackConfiguration.per_dataset( + dataset_names=["airt_hate", "airt_fairness"], + max_dataset_size=4, + ) + scenario_instance = mock_all_registries["scenario_instance"] + scenario_instance._default_dataset_config = default_config + + service = ScenarioRunService() + await service.start_run_async(request=_make_request(dataset_names=["airt_fairness"])) + + init_call = mock_all_registries["scenario_registry"].create_and_initialize_async.await_args + built_config = init_call.kwargs["dataset_config"] + assert isinstance(built_config, CompoundDatasetAttackConfiguration) + assert built_config.dataset_names == ["airt_fairness"] + assert [child.max_dataset_size for child in built_config._configurations] == [4] + assert default_config.dataset_names == ["airt_hate", "airt_fairness"] + + async def test_start_run_max_dataset_size_updates_each_default_compound_child(self, mock_all_registries) -> None: + """An unchanged default selection keeps compound caps per dataset.""" + default_config = CompoundDatasetAttackConfiguration.per_dataset( + dataset_names=["airt_hate", "airt_fairness"], + max_dataset_size=4, + ) + scenario_instance = mock_all_registries["scenario_instance"] + scenario_instance._default_dataset_config = default_config + + service = ScenarioRunService() + await service.start_run_async(request=_make_request(max_dataset_size=2)) + + init_call = mock_all_registries["scenario_registry"].create_and_initialize_async.await_args + built_config = init_call.kwargs["dataset_config"] + assert isinstance(built_config, CompoundDatasetAttackConfiguration) + assert built_config is default_config + assert built_config.dataset_names == ["airt_hate", "airt_fairness"] + assert [child.max_dataset_size for child in built_config._configurations] == [2, 2] + + async def test_start_run_non_name_overrides_preserve_shaped_compound_children(self, mock_all_registries) -> None: + """Size and filter overrides do not rebuild scenario-specific child configurations.""" + + class _ShapedDatasetConfiguration(DatasetAttackConfiguration): + pass + + default_config = CompoundDatasetAttackConfiguration( + configurations=[ + _ShapedDatasetConfiguration(dataset_names=["d1"], max_dataset_size=4), + _ShapedDatasetConfiguration(dataset_names=["d2"], max_dataset_size=4), + ], + ) + scenario_instance = mock_all_registries["scenario_instance"] + scenario_instance._default_dataset_config = default_config + + service = ScenarioRunService() + await service.start_run_async( + request=_make_request( + dataset_names=["d1", "d2"], + max_dataset_size=2, + dataset_filters={"harm_categories": ["cyber"]}, + ) + ) + + init_call = mock_all_registries["scenario_registry"].create_and_initialize_async.await_args + built_config = init_call.kwargs["dataset_config"] + assert built_config is default_config + assert [type(child) for child in built_config._configurations] == [ + _ShapedDatasetConfiguration, + _ShapedDatasetConfiguration, + ] + assert [child.max_dataset_size for child in built_config._configurations] == [2, 2] + assert [child.filters for child in built_config._configurations] == [ + {"harm_categories": ["cyber"]}, + {"harm_categories": ["cyber"]}, + ] + async def test_start_run_dataset_names_rejects_incompatible_subclass_constructor(self, mock_all_registries) -> None: """Reject overrides that cannot preserve scenario-specific dataset configuration.""" diff --git a/tests/unit/backend/test_scenario_service.py b/tests/unit/backend/test_scenario_service.py index 76a03ef9c9..93d6d77ad4 100644 --- a/tests/unit/backend/test_scenario_service.py +++ b/tests/unit/backend/test_scenario_service.py @@ -7,6 +7,7 @@ import asyncio from collections import OrderedDict +from collections.abc import Awaitable from typing import Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -26,13 +27,14 @@ from pyrit.models import ( Parameter, ScenarioDatasetSizeCap, + ScenarioDatasetSizeLimit, ScenarioDatasetSummary, ScenarioDefaultRunSizeEstimate, ScenarioRunSizeComponent, ScenarioRunSizeEstimateRequest, ScenarioRunSizeEstimateStatus, ) -from pyrit.models.catalog.scenario import RegisteredScenario +from pyrit.models.catalog import RegisteredScenario from pyrit.registry import ScenarioMetadata from pyrit.scenario.core import DatasetAttackConfiguration, ScenarioTechnique @@ -66,6 +68,15 @@ def clear_service_cache(): get_scenario_service.cache_clear() +def _initialize_test_service(service: ScenarioService) -> None: + """Initialize service state without binding the process-wide registry.""" + service._estimate_cache = OrderedDict() + service._estimate_tasks = OrderedDict() + service._estimate_task_lock = asyncio.Lock() + service._estimate_semaphore = asyncio.Semaphore(4) + service._configured_estimate_semaphore = asyncio.Semaphore(4) + + def _make_scenario_metadata( *, registry_name: str = "test.scenario", @@ -84,6 +95,7 @@ def _make_scenario_metadata( default_datasets: tuple[str, ...] = ("test_dataset",), baseline_policy: str = "enabled", include_baseline_by_default: bool = True, + dataset_size_limit: ScenarioDatasetSizeLimit | None = None, ) -> ScenarioMetadata: """Create a ScenarioMetadata instance for testing.""" return ScenarioMetadata( @@ -99,6 +111,7 @@ def _make_scenario_metadata( aggregate_techniques=aggregate_techniques, aggregate_technique_expansions=aggregate_technique_expansions, default_datasets=default_datasets, + dataset_size_limit=dataset_size_limit or ScenarioDatasetSizeLimit(), baseline_policy=baseline_policy, include_baseline_by_default=include_baseline_by_default, ) @@ -114,7 +127,7 @@ class TestScenarioServiceListScenarios: async def test_list_scenarios_returns_empty_when_no_scenarios(self) -> None: """Test that list returns empty list when no scenarios are registered.""" - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_all_registered_class_metadata.return_value = [] @@ -128,7 +141,7 @@ async def test_list_scenarios_returns_scenarios_from_registry(self) -> None: """Test that list returns scenarios from registry.""" metadata = _make_scenario_metadata() - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_all_registered_class_metadata.return_value = [metadata] @@ -146,9 +159,28 @@ async def test_list_scenarios_returns_scenarios_from_registry(self) -> None: assert result.items[0].aggregate_technique_expansions["default"] == ["role_play"] assert result.items[0].all_techniques == ["role_play", "many_shot"] assert result.items[0].default_datasets == ["test_dataset"] + assert result.items[0].dataset_size_limit == ScenarioDatasetSizeLimit() assert result.items[0].baseline_policy == "enabled" assert result.items[0].include_baseline_by_default is True + async def test_list_scenarios_projects_dataset_size_limit_metadata(self) -> None: + """Catalog responses preserve structured scenario-owned limit semantics.""" + limit = ScenarioDatasetSizeLimit( + default_scope="per_dataset", + default_count=4, + override_scope="per_dataset", + ) + metadata = _make_scenario_metadata(dataset_size_limit=limit) + + with patch.object(ScenarioService, "__init__", _initialize_test_service): + service = ScenarioService() + service._registry = MagicMock() + service._registry.get_all_registered_class_metadata.return_value = [metadata] + + result = await service.list_scenarios_async() + + assert result.items[0].dataset_size_limit == limit + async def test_estimate_is_offloaded_and_cached(self) -> None: """Scenario-owned estimates run in a worker once and are reused by subsequent reads.""" metadata = _make_scenario_metadata() @@ -175,7 +207,7 @@ async def test_estimate_is_offloaded_and_cached(self) -> None: scenario = MagicMock() scenario.get_default_run_size_estimate_async = AsyncMock(return_value=estimate) - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_registered_class_metadata.return_value = metadata @@ -211,7 +243,7 @@ async def estimate_async() -> ScenarioDefaultRunSizeEstimate: scenario = MagicMock() scenario.get_default_run_size_estimate_async = AsyncMock(side_effect=estimate_async) - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.create_instance.return_value = scenario @@ -262,7 +294,7 @@ async def estimate_async() -> ScenarioDefaultRunSizeEstimate: scenario = MagicMock() scenario.get_default_run_size_estimate_async = AsyncMock(side_effect=estimate_async) - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.create_instance.return_value = scenario @@ -293,7 +325,7 @@ async def test_completed_stale_task_cannot_block_inflight_capacity(self) -> None scenario.get_default_run_size_estimate_async = AsyncMock(return_value=estimate) with ( - patch.object(ScenarioService, "__init__", lambda self: None), + patch.object(ScenarioService, "__init__", _initialize_test_service), patch("pyrit.backend.services.scenario_service._ESTIMATE_INFLIGHT_SIZE", 1), ): service = ScenarioService() @@ -326,7 +358,7 @@ async def test_one_failed_estimate_does_not_break_catalog(self) -> None: bad_scenario = MagicMock() bad_scenario.get_default_run_size_estimate_async = AsyncMock(side_effect=RuntimeError("dataset unavailable")) - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_all_registered_class_metadata.return_value = metadata @@ -341,6 +373,170 @@ async def test_one_failed_estimate_does_not_break_catalog(self) -> None: assert result.items[1].default_run_size.status is ScenarioRunSizeEstimateStatus.Unavailable assert "RuntimeError" in result.items[1].default_run_size.note + async def test_catalog_estimates_use_bounded_parallelism(self) -> None: + """Catalog cards estimate concurrently without exceeding the configured bound.""" + metadata = [_make_scenario_metadata(registry_name=f"test.scenario_{index}") for index in range(6)] + estimate = ScenarioDefaultRunSizeEstimate( + status=ScenarioRunSizeEstimateStatus.Exact, + total_attack_count=1, + components=[ScenarioRunSizeComponent(label="Default sweep", count=1)], + ) + active = 0 + maximum_active = 0 + + async def estimate_async() -> ScenarioDefaultRunSizeEstimate: + nonlocal active, maximum_active + active += 1 + maximum_active = max(maximum_active, active) + await asyncio.sleep(0.01) + active -= 1 + return estimate + + scenario = MagicMock() + scenario.get_default_run_size_estimate_async = AsyncMock(side_effect=estimate_async) + service = ScenarioService() + service._registry = MagicMock() + service._registry.get_all_registered_class_metadata.return_value = metadata + service._registry.create_instance.return_value = scenario + service._estimate_semaphore = asyncio.Semaphore(2) + + result = await service.list_scenarios_async() + + assert maximum_active == 2 + assert all(item.default_run_size == estimate for item in result.items) + + async def test_catalog_queue_wait_does_not_start_execution_timeout(self) -> None: + """A queued catalog estimate starts its execution timeout only after acquiring capacity.""" + metadata = [_make_scenario_metadata(registry_name=f"test.scenario_{index}") for index in range(2)] + estimate = ScenarioDefaultRunSizeEstimate( + status=ScenarioRunSizeEstimateStatus.Exact, + total_attack_count=1, + components=[ScenarioRunSizeComponent(label="Default sweep", count=1)], + ) + first_estimate_started = asyncio.Event() + release_first_estimate = asyncio.Event() + second_timeout_started = asyncio.Event() + estimate_count = 0 + timeout_count = 0 + + async def estimate_async() -> ScenarioDefaultRunSizeEstimate: + nonlocal estimate_count + estimate_count += 1 + if estimate_count == 1: + first_estimate_started.set() + await release_first_estimate.wait() + return estimate + + async def wait_for_async( + awaitable: Awaitable[ScenarioDefaultRunSizeEstimate], *, timeout: float + ) -> ScenarioDefaultRunSizeEstimate: + nonlocal timeout_count + timeout_count += 1 + if timeout_count == 2: + second_timeout_started.set() + return await awaitable + + scenario = MagicMock() + scenario.get_default_run_size_estimate_async = AsyncMock(side_effect=estimate_async) + service = ScenarioService() + service._registry = MagicMock() + service._registry.get_all_registered_class_metadata.return_value = metadata + service._registry.create_instance.return_value = scenario + service._estimate_semaphore = asyncio.Semaphore(1) + + with patch("pyrit.backend.services.scenario_service.asyncio.wait_for", side_effect=wait_for_async): + catalog_task = asyncio.create_task(service.list_scenarios_async()) + await first_estimate_started.wait() + await asyncio.sleep(0) + assert not second_timeout_started.is_set() + + release_first_estimate.set() + result = await catalog_task + + assert second_timeout_started.is_set() + assert timeout_count == 2 + assert all(item.default_run_size == estimate for item in result.items) + + async def test_catalog_execution_timeout_is_unavailable_and_cached(self) -> None: + """A genuine estimate execution timeout is unavailable and reused from cache.""" + metadata = _make_scenario_metadata() + estimate_started = asyncio.Event() + estimate_cancelled = asyncio.Event() + block_estimate = asyncio.Event() + + async def slow_estimate_async() -> ScenarioDefaultRunSizeEstimate: + estimate_started.set() + try: + await block_estimate.wait() + except asyncio.CancelledError: + estimate_cancelled.set() + raise + raise AssertionError("The blocked estimate should be cancelled by its execution timeout.") + + scenario = MagicMock() + scenario.get_default_run_size_estimate_async = AsyncMock(side_effect=slow_estimate_async) + service = ScenarioService() + service._registry = MagicMock() + service._registry.create_instance.return_value = scenario + + with patch("pyrit.backend.services.scenario_service._DEFAULT_ESTIMATE_TIMEOUT_SECONDS", 0.01): + estimate_task = asyncio.create_task(service._get_default_run_size_estimate_async(metadata=metadata)) + await estimate_started.wait() + result = await estimate_task + cached = await service._get_default_run_size_estimate_async(metadata=metadata) + await asyncio.sleep(0) + + assert result.status is ScenarioRunSizeEstimateStatus.Unavailable + assert cached is result + assert estimate_cancelled.is_set() + service._registry.create_instance.assert_called_once_with(metadata.registry_name) + scenario.get_default_run_size_estimate_async.assert_awaited_once() + assert service._estimate_tasks == {} + + async def test_configured_estimate_does_not_wait_for_catalog_estimate(self) -> None: + """Interactive detail estimates use separate capacity from default catalog cards.""" + metadata = _make_scenario_metadata() + default_estimate = ScenarioDefaultRunSizeEstimate( + status=ScenarioRunSizeEstimateStatus.Exact, + total_attack_count=1, + components=[ScenarioRunSizeComponent(label="Default sweep", count=1)], + ) + configured_estimate = ScenarioDefaultRunSizeEstimate( + status=ScenarioRunSizeEstimateStatus.Exact, + total_attack_count=2, + components=[ScenarioRunSizeComponent(label="Configured sweep", count=2)], + ) + started = asyncio.Event() + release = asyncio.Event() + + async def default_estimate_async() -> ScenarioDefaultRunSizeEstimate: + started.set() + await release.wait() + return default_estimate + + scenario = MagicMock() + scenario.get_default_run_size_estimate_async = AsyncMock(side_effect=default_estimate_async) + service = ScenarioService() + service._registry = MagicMock() + service._registry.get_registered_class_metadata.return_value = metadata + service._registry.create_instance.return_value = scenario + service._estimate_semaphore = asyncio.Semaphore(1) + service._estimate_configured_run_size_async = AsyncMock(return_value=configured_estimate) + + catalog_task = asyncio.create_task(service._get_default_run_size_estimate_async(metadata=metadata)) + await started.wait() + interactive = await asyncio.wait_for( + service.estimate_scenario_run_size_async( + scenario_name=metadata.registry_name, + request=ScenarioRunSizeEstimateRequest(), + ), + timeout=0.2, + ) + release.set() + + assert interactive == configured_estimate + assert await catalog_task == default_estimate + async def test_unavailable_estimate_cache_expires(self) -> None: """A transient estimate failure is retried after the unavailable-result TTL.""" metadata = _make_scenario_metadata() @@ -355,7 +551,7 @@ async def test_unavailable_estimate_cache_expires(self) -> None: ) with ( - patch.object(ScenarioService, "__init__", lambda self: None), + patch.object(ScenarioService, "__init__", _initialize_test_service), patch("pyrit.backend.services.scenario_service._UNAVAILABLE_CACHE_TTL_SECONDS", 0), ): service = ScenarioService() @@ -383,7 +579,7 @@ async def test_estimate_cache_is_version_aware_and_bounded(self) -> None: scenario.get_default_run_size_estimate_async = AsyncMock(return_value=estimate) with ( - patch.object(ScenarioService, "__init__", lambda self: None), + patch.object(ScenarioService, "__init__", _initialize_test_service), patch("pyrit.backend.services.scenario_service._ESTIMATE_CACHE_SIZE", 1), ): service = ScenarioService() @@ -402,7 +598,7 @@ async def test_list_scenarios_preserves_disabled_baseline_policy(self) -> None: include_baseline_by_default=False, ) - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_all_registered_class_metadata.return_value = [metadata] @@ -418,7 +614,7 @@ async def test_list_scenarios_paginates_with_limit(self) -> None: _make_scenario_metadata(registry_name=f"test.scenario_{i}", class_name=f"Scenario{i}") for i in range(5) ] - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_all_registered_class_metadata.return_value = metadata_list @@ -440,7 +636,7 @@ async def test_list_scenarios_paginates_with_cursor(self) -> None: _make_scenario_metadata(registry_name=f"test.scenario_{i}", class_name=f"Scenario{i}") for i in range(5) ] - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_all_registered_class_metadata.return_value = metadata_list @@ -458,7 +654,7 @@ async def test_list_scenarios_last_page_has_more_false(self) -> None: _make_scenario_metadata(registry_name=f"test.scenario_{i}", class_name=f"Scenario{i}") for i in range(3) ] - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_all_registered_class_metadata.return_value = metadata_list @@ -488,7 +684,7 @@ async def test_configured_estimate_uses_shared_launch_resolution(self) -> None: objective_target = MagicMock() with ( - patch.object(ScenarioService, "__init__", lambda self: None), + patch.object(ScenarioService, "__init__", _initialize_test_service), patch.object(ScenarioRunService, "resolve_target_name", return_value=objective_target) as resolve_target, ): service = ScenarioService() @@ -539,7 +735,7 @@ async def test_configured_estimate_rejects_incompatible_v4_jailbreak_technique(s introspection_instance._default_dataset_config = DatasetAttackConfiguration(dataset_names=["harmbench"]) scenario_class = MagicMock(return_value=introspection_instance) - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_registered_class_metadata.return_value = metadata @@ -567,7 +763,7 @@ async def test_configured_estimate_without_target_does_not_resolve_or_send_to_ta scenario_class = MagicMock(return_value=introspection_instance) with ( - patch.object(ScenarioService, "__init__", lambda self: None), + patch.object(ScenarioService, "__init__", _initialize_test_service), patch.object(ScenarioRunService, "resolve_target_name") as resolve_target, ): service = ScenarioService() @@ -591,7 +787,7 @@ async def test_get_scenario_returns_matching_scenario(self) -> None: """Test that get returns the matching scenario.""" metadata = _make_scenario_metadata(registry_name="foundry.red_team_agent") - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_registered_class_metadata.return_value = metadata @@ -603,7 +799,7 @@ async def test_get_scenario_returns_matching_scenario(self) -> None: async def test_get_scenario_returns_none_for_missing(self) -> None: """Test that get returns None when scenario not found.""" - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_registered_class_metadata.return_value = None @@ -729,6 +925,8 @@ def test_get_scenario_returns_200(self, client: TestClient) -> None: default_run_size=ScenarioDefaultRunSizeEstimate( status=ScenarioRunSizeEstimateStatus.Exact, total_attack_count=8, + minimum_attack_count=8, + maximum_attack_count=8, components=[ ScenarioRunSizeComponent( label="Default technique sweep", @@ -752,6 +950,8 @@ def test_get_scenario_returns_200(self, client: TestClient) -> None: assert data["default_run_size"]["version"] == 1 assert data["default_run_size"]["status"] == "exact" assert data["default_run_size"]["total_attack_count"] == 8 + assert data["default_run_size"]["minimum_attack_count"] == 8 + assert data["default_run_size"]["maximum_attack_count"] == 8 def test_get_scenario_returns_404_when_not_found(self, client: TestClient) -> None: """Test that GET /api/scenarios/catalog/{name} returns 404 when not found.""" @@ -793,6 +993,8 @@ def test_estimate_scenario_returns_configured_projection(self, client: TestClien assert response.status_code == status.HTTP_200_OK assert response.json()["total_attack_count"] == 8 + assert response.json()["minimum_attack_count"] is None + assert response.json()["maximum_attack_count"] is None request = mock_service.estimate_scenario_run_size_async.await_args.kwargs["request"] assert request.techniques == ["prompt_sending"] assert request.include_baseline is False @@ -914,7 +1116,7 @@ async def test_list_scenarios_includes_supported_parameters(self) -> None: ), ) - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_all_registered_class_metadata.return_value = [metadata] @@ -943,7 +1145,7 @@ async def test_scenario_with_no_parameters_has_empty_list(self) -> None: """Test that scenarios without parameters have empty supported_parameters.""" metadata = _make_scenario_metadata() - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_all_registered_class_metadata.return_value = [metadata] @@ -973,7 +1175,7 @@ async def test_supported_parameters_with_none_default(self) -> None: ), ) - with patch.object(ScenarioService, "__init__", lambda self: None): + with patch.object(ScenarioService, "__init__", _initialize_test_service): service = ScenarioService() service._registry = MagicMock() service._registry.get_all_registered_class_metadata.return_value = [metadata] diff --git a/tests/unit/models/test_scenario_catalog.py b/tests/unit/models/test_scenario_catalog.py index c8451ca502..bd56e89b4a 100644 --- a/tests/unit/models/test_scenario_catalog.py +++ b/tests/unit/models/test_scenario_catalog.py @@ -7,11 +7,13 @@ from pydantic import ValidationError from pyrit.models import ( + ScenarioAdaptiveRunSizeDetails, ScenarioDatasetSizeCap, ScenarioDatasetSummary, ScenarioDefaultRunSizeEstimate, ScenarioRunSizeComponent, ScenarioRunSizeEstimate, + ScenarioRunSizeEstimateCondition, ScenarioRunSizeEstimateRequest, ScenarioRunSizeEstimateStatus, ScenarioRunSizeFactor, @@ -46,6 +48,8 @@ def test_run_size_estimate_accepts_legacy_fields_and_serializes_canonically() -> payload = estimate.model_dump(mode="json") assert payload["version"] == 1 assert payload["total_attack_count"] == 2 + assert payload["minimum_attack_count"] is None + assert payload["maximum_attack_count"] is None assert payload["note"] == "Legacy explanation." assert payload["datasets"][0]["logical_seed_group_count"] == 100 assert "total" not in payload @@ -85,6 +89,42 @@ def test_exact_default_run_size_requires_component_total() -> None: ) +@pytest.mark.parametrize("field_name", ["minimum_attack_count", "maximum_attack_count"]) +def test_exact_default_run_size_requires_bounds_to_match_total(field_name: str) -> None: + """Exact estimates reject bounds that disagree with their authoritative total.""" + with pytest.raises(ValidationError, match=f"{field_name} to equal total_attack_count"): + ScenarioDefaultRunSizeEstimate( + status=ScenarioRunSizeEstimateStatus.Exact, + total_attack_count=6, + components=[ScenarioRunSizeComponent(label="Techniques", count=6)], + **{field_name: 5}, + ) + + +def test_default_run_size_requires_ordered_nonnegative_bounds() -> None: + """Conditional estimate bounds remain nonnegative and ordered.""" + with pytest.raises(ValidationError, match="greater than or equal to 0"): + ScenarioDefaultRunSizeEstimate( + status=ScenarioRunSizeEstimateStatus.Conditional, + minimum_attack_count=-1, + ) + + with pytest.raises(ValidationError, match="minimum_attack_count must be less than or equal"): + ScenarioDefaultRunSizeEstimate( + status=ScenarioRunSizeEstimateStatus.Conditional, + minimum_attack_count=20, + maximum_attack_count=12, + ) + + +def test_conditional_default_run_size_allows_unknown_bounds() -> None: + """Conditional estimates may remain unbounded when no truthful range is available.""" + estimate = ScenarioDefaultRunSizeEstimate(status=ScenarioRunSizeEstimateStatus.Conditional) + + assert estimate.minimum_attack_count is None + assert estimate.maximum_attack_count is None + + def test_run_size_component_requires_factor_product() -> None: """Components reject counts that disagree with their ordered formula factors.""" with pytest.raises(ValidationError, match="factor product \\(6\\)"): @@ -119,6 +159,9 @@ def test_default_run_size_serializes_versioned_api_shape() -> None: "version": 1, "status": "exact", "total_attack_count": 6, + "minimum_attack_count": None, + "maximum_attack_count": None, + "condition": None, "components": [ { "label": "Techniques", @@ -129,18 +172,87 @@ def test_default_run_size_serializes_versioned_api_shape() -> None: ], "note": None, "is_baseline": False, + "condition": None, } ], "datasets": [], + "adaptive_details": None, "note": None, "retries_included": False, } +def test_adaptive_run_size_details_serialize_derived_attempt_bounds() -> None: + """Adaptive estimates expose progress objectives and underlying attempt bounds separately.""" + details = ScenarioAdaptiveRunSizeDetails( + objective_count=21, + selected_candidate_technique_count=2, + candidate_technique_count=2, + max_attempts_per_objective=3, + techniques_per_objective_upper_bound=2, + technique_attempt_count_upper_bound=42, + ) + + assert details.model_dump(mode="json") == { + "objective_count": 21, + "selected_candidate_technique_count": 2, + "candidate_technique_count": 2, + "max_attempts_per_objective": 3, + "techniques_per_objective_upper_bound": 2, + "technique_attempt_count_upper_bound": 42, + "stop_on_first_success": True, + "compatibility_may_reduce_attempts": True, + } + + +def test_adaptive_run_size_details_reject_inconsistent_attempt_bounds() -> None: + """Adaptive work bounds cannot drift from the selected pool and configured cap.""" + with pytest.raises(ValidationError, match="min\\(candidate_technique_count, max_attempts_per_objective\\)"): + ScenarioAdaptiveRunSizeDetails( + objective_count=21, + selected_candidate_technique_count=2, + candidate_technique_count=2, + max_attempts_per_objective=3, + techniques_per_objective_upper_bound=3, + technique_attempt_count_upper_bound=63, + ) + + +def test_adaptive_run_size_details_accept_legacy_version_one_payload() -> None: + """Version-one payloads without the additive selected count remain readable.""" + details = ScenarioAdaptiveRunSizeDetails.model_validate( + { + "objective_count": 21, + "candidate_technique_count": 2, + "max_attempts_per_objective": 3, + "techniques_per_objective_upper_bound": 2, + "technique_attempt_count_upper_bound": 42, + } + ) + + assert details.selected_candidate_technique_count == 2 + + +def test_adaptive_run_size_details_reject_more_compatible_than_selected_candidates() -> None: + """Resolved compatible candidates cannot exceed the concrete selected pool.""" + with pytest.raises(ValidationError, match="cannot exceed selected_candidate_technique_count"): + ScenarioAdaptiveRunSizeDetails( + objective_count=21, + selected_candidate_technique_count=2, + candidate_technique_count=3, + max_attempts_per_objective=3, + techniques_per_objective_upper_bound=3, + technique_attempt_count_upper_bound=63, + ) + + def test_conditional_estimate_exposes_dataset_counts_structurally() -> None: """Conditionality and effective dataset selection are machine-readable.""" estimate = ScenarioDefaultRunSizeEstimate( status=ScenarioRunSizeEstimateStatus.Conditional, + minimum_attack_count=12, + maximum_attack_count=20, + condition=ScenarioRunSizeEstimateCondition.TargetCapabilities, datasets=[ ScenarioDatasetSummary( name="harmbench", @@ -163,6 +275,9 @@ def test_conditional_estimate_exposes_dataset_counts_structurally() -> None: payload = estimate.model_dump(mode="json") assert payload["status"] == "conditional" assert payload["total_attack_count"] is None + assert payload["minimum_attack_count"] == 12 + assert payload["maximum_attack_count"] == 20 + assert payload["condition"] == "target_capabilities" assert payload["datasets"] == [ { "name": "harmbench", diff --git a/tests/unit/registry/test_scenario_registry.py b/tests/unit/registry/test_scenario_registry.py index b3abf2b2b0..2b40e3a267 100644 --- a/tests/unit/registry/test_scenario_registry.py +++ b/tests/unit/registry/test_scenario_registry.py @@ -3,12 +3,21 @@ """Tests for ScenarioRegistry._build_metadata and create_and_initialize_async.""" -from unittest.mock import AsyncMock, MagicMock +from typing import Literal +from unittest.mock import AsyncMock, MagicMock, patch import pytest -from pyrit.registry.components.scenario_registry import ScenarioRegistry -from pyrit.scenario.core import BaselineAttackPolicy, ScenarioTechnique +from pyrit.registry import ScenarioRegistry +from pyrit.scenario import ( + BaselineAttackPolicy, + CompoundDatasetAttackConfiguration, + DatasetAttackConfiguration, + ScenarioTechnique, +) +from pyrit.scenario.scenarios.adaptive import TextAdaptive +from pyrit.scenario.scenarios.airt import Psychosocial +from pyrit.scenario.scenarios.garak import WebInjection class _NotNoArgScenario: @@ -61,6 +70,10 @@ def _resolve_scenario_techniques(self, *, scenario_techniques): """Resolve the concrete defaults.""" return _MetadataTechnique.resolve(scenario_techniques, default=self._default_technique) + def get_dataset_size_limit_override_scope(self) -> Literal["per_dataset"]: + """Return the test scenario's conventional single-dataset override scope.""" + return "per_dataset" + class _MarkdownMetadataScenario(_MetadataScenario): """ @@ -94,6 +107,92 @@ def test_build_metadata_expands_ordered_default_techniques() -> None: } +@pytest.mark.parametrize( + ("configuration", "declared_override_scope", "default_scope", "default_count", "override_scope"), + [ + (DatasetAttackConfiguration(dataset_names=["sample"]), "per_dataset", "none", None, "per_dataset"), + ( + DatasetAttackConfiguration(dataset_names=["one", "two"]), + "per_dataset", + "none", + None, + "per_dataset", + ), + ( + DatasetAttackConfiguration(dataset_names=["one", "two"]), + "unsupported", + "none", + None, + "unsupported", + ), + ( + DatasetAttackConfiguration(dataset_names=["one", "two"], max_dataset_size=6), + "combined", + "combined", + 6, + "combined", + ), + ( + CompoundDatasetAttackConfiguration.per_dataset( + dataset_names=["one", "two"], + max_dataset_size=4, + ), + "per_dataset", + "per_dataset", + 4, + "per_dataset", + ), + ( + CompoundDatasetAttackConfiguration( + configurations=[ + DatasetAttackConfiguration(dataset_names=["one"], max_dataset_size=3), + DatasetAttackConfiguration(dataset_names=["two"], max_dataset_size=4), + ] + ), + "per_dataset", + "heterogeneous", + None, + "per_dataset", + ), + ], +) +def test_build_dataset_size_limit_normalizes_configuration_semantics( + configuration: DatasetAttackConfiguration, + declared_override_scope: Literal["per_dataset", "combined", "unsupported"], + default_scope: str, + default_count: int | None, + override_scope: str, +) -> None: + """Catalog limit metadata preserves no-cap, combined, per-dataset, and heterogeneous defaults.""" + limit = ScenarioRegistry._build_dataset_size_limit( + default_dataset_config=configuration, + override_scope=declared_override_scope, + ) + + assert limit.default_scope == default_scope + assert limit.default_count == default_count + assert limit.override_scope == override_scope + + +def test_specialized_scenarios_declare_nonstandard_dataset_override_semantics() -> None: + """Catalog metadata can remain truthful when a scenario reshapes or ignores generic dataset caps.""" + assert Psychosocial.DATASET_SIZE_LIMIT_OVERRIDE_SCOPE == "per_dataset" + assert WebInjection.DATASET_SIZE_LIMIT_OVERRIDE_SCOPE == "unsupported" + + +def test_text_adaptive_metadata_exposes_per_dataset_default_limit() -> None: + """TextAdaptive publishes its canonical four-objective child cap without scenario-name special cases.""" + with ( + patch.object(TextAdaptive, "_get_default_objective_scorer", return_value=MagicMock()), + patch("pyrit.scenario.core.scenario.CentralMemory.get_memory_instance", return_value=MagicMock()), + ): + metadata = ScenarioRegistry()._build_metadata("adaptive.text_adaptive", TextAdaptive) + + assert metadata.dataset_size_limit.default_scope == "per_dataset" + assert metadata.dataset_size_limit.default_count == 4 + assert metadata.dataset_size_limit.override_scope == "per_dataset" + + def test_build_metadata_preserves_structured_markdown_separately() -> None: """Scenario metadata keeps plain compatibility text and Markdown source.""" metadata = ScenarioRegistry()._build_metadata("markdown", _MarkdownMetadataScenario) diff --git a/tests/unit/scenario/airt/test_jailbreak.py b/tests/unit/scenario/airt/test_jailbreak.py index d9944ce7ed..3e93d768cd 100644 --- a/tests/unit/scenario/airt/test_jailbreak.py +++ b/tests/unit/scenario/airt/test_jailbreak.py @@ -3,12 +3,13 @@ """Tests for the Jailbreak class.""" +import logging from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest -from pyrit.backend.services.scenario_run_service import ScenarioRunService +from pyrit.backend.services import ScenarioRunService from pyrit.common.path import JAILBREAK_TEMPLATES_PATH from pyrit.converter import TextJailbreakConverter from pyrit.datasets import TextJailBreak @@ -21,9 +22,8 @@ SeedPrompt, ) from pyrit.prompt_target import PromptTarget -from pyrit.registry import TargetRegistry +from pyrit.registry import ScenarioRegistry, TargetRegistry from pyrit.registry.components.attack_technique_registry import AttackTechniqueRegistry -from pyrit.registry.components.scenario_registry import ScenarioRegistry from pyrit.scenario.core import BaselineAttackPolicy from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory from pyrit.scenario.scenarios.airt.jailbreak import ( @@ -270,6 +270,9 @@ async def test_run_size_is_conditional_when_system_delivery_target_is_not_select assert estimate.status is ScenarioRunSizeEstimateStatus.Conditional assert estimate.total_attack_count is None + assert estimate.minimum_attack_count == 2 + assert estimate.maximum_attack_count == 4 + assert estimate.condition.value == "target_capabilities" assert [component.label for component in estimate.components] == [ "Inline jailbreak delivery", "Native system-prompt jailbreak delivery", @@ -814,6 +817,22 @@ def test_only_compatible_direct_registry_techniques_are_available(self): assert {_PROMPT_SENDING, _JAILBREAK_SYSTEM_PROMPT, "flip"}.issubset(available) assert incompatible.isdisjoint(available) + def test_warns_when_registered_factories_cannot_compose_jailbreak_converter(self, caplog): + custom_factory = AttackTechniqueFactory( + name="custom_without_converter_composition", + attack_class=PromptSendingAttack, + technique_tags=["single_turn"], + ) + AttackTechniqueRegistry.get_registry_singleton().register_from_factories([custom_factory]) + _build_jailbreak_technique.cache_clear() + + with caplog.at_level(logging.WARNING): + technique_class = _build_jailbreak_technique() + + assert "custom_without_converter_composition" in caplog.text + assert "cannot compose the required request converter" in caplog.text + assert custom_factory.name not in {technique.value for technique in technique_class.get_all_techniques()} + def test_registry_metadata_omits_incompatible_techniques(self): metadata = ScenarioRegistry()._build_metadata("airt.jailbreak", Jailbreak) incompatible = { diff --git a/tests/unit/scenario/core/test_dataset_configuration.py b/tests/unit/scenario/core/test_dataset_configuration.py index 3c3ad9c25c..5e1aabd103 100644 --- a/tests/unit/scenario/core/test_dataset_configuration.py +++ b/tests/unit/scenario/core/test_dataset_configuration.py @@ -262,6 +262,16 @@ async def test_max_sample_is_a_single_global_budget(self, mock_memory: MagicMock result = await config.get_attack_groups_by_dataset_async() assert sum(len(groups) for groups in result.values()) == 2 + async def test_estimate_resolution_fetches_full_and_sampled_groups_once(self, mock_memory: MagicMock) -> None: + mock_memory.get_seeds.return_value = make_objectives("a", "b", "c") + config = DatasetAttackConfiguration(dataset_names=["d1"], max_dataset_size=1) + + full, selected = await config.resolve_attack_groups_for_estimate_async() + + assert len(full["d1"]) == 3 + assert len(selected["d1"]) == 1 + mock_memory.get_seeds.assert_called_once() + async def test_loud_raise_when_a_dataset_is_empty(self, mock_memory: MagicMock) -> None: mock_memory.get_seeds.side_effect = [make_objectives("a"), []] config = DatasetAttackConfiguration(dataset_names=["d1", "d2"], auto_fetch=False) @@ -507,6 +517,95 @@ def test_per_dataset_builds_one_child_per_name(self) -> None: assert [child.dataset_names for child in config._configurations] == [["d1"], ["d2"]] assert all(child.max_dataset_size == 4 for child in config._configurations) + def test_with_dataset_names_preserves_per_dataset_defaults(self) -> None: + config = CompoundDatasetAttackConfiguration.per_dataset( + dataset_names=["d1", "d2"], + max_dataset_size=4, + filters={"harm_categories": ["original"]}, + ) + + overridden = config.with_dataset_names( + dataset_names=["selected"], + filters={"data_types": ["text"]}, + ) + + assert overridden.dataset_names == ["selected"] + assert len(overridden._configurations) == 1 + assert overridden._configurations[0].max_dataset_size == 4 + assert overridden._configurations[0].filters == { + "harm_categories": ["original"], + "data_types": ["text"], + } + assert overridden._configurations[0]._validators == config._configurations[0]._validators + assert overridden._validators == config._validators + assert config.dataset_names == ["d1", "d2"] + + def test_with_dataset_names_rejects_shaped_children(self) -> None: + class _ShapedDatasetConfiguration(DatasetAttackConfiguration): + pass + + config = CompoundDatasetAttackConfiguration( + configurations=[_ShapedDatasetConfiguration(dataset_names=["d1"])], + ) + + with pytest.raises(TypeError, match="homogeneous single-dataset"): + config.with_dataset_names(dataset_names=["selected"]) + + def test_with_dataset_names_rejects_duplicates(self) -> None: + config = CompoundDatasetAttackConfiguration.per_dataset(dataset_names=["d1", "d2"]) + + with pytest.raises(ValueError, match="cannot contain duplicates"): + config.with_dataset_names(dataset_names=["selected", "selected"]) + + def test_with_dataset_names_rejects_different_child_validators(self) -> None: + first_validator = require_min_size(1) + second_validator = require_min_size(2) + config = CompoundDatasetAttackConfiguration( + configurations=[ + DatasetAttackConfiguration(dataset_names=["d1"], validators=[first_validator]), + DatasetAttackConfiguration(dataset_names=["d2"], validators=[second_validator]), + ], + ) + + with pytest.raises(TypeError, match="shared caps, filters, validators"): + config.with_dataset_names(dataset_names=["selected"]) + + def test_with_dataset_names_supports_shared_unhashable_validator(self) -> None: + class _UnhashableValidator: + __hash__ = None + + def __call__(self, resolved: ResolvedDataset) -> None: + del resolved + + validator = _UnhashableValidator() + config = CompoundDatasetAttackConfiguration.per_dataset( + dataset_names=["d1", "d2"], + validators=[validator], + ) + + overridden = config.with_dataset_names(dataset_names=["selected"]) + + assert overridden._configurations[0]._custom_validators == [validator] + + def test_update_child_max_dataset_size_preserves_shaped_children(self) -> None: + class _ShapedDatasetConfiguration(DatasetAttackConfiguration): + pass + + config = CompoundDatasetAttackConfiguration( + configurations=[ + _ShapedDatasetConfiguration(dataset_names=["d1"], max_dataset_size=4), + _ShapedDatasetConfiguration(dataset_names=["d2"], max_dataset_size=4), + ], + ) + + config.update_child_max_dataset_size(max_dataset_size=2) + + assert [type(child) for child in config._configurations] == [ + _ShapedDatasetConfiguration, + _ShapedDatasetConfiguration, + ] + assert [child.max_dataset_size for child in config._configurations] == [2, 2] + def test_size_caps_report_child_and_combined_limits(self) -> None: """Planning metadata explains independent child caps and the final compound cap.""" config = CompoundDatasetAttackConfiguration.per_dataset(dataset_names=["d1", "d2"], max_dataset_size=4) diff --git a/tests/unit/scenario/scenarios/adaptive/test_dispatcher.py b/tests/unit/scenario/scenarios/adaptive/test_dispatcher.py index eceb2b5cf9..b33a769829 100644 --- a/tests/unit/scenario/scenarios/adaptive/test_dispatcher.py +++ b/tests/unit/scenario/scenarios/adaptive/test_dispatcher.py @@ -18,6 +18,8 @@ ) from pyrit.scenario.scenarios.adaptive.dispatcher import ( ADAPTIVE_ATTEMPT_LABEL, + ADAPTIVE_TECHNIQUE_ID_LABEL, + ADAPTIVE_TECHNIQUE_NAME_LABEL, AdaptiveTechniqueDispatcher, TechniqueBundle, ) @@ -127,6 +129,10 @@ async def test_builds_sequential_attack(self, target, seed_group): # 1-based per-attempt label stamped on each child assert attack._child_attacks[0].memory_labels[ADAPTIVE_ATTEMPT_LABEL] == "1" assert attack._child_attacks[1].memory_labels[ADAPTIVE_ATTEMPT_LABEL] == "2" + assert attack._child_attacks[0].memory_labels[ADAPTIVE_TECHNIQUE_ID_LABEL] == "a" + assert attack._child_attacks[1].memory_labels[ADAPTIVE_TECHNIQUE_ID_LABEL] == "b" + assert attack._child_attacks[0].memory_labels[ADAPTIVE_TECHNIQUE_NAME_LABEL] == "a" + assert attack._child_attacks[1].memory_labels[ADAPTIVE_TECHNIQUE_NAME_LABEL] == "b" # default policy is FIRST_SUCCESS assert attack._completion_policy is SequenceCompletionPolicy.FIRST_SUCCESS @@ -239,37 +245,25 @@ async def test_merges_real_system_prompt_technique_onto_user_turn_at_sequence_ze @pytest.mark.usefixtures("patch_central_database") -class TestEvalHashRoundTrip: +class TestRegisteredTechniqueIdentityRoundTrip: """ - Pin the load-bearing invariant that ``compute_inner_attack_eval_hash`` - (used by ``AdaptiveScenario._build_techniques_dict`` to key the - ``techniques`` dict and by the selector to look up historical stats) - equals the ``eval_hash`` the executor stamps on persisted child rows. - - If the prediction helper and the write path ever drift (e.g. a new - field is added to the eval-hash rule on one side only), the selector - silently reads zero history for every technique and epsilon-greedy - degrades to random with no error. This test runs a real - ``PromptSendingAttack`` through the dispatcher's ``SequentialAttack`` - end-to-end and asserts the round-trip holds. + Pin the selector identity labels through real child-result persistence. """ - async def test_predicted_hash_matches_persisted_row(self, sqlite_instance): + async def test_registered_identity_and_name_are_persisted(self, sqlite_instance): from pyrit.executor.attack.single_turn.prompt_sending import PromptSendingAttack from pyrit.memory.memory_models import AttackResultEntry from pyrit.models import AttackSeedGroup, SeedObjective - from pyrit.models.identifiers import compute_inner_attack_eval_hash from tests.unit.mocks import MockPromptTarget live_target = MockPromptTarget() attack = PromptSendingAttack(objective_target=live_target) - predicted_hash = compute_inner_attack_eval_hash(attack=attack) - - bundles = {predicted_hash: TechniqueBundle(attack=attack, name="prompt_sending")} + technique_identifier = "factory-identity-hash" + bundles = {technique_identifier: TechniqueBundle(attack=attack, name="prompt_sending")} dispatcher = AdaptiveTechniqueDispatcher( objective_target=live_target, techniques=bundles, - selector=_StubSelector(technique_order=[predicted_hash]), + selector=_StubSelector(technique_order=[technique_identifier]), max_attempts_per_objective=1, ) @@ -280,8 +274,6 @@ async def test_predicted_hash_matches_persisted_row(self, sqlite_instance): with sqlite_instance.get_session() as session: rows = session.query(AttackResultEntry).all() - # Drill into the persisted envelope to find rows whose inner attack is PromptSendingAttack, - # then assert the eval_hash on those rows matches what the selector predicted. matching_rows = [ r for r in rows @@ -297,10 +289,5 @@ async def test_predicted_hash_matches_persisted_row(self, sqlite_instance): f"Expected at least one persisted row whose inner attack is PromptSendingAttack; " f"found rows: {[(r.id, r.atomic_attack_identifier) for r in rows]}" ) - for row in matching_rows: - stamped_hash = row.atomic_attack_identifier["eval_hash"] - assert stamped_hash == predicted_hash, ( - f"Selector-side eval_hash ({predicted_hash}) drifted from executor-stamped " - f"eval_hash ({stamped_hash}) on persisted row {row.id}. " - f"compute_inner_attack_eval_hash and AtomicAttackIdentifier.build must agree." - ) + assert all(row.labels[ADAPTIVE_TECHNIQUE_ID_LABEL] == technique_identifier for row in matching_rows) + assert all(row.labels[ADAPTIVE_TECHNIQUE_NAME_LABEL] == "prompt_sending" for row in matching_rows) diff --git a/tests/unit/scenario/scenarios/adaptive/test_epsilon_greedy.py b/tests/unit/scenario/scenarios/adaptive/test_epsilon_greedy.py index 21144721f5..23d4ee12f9 100644 --- a/tests/unit/scenario/scenarios/adaptive/test_epsilon_greedy.py +++ b/tests/unit/scenario/scenarios/adaptive/test_epsilon_greedy.py @@ -6,14 +6,15 @@ import pytest from pyrit.analytics.result_analysis import AttackStats -from pyrit.scenario.scenarios.adaptive.selectors import ( +from pyrit.scenario.scenarios.adaptive import ( + AdaptiveTechniqueIdentifier, EpsilonGreedyTechniqueSelector, SelectorScope, ) TECHNIQUES = ["a", "b", "c", "d"] -_COMPUTE_PATH = "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_technique_stats" +_COMPUTE_PATH = "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_labeled_technique_stats" def _seeded_selector(*, epsilon: float = 0.0, random_seed: int = 0) -> EpsilonGreedyTechniqueSelector: @@ -66,7 +67,7 @@ def test_init_rejects_out_of_range_epsilon(self, bad_epsilon): class TestEpsilonGreedyTechniqueSelectorSelect: @patch( - "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_technique_stats", + "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_labeled_technique_stats", side_effect=_empty_rates, ) async def test_select_empty_techniques_raises(self, _mock): @@ -75,7 +76,7 @@ async def test_select_empty_techniques_raises(self, _mock): await selector.select_async(technique_identifiers=[], objective="obj") @patch( - "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_technique_stats", + "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_labeled_technique_stats", side_effect=_empty_rates, ) async def test_select_all_unseen_ties_resolved_randomly(self, _mock): @@ -88,7 +89,7 @@ async def test_select_all_unseen_ties_resolved_randomly(self, _mock): assert winners.issubset(set(TECHNIQUES)) @patch( - "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_technique_stats", + "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_labeled_technique_stats", side_effect=_rates_with_winner("b"), ) async def test_select_exploits_clear_winner(self, _mock): @@ -98,7 +99,7 @@ async def test_select_exploits_clear_winner(self, _mock): assert result[0] == "b" @patch( - "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_technique_stats", + "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_labeled_technique_stats", side_effect=_empty_rates, ) async def test_select_epsilon_one_is_pure_random(self, _mock): @@ -110,7 +111,7 @@ async def test_select_epsilon_one_is_pure_random(self, _mock): assert picks == set(TECHNIQUES) @patch( - "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_technique_stats", + "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_labeled_technique_stats", side_effect=_empty_rates, ) async def test_select_returns_multiple_techniques(self, _mock): @@ -120,7 +121,7 @@ async def test_select_returns_multiple_techniques(self, _mock): assert len(set(result)) == 3 # no duplicates @patch( - "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_technique_stats", + "pyrit.scenario.scenarios.adaptive.selectors.epsilon_greedy.compute_labeled_technique_stats", side_effect=_empty_rates, ) async def test_select_caps_at_available_techniques(self, _mock): @@ -130,6 +131,18 @@ async def test_select_caps_at_available_techniques(self, _mock): class TestEpsilonGreedySelectorScope: + @patch(_COMPUTE_PATH, side_effect=_empty_rates) + async def test_forwards_full_technique_eval_hash_for_cross_scenario_history(self, mock_compute): + arm = AdaptiveTechniqueIdentifier( + factory_hash="factory-hash", + technique_eval_hash="full-technique-eval-hash", + ).serialize() + selector = _seeded_selector() + + await selector.select_async(technique_identifiers=[arm], objective="obj") + + assert mock_compute.call_args.kwargs["technique_eval_hashes_by_identifier"] == {arm: "full-technique-eval-hash"} + @patch(_COMPUTE_PATH, side_effect=_empty_rates) async def test_default_scope_passes_none_scenario_result_id(self, mock_compute): selector = _seeded_selector() diff --git a/tests/unit/scenario/scenarios/adaptive/test_technique_identity.py b/tests/unit/scenario/scenarios/adaptive/test_technique_identity.py new file mode 100644 index 0000000000..c491d6a1f7 --- /dev/null +++ b/tests/unit/scenario/scenarios/adaptive/test_technique_identity.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from pyrit.scenario.scenarios.adaptive.technique_identity import ( + AdaptiveTechniqueIdentifier, + get_history_eval_hash, +) + + +def test_adaptive_technique_identifier_round_trip() -> None: + identifier = AdaptiveTechniqueIdentifier( + factory_hash="factory-hash", + technique_eval_hash="technique-eval-hash", + ) + + serialized = identifier.serialize() + + assert AdaptiveTechniqueIdentifier.parse(serialized) == identifier + assert get_history_eval_hash(technique_identifier=serialized) == "technique-eval-hash" + + +def test_history_eval_hash_falls_back_for_custom_selector_identifier() -> None: + assert get_history_eval_hash(technique_identifier="custom-arm") == "custom-arm" diff --git a/tests/unit/scenario/scenarios/adaptive/test_text_adaptive.py b/tests/unit/scenario/scenarios/adaptive/test_text_adaptive.py index 8375bc1865..11f37f627a 100644 --- a/tests/unit/scenario/scenarios/adaptive/test_text_adaptive.py +++ b/tests/unit/scenario/scenarios/adaptive/test_text_adaptive.py @@ -11,17 +11,29 @@ import pytest -from pyrit.models import AttackSeedGroup, SeedObjective +from pyrit.models import AttackSeedGroup, ScenarioDatasetSummary, SeedObjective from pyrit.models.identifiers import ComponentIdentifier from pyrit.prompt_target import PromptTarget from pyrit.registry.components.attack_technique_registry import AttackTechniqueRegistry from pyrit.scenario.core.dataset_configuration import CompoundDatasetAttackConfiguration from pyrit.scenario.core.scenario import BaselineAttackPolicy from pyrit.scenario.scenarios.adaptive.dispatcher import AdaptiveTechniqueDispatcher +from pyrit.scenario.scenarios.adaptive.technique_identity import AdaptiveTechniqueIdentifier from pyrit.scenario.scenarios.adaptive.text_adaptive import TextAdaptive from pyrit.score import TrueFalseScorer _MOCK_MANY_SHOT_EXAMPLES = [{"question": f"q{i}", "answer": f"a{i}"} for i in range(100)] +_LIGHT_TECHNIQUES = { + "role_play_movie_script", + "role_play_video_game", + "role_play_trivia_game", + "role_play_persuasion", + "role_play_persuasion_written", + "many_shot", + "red_teaming", + "context_compliance", + "flip", +} def _mock_id(name: str) -> ComponentIdentifier: @@ -82,30 +94,43 @@ def _make_seed_group(*, value: str, harm_categories: list[str] | None = None) -> return AttackSeedGroup(seeds=[SeedObjective(value=value, harm_categories=harm_categories)]) -def _make_fake_factory(*, seed_technique=None, adversarial_chat=None, scoring_config_type=None) -> MagicMock: +def _make_fake_factory( + *, + seed_technique=None, + adversarial_chat=None, + scoring_config_type=None, + attack_identifier: ComponentIdentifier | None = None, + factory_identifier: ComponentIdentifier | None = None, + technique_identifier: ComponentIdentifier | None = None, +) -> MagicMock: """Return a stub attack-technique factory that produces a fake ``AttackTechnique``. Mocks the surface ``AdaptiveScenario._build_techniques_dict`` consumes - (``factory.create(...)``, ``factory.adversarial_chat``, and - ``factory.scoring_config_type``). Each call assigns a unique fake - attack identifier (via a fresh UUID) so the bundle dict keys (eval - hashes) don't collide across calls — no shared mutable test state, so - test execution order doesn't shift hash values. + (``factory.create(...)``, ``factory.get_identifier()``, + ``factory.adversarial_chat``, and ``factory.scoring_config_type``). + Each call assigns unique attack and factory identities unless a test + deliberately supplies shared identities. """ fake_id = uuid.uuid4().hex[:8] fake_technique = MagicMock() fake_attack = MagicMock(name=f"fake-attack-technique-{fake_id}") - fake_attack.get_identifier.return_value = ComponentIdentifier( - class_name=f"FakeAttack{fake_id}", - class_module="test_text_adaptive", + fake_attack.get_identifier.return_value = attack_identifier or ComponentIdentifier( + class_name=f"FakeAttack{fake_id}", class_module="test_text_adaptive" ) fake_technique.attack = fake_attack fake_technique.seed_technique = seed_technique + fake_technique.get_identifier.return_value = technique_identifier or ComponentIdentifier( + class_name=f"FakeTechnique{fake_id}", class_module="test_text_adaptive" + ) factory = MagicMock() factory.create.return_value = fake_technique factory.adversarial_chat = adversarial_chat + factory.uses_adversarial = adversarial_chat is not None factory.scoring_config_type = scoring_config_type + factory.get_identifier.return_value = factory_identifier or ComponentIdentifier( + class_name=f"FakeFactory{fake_id}", class_module="test_text_adaptive" + ) return factory @@ -129,6 +154,17 @@ def test_default_dataset_config(self): def test_required_datasets_non_empty(self): assert len(TextAdaptive.required_datasets()) > 0 + def test_max_attempts_parameter_distinguishes_techniques_from_retries(self): + parameter = next( + parameter + for parameter in TextAdaptive.additional_parameters() + if parameter.name == "max_attempts_per_objective" + ) + assert parameter.default == 3 + assert "different compatible techniques" in parameter.description + assert "stopping after the first success" in parameter.description + assert "separate from retries" in parameter.description + def test_get_technique_class_is_cached(self): cls_a = TextAdaptive.get_technique_class() cls_b = TextAdaptive.get_technique_class() @@ -333,6 +369,139 @@ async def test_techniques_with_seed_technique_are_kept(self, mock_objective_targ assert "role_play_movie_script" in technique_names assert "many_shot" in technique_names + @pytest.mark.parametrize(("max_attempts", "expected_attempts"), [(4, 84), (5, 105)]) + async def test_light_keeps_nine_distinct_factory_arms_and_attempt_bound( + self, + mock_objective_target, + mock_objective_scorer, + max_attempts, + expected_attempts, + ): + shared_attack_identifier = _mock_id("SharedPromptSendingAttack") + factories = { + name: _make_fake_factory( + attack_identifier=shared_attack_identifier, + factory_identifier=_mock_id(f"Factory_{name}"), + ) + for name in _LIGHT_TECHNIQUES + } + groups = {"adaptive": [_make_seed_group(value=f"obj-{index}") for index in range(21)]} + summaries = [ + ScenarioDatasetSummary( + name="adaptive", + logical_seed_group_count=21, + selected_seed_group_count=21, + ) + ] + scenario = TextAdaptive(objective_scorer=mock_objective_scorer) + technique_class = scenario.get_technique_class() + scenario.set_params_from_args( + args={ + "objective_target": mock_objective_target, + "scenario_techniques": [technique_class("light")], + "include_baseline": False, + "max_attempts_per_objective": max_attempts, + } + ) + scenario._resolve_dataset_groups_for_estimate_async = AsyncMock(return_value=(groups, summaries)) + + with patch.object(scenario, "_get_attack_technique_factories", return_value=factories): + estimate = await scenario.get_run_size_estimate_async() + techniques = scenario._build_techniques_dict(objective_target=mock_objective_target) + + assert len(techniques) == 9 + assert {bundle.name for bundle in techniques.values()} == _LIGHT_TECHNIQUES + parsed_identifiers = [AdaptiveTechniqueIdentifier.parse(identifier) for identifier in techniques] + assert all(identifier is not None for identifier in parsed_identifiers) + assert len({identifier.factory_hash for identifier in parsed_identifiers if identifier is not None}) == 9 + assert len({identifier.technique_eval_hash for identifier in parsed_identifiers if identifier is not None}) == 9 + assert estimate.adaptive_details is not None + assert estimate.adaptive_details.selected_candidate_technique_count == 9 + assert estimate.adaptive_details.candidate_technique_count == 9 + assert estimate.adaptive_details.techniques_per_objective_upper_bound == max_attempts + assert estimate.adaptive_details.technique_attempt_count_upper_bound == expected_attempts + + @pytest.mark.parametrize(("aggregate_name", "expected_candidate_count"), [("light", 9), ("core", 14)]) + async def test_aggregate_attempt_bound_increases_until_distinct_candidate_count( + self, + mock_objective_target, + mock_objective_scorer, + aggregate_name, + expected_candidate_count, + ): + technique_class = TextAdaptive.get_technique_class() + aggregate = technique_class(aggregate_name) + selected_names = {technique.value for technique in technique_class.expand({aggregate})} + assert len(selected_names) == expected_candidate_count + + shared_attack_identifier = _mock_id("SharedAttackImplementation") + factories = { + name: _make_fake_factory( + attack_identifier=shared_attack_identifier, + factory_identifier=_mock_id(f"Factory_{name}"), + ) + for name in selected_names + } + groups = {"adaptive": [_make_seed_group(value=f"obj-{index}") for index in range(21)]} + summaries = [ + ScenarioDatasetSummary( + name="adaptive", + logical_seed_group_count=21, + selected_seed_group_count=21, + ) + ] + scenario = TextAdaptive(objective_scorer=mock_objective_scorer) + scenario._resolve_dataset_groups_for_estimate_async = AsyncMock(return_value=(groups, summaries)) + effective_caps: list[int] = [] + + with patch.object(scenario, "_get_attack_technique_factories", return_value=factories): + for limit in range(1, expected_candidate_count + 3): + scenario.set_params_from_args( + args={ + "objective_target": mock_objective_target, + "scenario_techniques": [aggregate], + "include_baseline": False, + "max_attempts_per_objective": limit, + } + ) + estimate = await scenario.get_run_size_estimate_async() + + assert estimate.adaptive_details is not None + expected_effective_cap = min(limit, expected_candidate_count) + effective_caps.append(estimate.adaptive_details.techniques_per_objective_upper_bound) + assert estimate.adaptive_details.candidate_technique_count == expected_candidate_count + assert estimate.adaptive_details.techniques_per_objective_upper_bound == expected_effective_cap + assert estimate.adaptive_details.technique_attempt_count_upper_bound == 21 * expected_effective_cap + + assert effective_caps == [ + *range(1, expected_candidate_count + 1), + expected_candidate_count, + expected_candidate_count, + ] + + def test_exact_duplicate_registered_technique_dedupes_only_itself( + self, + mock_objective_target, + mock_objective_scorer, + ): + scenario = TextAdaptive(objective_scorer=mock_objective_scorer) + technique_class = scenario.get_technique_class() + scenario._scenario_techniques = [ + technique_class("role_play_movie_script"), + technique_class("role_play_movie_script"), + ] + factory = _make_fake_factory() + + with patch.object( + scenario, + "_get_attack_technique_factories", + return_value={"role_play_movie_script": factory}, + ): + techniques = scenario._build_techniques_dict(objective_target=mock_objective_target) + + assert len(techniques) == 1 + factory.create.assert_called_once() + async def test_incompatible_seed_technique_is_filtered_per_objective( self, mock_objective_target, mock_objective_scorer ): @@ -631,11 +800,19 @@ async def test_baseline_emitted_at_index_zero_by_default(self, mock_objective_ta scenario must prepend a baseline atomic attack at index 0. """ groups = {"violence": [_make_seed_group(value="obj", harm_categories=["violence"])]} - with patch.object( - CompoundDatasetAttackConfiguration, - "get_attack_groups_by_dataset_async", - new_callable=AsyncMock, - return_value=groups, + with ( + patch.object( + CompoundDatasetAttackConfiguration, + "get_attack_groups_by_dataset_async", + new_callable=AsyncMock, + return_value=groups, + ), + patch.object( + CompoundDatasetAttackConfiguration, + "resolve_attack_groups_for_estimate_async", + new_callable=AsyncMock, + return_value=(groups, groups), + ), ): scenario = TextAdaptive(objective_scorer=mock_objective_scorer) with warnings.catch_warnings(): @@ -647,3 +824,9 @@ async def test_baseline_emitted_at_index_zero_by_default(self, mock_objective_ta assert scenario._atomic_attacks[0].atomic_attack_name == "baseline", ( f"baseline must be prepended at index 0; got {[a.atomic_attack_name for a in scenario._atomic_attacks]}" ) + estimate = await scenario.get_run_size_estimate_async() + plan = scenario._build_run_plan() + planned_units = sum(len(group.seed_group_ids) for group in plan.atomic_groups) + assert planned_units == 2 + assert estimate.total_attack_count == planned_units + assert [component.count for component in estimate.components] == [1, 1] diff --git a/tests/unit/scenario/test_default_run_size_estimates.py b/tests/unit/scenario/test_default_run_size_estimates.py index 509985104f..c9b1737be0 100644 --- a/tests/unit/scenario/test_default_run_size_estimates.py +++ b/tests/unit/scenario/test_default_run_size_estimates.py @@ -8,26 +8,25 @@ import pytest -from pyrit.executor.attack.core.attack_config import AttackScoringConfig +from pyrit.executor.attack import AttackScoringConfig from pyrit.models import ( AttackSeedGroup, AttackTechniqueSeedGroup, ComponentIdentifier, ScenarioDatasetSummary, + ScenarioRunSizeEstimateCondition, ScenarioRunSizeEstimateStatus, SeedObjective, SeedPrompt, SeedSimulatedConversation, ) from pyrit.prompt_target import PromptTarget -from pyrit.scenario.core import BaselineAttackPolicy, DatasetAttackConfiguration, Scenario, ScenarioTechnique -from pyrit.scenario.scenarios.adaptive.text_adaptive import TextAdaptive -from pyrit.scenario.scenarios.airt.jailbreak import Jailbreak -from pyrit.scenario.scenarios.airt.psychosocial import Psychosocial -from pyrit.scenario.scenarios.benchmark.adversarial import AdversarialBenchmark -from pyrit.scenario.scenarios.foundry.red_team_agent import FoundryComposite, FoundryTechnique, RedTeamAgent -from pyrit.scenario.scenarios.garak.encoding import Encoding -from pyrit.scenario.scenarios.garak.web_injection import WebInjection +from pyrit.scenario import BaselineAttackPolicy, DatasetAttackConfiguration, Scenario, ScenarioTechnique +from pyrit.scenario.scenarios.adaptive import TextAdaptive +from pyrit.scenario.scenarios.airt import Jailbreak, Psychosocial +from pyrit.scenario.scenarios.benchmark import AdversarialBenchmark +from pyrit.scenario.scenarios.foundry import FoundryComposite, FoundryTechnique, RedTeamAgent +from pyrit.scenario.scenarios.garak import Encoding, WebInjection from pyrit.score import TrueFalseScorer @@ -310,7 +309,16 @@ async def test_adaptive_estimate_is_target_conditional_and_does_not_multiply_tec assert estimate.status is ScenarioRunSizeEstimateStatus.Conditional assert estimate.total_attack_count is None + assert estimate.minimum_attack_count == 3 + assert estimate.maximum_attack_count == 6 assert [component.count for component in estimate.components] == [3, 3] + assert estimate.adaptive_details is not None + assert estimate.adaptive_details.objective_count == 3 + assert estimate.adaptive_details.selected_candidate_technique_count == 2 + assert estimate.adaptive_details.candidate_technique_count == 2 + assert estimate.adaptive_details.max_attempts_per_objective == 3 + assert estimate.adaptive_details.techniques_per_objective_upper_bound == 2 + assert estimate.adaptive_details.technique_attempt_count_upper_bound == 6 @pytest.mark.usefixtures("patch_central_database") @@ -342,7 +350,13 @@ async def test_adaptive_estimate_counts_exact_compatible_outer_envelopes_with_ta assert estimate.status is ScenarioRunSizeEstimateStatus.Exact assert estimate.total_attack_count == 2 assert [component.count for component in estimate.components] == [2] - assert "7 selected technique attempts" in estimate.note + assert "Up to 1 selected technique attempts" in estimate.note + assert estimate.adaptive_details is not None + assert estimate.adaptive_details.objective_count == 2 + assert estimate.adaptive_details.selected_candidate_technique_count == 2 + assert estimate.adaptive_details.candidate_technique_count == 1 + assert estimate.adaptive_details.techniques_per_objective_upper_bound == 1 + assert estimate.adaptive_details.technique_attempt_count_upper_bound == 2 scenario.set_params_from_args(args={"include_baseline": False}) estimate_without_target = await scenario.get_run_size_estimate_async() @@ -351,6 +365,102 @@ async def test_adaptive_estimate_counts_exact_compatible_outer_envelopes_with_ta assert estimate_without_target.total_attack_count is None +@pytest.mark.usefixtures("patch_central_database") +async def test_adaptive_estimate_caps_attempts_below_candidate_pool() -> None: + """A lower configured max-attempt cap bounds each objective before pool size.""" + with patch.object(TextAdaptive, "get_technique_class", return_value=_TwoTechniqueDefault): + scenario = TextAdaptive(objective_scorer=_scorer()) + target = MagicMock(spec=PromptTarget) + scenario.set_params_from_args( + args={ + "objective_target": target, + "include_baseline": False, + "max_attempts_per_objective": 1, + } + ) + scenario._resolve_dataset_groups_for_estimate_async = AsyncMock(return_value=_resolved_groups({"adaptive": 3})) + dispatcher = MagicMock() + dispatcher.compatible_techniques.side_effect = [["one", "two"], ["one"], ["two"]] + + with ( + patch.object( + scenario, + "_build_techniques_dict", + return_value={"one": MagicMock(), "two": MagicMock()}, + ), + patch( + "pyrit.scenario.scenarios.adaptive.adaptive_scenario.AdaptiveTechniqueDispatcher", + return_value=dispatcher, + ), + ): + estimate = await scenario.get_run_size_estimate_async() + + assert estimate.adaptive_details is not None + assert estimate.adaptive_details.objective_count == 3 + assert estimate.adaptive_details.selected_candidate_technique_count == 2 + assert estimate.adaptive_details.candidate_technique_count == 2 + assert estimate.adaptive_details.max_attempts_per_objective == 1 + assert estimate.adaptive_details.techniques_per_objective_upper_bound == 1 + assert estimate.adaptive_details.technique_attempt_count_upper_bound == 3 + + +@pytest.mark.usefixtures("patch_central_database") +async def test_adaptive_conditional_attempt_bound_uses_launch_wide_objective_maximum() -> None: + """A sampled compatibility preview cannot understate a capped launch's attempt bound.""" + with patch.object(TextAdaptive, "get_technique_class", return_value=_TwoTechniqueDefault): + scenario = TextAdaptive(objective_scorer=_scorer()) + target = MagicMock(spec=PromptTarget) + scenario.set_params_from_args( + args={ + "objective_target": target, + "include_baseline": False, + "max_attempts_per_objective": 3, + } + ) + + async def resolve_groups() -> tuple[dict[str, list[AttackSeedGroup]], list[ScenarioDatasetSummary]]: + scenario._estimate_has_binding_size_cap = True + return _resolved_groups({"adaptive": 3}) + + scenario._resolve_dataset_groups_for_estimate_async = AsyncMock(side_effect=resolve_groups) + dispatcher = MagicMock() + dispatcher.compatible_techniques.side_effect = [["one"], [], []] + + with ( + patch.object( + scenario, + "_build_techniques_dict", + return_value={"one": MagicMock(), "two": MagicMock()}, + ), + patch( + "pyrit.scenario.scenarios.adaptive.adaptive_scenario.AdaptiveTechniqueDispatcher", + return_value=dispatcher, + ), + ): + estimate = await scenario.get_run_size_estimate_async() + + assert estimate.status is ScenarioRunSizeEstimateStatus.Conditional + assert estimate.minimum_attack_count is None + assert estimate.maximum_attack_count == 3 + assert estimate.components[0].count == 1 + assert estimate.adaptive_details is not None + assert estimate.adaptive_details.objective_count == 3 + assert estimate.adaptive_details.techniques_per_objective_upper_bound == 2 + assert estimate.adaptive_details.technique_attempt_count_upper_bound == 6 + + +@pytest.mark.usefixtures("patch_central_database") +async def test_adaptive_estimate_rejects_non_positive_attempt_limit_without_target() -> None: + """Invalid attempt limits fail explicitly before constructing estimate metadata.""" + with patch.object(TextAdaptive, "get_technique_class", return_value=_TwoTechniqueDefault): + scenario = TextAdaptive(objective_scorer=_scorer()) + scenario.set_params_from_args(args={"include_baseline": False, "max_attempts_per_objective": 0}) + scenario._resolve_dataset_groups_for_estimate_async = AsyncMock(return_value=_resolved_groups({"adaptive": 3})) + + with pytest.raises(ValueError, match="max_attempts_per_objective must be >= 1, got 0"): + await scenario.get_run_size_estimate_async() + + @pytest.mark.usefixtures("patch_central_database") async def test_jailbreak_estimate_exposes_template_attempt_and_target_capability_axes() -> None: """Jailbreak reports guaranteed inline work separately from conditional system delivery.""" @@ -362,6 +472,11 @@ async def test_jailbreak_estimate_exposes_template_attempt_and_target_capability assert estimate.status is ScenarioRunSizeEstimateStatus.Conditional assert estimate.total_attack_count is None + assert estimate.minimum_attack_count == 12 + assert estimate.maximum_attack_count == 20 + assert estimate.components[2].condition is ScenarioRunSizeEstimateCondition.TargetCapabilities + assert estimate.model_dump(mode="json")["minimum_attack_count"] == 12 + assert estimate.model_dump(mode="json")["maximum_attack_count"] == 20 assert [component.count for component in estimate.components] == [4, 8, 8] assert [factor.count for factor in estimate.components[1].factors] == [4, 2, 1, 1] assert "2 template(s) x 4 selected logical seed group(s) x 1 selected" in estimate.note