Skip to content

Refine Scenario configuration and sizing #2377

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Roman Lutz (romanlutz) wants to merge 1 commit into from
+6,110 −914
Draft

Refine Scenario configuration and sizing #2377

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 54 additions & 42 deletions frontend/e2e/api.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,31 @@
import { test, expect } from "@playwright/test";
import type { APIRequestContext } from "@playwright/test";

// API tests go through the Vite dev server proxy (/api -> configured backend)
// rather than hitting the backend directly, so they work as soon as
// Playwright's webServer is ready.

test.describe("API Health Check", () => {
// The backend may still be starting when Vite is already up.
// Poll the health endpoint through the proxy until the backend is ready.
test.beforeAll(async ({ request }) => {
const maxWait = 30_000;
const interval = 1_000;
const start = Date.now();
while (Date.now() - start < maxWait) {
try {
const resp = await request.get("/api/health", { timeout: 2_000 });
if (resp.ok()) return;
} catch {
// Backend not ready yet
async function waitForBackend(request: APIRequestContext): Promise<void> {
const maxWait = 30_000;
const interval = 1_000;
const start = Date.now();
while (Date.now() - start < maxWait) {
try {
const response = await request.get("/api/health", { timeout: 2_000 });
if (response.ok()) {
return;
}
await new Promise((r) => setTimeout(r, interval));
} catch {
// Backend not ready yet
}
throw new Error("Backend did not become healthy within 30 seconds");
await new Promise((resolve) => setTimeout(resolve, interval));
}
throw new Error("Backend did not become healthy within 30 seconds");
}

test.describe("API Health Check", () => {
test.beforeAll(async ({ request }) => {
await waitForBackend(request);
});

test("should have healthy backend API @seeded", async ({ request }) => {
Expand All @@ -38,24 +43,12 @@ test.describe("API Health Check", () => {
const data = await response.json();
expect(data).toBeDefined();
});

});

test.describe("Targets API", () => {
test.beforeAll(async ({ request }) => {
// Wait for backend readiness
const maxWait = 30_000;
const interval = 1_000;
const start = Date.now();
while (Date.now() - start < maxWait) {
try {
const resp = await request.get("/api/health", { timeout: 2_000 });
if (resp.ok()) return;
} catch {
// Backend not ready yet
}
await new Promise((r) => setTimeout(r, interval));
}
throw new Error("Backend did not become healthy within 30 seconds");
await waitForBackend(request);
});

test("should list targets @seeded", async ({ request }) => {
Expand Down Expand Up @@ -103,19 +96,7 @@ test.describe("Targets API", () => {

test.describe("Attacks API", () => {
test.beforeAll(async ({ request }) => {
const maxWait = 30_000;
const interval = 1_000;
const start = Date.now();
while (Date.now() - start < maxWait) {
try {
const resp = await request.get("/api/health", { timeout: 2_000 });
if (resp.ok()) return;
} catch {
// Backend not ready yet
}
await new Promise((r) => setTimeout(r, interval));
}
throw new Error("Backend did not become healthy within 30 seconds");
await waitForBackend(request);
});

test("should list attacks @seeded", async ({ request }) => {
Expand All @@ -124,6 +105,37 @@ test.describe("Attacks API", () => {
});
});

test.describe("Scenarios API", () => {
test.beforeAll(async ({ request }) => {
await waitForBackend(request);
});

test("should expose scenario catalog details and queue state @seeded", async ({ request }) => {
test.setTimeout(90_000);
const catalogResponse = await request.get("/api/scenarios/catalog?limit=200");
expect(catalogResponse.ok()).toBe(true);
const catalog = await catalogResponse.json();
expect(catalog.items.length).toBeGreaterThan(0);

const scenarioName = catalog.items[0].scenario_name as string;
const detailResponse = await request.get(`/api/scenarios/catalog/${encodeURIComponent(scenarioName)}`);
expect(detailResponse.ok()).toBe(true);
const detail = await detailResponse.json();
expect(detail.scenario_name).toBe(scenarioName);
expect(detail.dataset_size_limit).toEqual(expect.objectContaining({
default_scope: expect.any(String),
override_scope: expect.any(String),
}));

const queueResponse = await request.get("/api/scenarios/runs/queue");
expect(queueResponse.ok()).toBe(true);
await expect(queueResponse.json()).resolves.toEqual(expect.objectContaining({
revision: expect.any(Number),
queued: expect.any(Array),
}));
});
});

test.describe("Error Handling", () => {
test("should display UI when backend is slow", async ({ page }) => {
// Intercept and delay API calls
Expand Down
95 changes: 87 additions & 8 deletions frontend/e2e/scenario-history.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ const configuredEstimate = {
version: 1,
status: "exact",
total_attack_count: 8,
minimum_attack_count: 8,
maximum_attack_count: 8,
condition: null,
components: [{
label: "Prompt sending",
count: 8,
Expand All @@ -49,6 +52,7 @@ const configuredEstimate = {
note: null,
}],
datasets: [datasetSummary],
adaptive_details: null,
note: "The backend total is authoritative.",
retries_included: false,
};
Expand All @@ -68,6 +72,11 @@ const catalogScenario = {
},
all_techniques: ["prompt_sending", "jailbreak_system_prompt", "flip"],
default_datasets: ["harmbench"],
dataset_size_limit: {
default_scope: "per_dataset",
default_count: 4,
override_scope: "per_dataset",
},
default_dataset_summaries: [datasetSummary],
baseline_policy: "enabled",
include_baseline_by_default: false,
Expand Down Expand Up @@ -104,6 +113,9 @@ const catalogScenario = {
version: 1,
status: "exact",
total_attack_count: 16,
minimum_attack_count: 16,
maximum_attack_count: 16,
condition: null,
components: [{
label: "Default attacks",
count: 16,
Expand All @@ -116,6 +128,7 @@ const catalogScenario = {
note: null,
}],
datasets: [datasetSummary],
adaptive_details: null,
note: "Retries and internal turns are excluded.",
retries_included: false,
},
Expand Down Expand Up @@ -197,6 +210,8 @@ const progressAttempt = {
timestamp: "2026-08-07T00:00:30Z",
total_retries: 1,
retries: [],
result_kind: "attack",
technique_name: "prompt_sending",
};

interface ScenarioMocks {
Expand Down Expand Up @@ -236,6 +251,14 @@ async function mockScenarioAPIs(page: Page): Promise<ScenarioMocks> {
});
});

await page.route(/\/api\/datasets(?:\?|$)/, async (route) => {
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify({ items: [{ name: "harmbench" }] }),
});
});

await page.route(new RegExp(`/api/scenarios/catalog/${SCENARIO_NAME.replace(".", "\\.")}/estimate$`), async (route) => {
const request = route.request().postDataJSON() as Record<string, unknown>;
estimateRequests.push(request);
Expand Down Expand Up @@ -396,15 +419,20 @@ async function mockScenarioAPIs(page: Page): Promise<ScenarioMocks> {

async function configurePromptSendingRun(page: Page): Promise<void> {
await expect(page.getByTestId("scenario-target-select")).toHaveValue("test-target");
await page.getByTestId("technique-prompt_sending").click();
await page.getByTestId("technique-mode-custom").click();
await expect(page.getByTestId("technique-prompt_sending")).toBeChecked();
await expect(page.getByTestId("technique-jailbreak_system_prompt")).toBeChecked();
await page.getByTestId("technique-jailbreak_system_prompt").click();
await page.getByTestId("scenario-param-num_jailbreaks").fill("2");
await page.getByTestId("scenario-param-num_jailbreak_attempts").fill("1");
await expect(page.getByTestId("baseline-checkbox")).not.toBeChecked();
await expect(page.getByText("8 planned attacks")).toBeVisible();
await expect(page.getByRole("group", {
name: "1 technique multiplied by 4 objectives multiplied by 2 jailbreak templates multiplied by 1 attempt equals 8 planned attacks.",
})).toBeVisible();
}

test.describe("Scenario catalog, history, and live run routing", () => {
test("renders the semantic catalog, full metadata, safe MyST, and both sidebar destinations", async ({ page }) => {
test("opens the Configure page from the semantic launch index with complete safe metadata", async ({ page }) => {
await mockScenarioAPIs(page);
await page.goto("/scenarios");

Expand All @@ -423,17 +451,52 @@ test.describe("Scenario catalog, history, and live run routing", () => {
]);
await expect(page.getByTitle("Scenarios")).toHaveAttribute("aria-current", "page");
await expect(page.getByRole("table", { name: "Registered scenarios" })).toBeVisible();
await expect(page.getByRole("columnheader", { name: "Default run size" })).toBeVisible();
await expect(page.getByRole("columnheader")).toHaveText([
"Scenario / purpose",
"Configure",
"Default dataset size",
"Default techniques",
"Default run size",
]);

const row = page.getByTestId(`scenario-card-${SCENARIO_NAME}`);
await row.getByRole("button", { name: "Configure run" }).click();
const cells = row.getByRole("cell");
await expect(cells).toHaveCount(5);
const configureButton = cells.nth(1).getByRole("button", { name: "Configure run" });
await expect(configureButton).toBeVisible();
const [scenarioCellBox, configureCellBox, datasetCellBox] = await Promise.all([
cells.nth(0).boundingBox(),
cells.nth(1).boundingBox(),
cells.nth(2).boundingBox(),
]);
expect(scenarioCellBox).not.toBeNull();
expect(configureCellBox).not.toBeNull();
expect(datasetCellBox).not.toBeNull();
expect(configureCellBox!.x).toBeGreaterThan(scenarioCellBox!.x);
expect(configureCellBox!.x).toBeLessThan(datasetCellBox!.x);
await expect(page.getByRole("button", { name: /show details|hide details/i })).toHaveCount(0);

await configureButton.click();
await expect(page).toHaveURL(`/scenarios/${SCENARIO_NAME}`);
await expect(page.getByRole("heading", { name: SCENARIO_NAME, level: 1 })).toBeVisible();
await expect(page.getByText("Jailbreak · v4")).toBeVisible();
const description = page.getByTestId("scenario-detail-description");
await expect(description.getByText("dataset")).toHaveCSS("font-weight", /^(600|700)$/);
await expect(description.locator("code").filter({ hasText: "num_jailbreaks" })).toBeVisible();
await expect(description.locator("img")).toHaveCount(0);
await expect(description).toContainText(RAW_IMAGE_HTML);
await expect(page.getByRole("radio", { name: /Recommended \(default\).*2 techniques/ })).toBeChecked();
await expect(page.getByRole("radio", { name: /Easy.*1 technique/ })).toBeVisible();
await expect(page.getByRole("radio", { name: "Custom" })).toBeVisible();
const members = page.getByTestId("selected-technique-set-members");
await expect(members.getByText("prompt_sending")).toBeVisible();
await expect(members.getByText("jailbreak_system_prompt")).toBeVisible();
const preview = page.getByRole("complementary", { name: "Run preview" });
await expect(preview.getByText("Jailbreak templates: 2")).toBeVisible();
await expect(preview.getByRole("group", {
name: "2 techniques multiplied by 4 objectives multiplied by 2 jailbreak templates equals 16 planned attacks.",
})).toBeVisible();
await expect(page.getByText("Include direct baseline comparison")).toBeVisible();
await expect(page.getByText(/Also send each selected objective directly/)).toBeVisible();

await page.getByTitle("Scenario History").click();
await expect(page).toHaveURL("/scenario-history");
Expand Down Expand Up @@ -471,7 +534,9 @@ test.describe("Scenario catalog, history, and live run routing", () => {
const requests = mocks.getEstimateRequests();
return requests[requests.length - 1];
}).toEqual(expectedEstimateRequest);
await expect(preview.getByText("Prompt sending: 2 jailbreak templates × 4 selected seed groups × 1 concrete techniques × 1 attempts = 8")).toBeVisible();
await expect(preview.getByRole("group", {
name: "1 technique multiplied by 4 objectives multiplied by 2 jailbreak templates multiplied by 1 attempt equals 8 planned attacks.",
})).toBeVisible();
await expect(preview).not.toContainText("context_compliance");

await page.getByTestId("launch-scenario-btn").click();
Expand Down Expand Up @@ -508,7 +573,21 @@ test.describe("Scenario catalog, history, and live run routing", () => {
const client = await page.context().newCDPSession(page);
await client.send("Emulation.setTouchEmulationEnabled", { enabled: true, maxTouchPoints: 1 });
await page.setViewportSize({ width: 390, height: 844 });
await page.goto(`/scenarios/${SCENARIO_NAME}`);
await page.goto("/scenarios");
const catalogRow = page.getByTestId(`scenario-card-${SCENARIO_NAME}`);
const configureButton = catalogRow.getByRole("button", { name: "Configure run" });
await expect(catalogRow).toBeVisible();
expect(await catalogRow.getByRole("cell").allInnerTexts()).toEqual([
expect.stringContaining("Scenario / purpose"),
expect.stringContaining("Configure"),
expect.stringContaining("Default dataset size"),
expect.stringContaining("Default techniques"),
expect.stringContaining("Default run size"),
]);
expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(390);
expect((await configureButton.boundingBox())?.height).toBeGreaterThanOrEqual(44);
await configureButton.press("Enter");
await expect(page).toHaveURL(`/scenarios/${SCENARIO_NAME}`);
await configurePromptSendingRun(page);

const formBox = await page.getByRole("form", { name: "Scenario run configuration" }).boundingBox();
Expand Down
Loading