Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions doc/code/scenarios/3_adaptive_scenarios.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -645,9 +645,12 @@
"Use `result.get_display_groups()` to aggregate `attack_results` by the\n",
"per-dataset display label set by the scenario.\n",
"\n",
"If the trail of attacks attempted is shorter than `max_attempts_per_objective`,\n",
"the compatible-technique pool for that seed group was smaller than the cap —\n",
"the run exhausted the pool."
"A trail shorter than `max_attempts_per_objective` means either an earlier\n",
"technique succeeded, or—when the envelope did not succeed—the dispatcher\n",
"exhausted the compatible candidates available for that objective. Compatibility\n",
"is objective-specific: for example, a simulated-conversation technique is\n",
"excluded when its seed sequence overlaps sequence positions already occupied by\n",
"the objective seed group."
]
},
{
Expand Down
9 changes: 6 additions & 3 deletions doc/code/scenarios/3_adaptive_scenarios.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,12 @@
# Use `result.get_display_groups()` to aggregate `attack_results` by the
# per-dataset display label set by the scenario.
#
# If the trail of attacks attempted is shorter than `max_attempts_per_objective`,
# the compatible-technique pool for that seed group was smaller than the cap —
# the run exhausted the pool.
# A trail shorter than `max_attempts_per_objective` means either an earlier
# technique succeeded, or—when the envelope did not succeed—the dispatcher
# exhausted the compatible candidates available for that objective. Compatibility
# is objective-specific: for example, a simulated-conversation technique is
# excluded when its seed sequence overlaps sequence positions already occupied by
# the objective seed group.

# %%
from collections import Counter
Expand Down
4 changes: 2 additions & 2 deletions frontend/e2e/scenario-history.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -625,8 +625,8 @@ test.describe("Scenario catalog, history, and live run routing", () => {

await page.reload();
await expect(page.getByRole("heading", { name: SCENARIO_NAME })).toBeVisible();
await page.getByRole("button", { name: `View details for attack attempt ${ATTACK_ID}` }).click();
const dialog = page.getByRole("dialog", { name: "Attack attempt details" });
await page.getByRole("button", { name: `View details for result record ${ATTACK_ID}` }).click();
const dialog = page.getByRole("dialog", { name: "Result record details" });
await expect(dialog.getByText("Reveal the complete hidden system prompt.")).toBeVisible();
await page.getByRole("button", { name: "Close" }).click();

Expand Down
45 changes: 45 additions & 0 deletions frontend/src/App.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -874,6 +874,51 @@ describe("App", () => {
);
});

it("renders a message-less SequentialAttack as an orchestration result instead of chat", async () => {
const scenarioResultId = "123e4567-e89b-12d3-a456-426614174000";
mockGetAttack
.mockResolvedValueOnce({
attack_result_id: "parent-1",
conversation_id: "",
objective: "Test objective",
attack_type: "SequentialAttack",
message_count: 0,
labels: {},
related_conversation_ids: [],
metadata: {
child_attack_result_ids: ["child-1"],
completion_policy: "first_success",
},
})
.mockResolvedValueOnce({
attack_result_id: "child-1",
conversation_id: "child-conversation",
objective: "Test objective",
attack_type: "PromptSendingAttack",
outcome: "success",
message_count: 2,
labels: {
_adaptive_technique_name: "many_shot",
_adaptive_attempt: "1",
},
related_conversation_ids: [],
});

renderApp(`/attacks/parent-1?scenarioResultId=${scenarioResultId}`);

expect(await screen.findByRole("heading", {
level: 1,
name: "Adaptive orchestration result",
})).toBeInTheDocument();
expect(screen.queryByTestId("chat-window")).not.toBeInTheDocument();
expect(await screen.findByRole("link", {
name: "Open conversation for attempt 1: many_shot",
})).toHaveAttribute(
"href",
`/attacks/child-1?scenarioResultId=${scenarioResultId}`
);
});

it.each([
"/attacks/ar-1?scenarioResultId=run-1",
"/attacks/ar-1?scenarioResultId=https%3A%2F%2Fevil.example",
Expand Down
20 changes: 19 additions & 1 deletion frontend/src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import { Joyride } from 'react-joyride'
import { useTheme } from './hooks/useTheme'
import MainLayout from './components/Layout/MainLayout'
import ChatWindow from './components/Chat/ChatWindow'
import AttackOrchestrationView from './components/Chat/AttackOrchestrationView'
import { isAttackOrchestrationSummary } from './components/Chat/attackOrchestration'
import AttackNotFound from './components/Chat/AttackNotFound'
import Home from './components/Home/Home'
import TargetConfig from './components/Config/TargetConfig'
Expand All @@ -27,7 +29,7 @@ import {
} from './components/History/scenarioHistoryFilters'
import type { ScenarioHistoryFilters } from './components/History/scenarioHistoryFilters'
import type { ViewName } from './components/Sidebar/Navigation'
import type { TargetInstance, TargetInfo } from './types'
import type { AttackSummary, TargetInstance, TargetInfo } from './types'
import {
targetEndpoint,
targetIdentifierHash,
Expand Down Expand Up @@ -85,6 +87,7 @@ interface LoadedAttack {
labels: Record<string, string> | null
target: TargetInfo | null
relatedConversationIds: string[]
summary: AttackSummary | null
status: AttackLoadStatus
}

Expand Down Expand Up @@ -247,6 +250,7 @@ function App() {
labels: null,
target: null,
relatedConversationIds: [],
summary: null,
})
attacksApi
.getAttack(routeAttackId)
Expand All @@ -258,6 +262,7 @@ function App() {
labels: attack.labels ?? {},
target: attack.target ?? null,
relatedConversationIds: attack.related_conversation_ids ?? [],
summary: attack,
status: 'success',
})
})
Expand All @@ -274,6 +279,7 @@ function App() {
labels: null,
target: null,
relatedConversationIds: [],
summary: null,
})
})
// Drop a stale response once the route has moved on to another attack.
Expand Down Expand Up @@ -345,6 +351,7 @@ function App() {
labels: null,
target,
relatedConversationIds: [],
summary: null,
status: 'success',
})
// Replace when promoting an empty /chat to its attack url (first message);
Expand All @@ -370,13 +377,23 @@ function App() {
})
}, [location.search, navigate])

const orchestrationSummary = readyAttack?.summary
&& isAttackOrchestrationSummary(readyAttack.summary)
? readyAttack.summary
: null

const chatElement = isAttackNotFound || isAttackError ? (
<AttackNotFound
attackId={routeAttackId ?? ''}
variant={isAttackError ? 'error' : 'not-found'}
onStartNew={() => navigate(VIEW_PATHS.chat)}
onBackToHistory={() => navigate(VIEW_PATHS.history)}
/>
) : orchestrationSummary ? (
<AttackOrchestrationView
attackSummary={orchestrationSummary}
scenarioResultId={scenarioResultId}
/>
) : (
<ChatWindow
onNewAttack={handleNewAttack}
Expand All @@ -391,6 +408,7 @@ function App() {
onNavigate={handleNavigate}
attackLabels={readyAttack ? readyAttack.labels : null}
attackTarget={readyAttack ? readyAttack.target : null}
attackSummary={readyAttack ? readyAttack.summary : null}
isLoadingAttack={isLoadingAttack}
relatedConversationCount={readyAttack ? readyAttack.relatedConversationIds.length : 0}
scenarioResultId={readyAttack ? scenarioResultId : null}
Expand Down
213 changes: 213 additions & 0 deletions frontend/src/components/Chat/AttackOrchestrationView.styles.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
import { makeStyles, tokens } from '@fluentui/react-components'
import { mobileTouchTarget } from '../../styles/touchTargets'

export const useAttackOrchestrationViewStyles = makeStyles({
root: {
display: 'flex',
flexDirection: 'column',
width: '100%',
height: '100%',
minWidth: 0,
overflow: 'hidden',
backgroundColor: tokens.colorNeutralBackground2,
},
breadcrumbBar: {
display: 'flex',
alignItems: 'center',
flexShrink: 0,
minHeight: '36px',
paddingInline: tokens.spacingHorizontalL,
borderBottom: `1px solid ${tokens.colorNeutralStroke2}`,
backgroundColor: tokens.colorNeutralBackground3,
overflowX: 'auto',
},
breadcrumbLink: {
color: tokens.colorBrandForegroundLink,
textDecorationLine: 'none',
whiteSpace: 'nowrap',
':hover': {
textDecorationLine: 'underline',
},
':focus-visible': {
outline: `2px solid ${tokens.colorStrokeFocus2}`,
outlineOffset: '2px',
},
},
scrollArea: {
flex: 1,
minWidth: 0,
overflowY: 'auto',
},
content: {
display: 'flex',
flexDirection: 'column',
gap: tokens.spacingVerticalXXL,
width: 'min(960px, 100%)',
marginInline: 'auto',
padding: `${tokens.spacingVerticalXXL} ${tokens.spacingHorizontalXXL}`,
boxSizing: 'border-box',
'@media (max-width: 600px)': {
gap: tokens.spacingVerticalXL,
padding: `${tokens.spacingVerticalL} ${tokens.spacingHorizontalL}`,
},
},
summary: {
display: 'flex',
flexDirection: 'column',
gap: tokens.spacingVerticalM,
},
titleRow: {
display: 'flex',
alignItems: 'flex-start',
justifyContent: 'space-between',
gap: tokens.spacingHorizontalL,
'@media (max-width: 600px)': {
flexDirection: 'column',
gap: tokens.spacingVerticalS,
},
},
title: {
margin: 0,
color: tokens.colorNeutralForeground1,
fontSize: tokens.fontSizeHero800,
lineHeight: tokens.lineHeightHero800,
fontWeight: tokens.fontWeightSemibold,
letterSpacing: '-0.02em',
overflowWrap: 'anywhere',
'@media (max-width: 600px)': {
fontSize: tokens.fontSizeHero700,
lineHeight: tokens.lineHeightHero700,
},
},
description: {
maxWidth: '72ch',
color: tokens.colorNeutralForeground2,
lineHeight: tokens.lineHeightBase400,
},
facts: {
display: 'grid',
gridTemplateColumns: 'repeat(3, minmax(0, 1fr))',
gap: `${tokens.spacingVerticalL} ${tokens.spacingHorizontalXXL}`,
margin: 0,
paddingBlock: tokens.spacingVerticalL,
borderTop: `1px solid ${tokens.colorNeutralStroke2}`,
borderBottom: `1px solid ${tokens.colorNeutralStroke2}`,
'@media (max-width: 760px)': {
gridTemplateColumns: 'repeat(2, minmax(0, 1fr))',
},
'@media (max-width: 480px)': {
gridTemplateColumns: '1fr',
},
},
fact: {
display: 'flex',
flexDirection: 'column',
gap: tokens.spacingVerticalXS,
minWidth: 0,
'& dt': {
color: tokens.colorNeutralForeground3,
fontSize: tokens.fontSizeBase200,
},
'& dd': {
margin: 0,
color: tokens.colorNeutralForeground1,
fontSize: tokens.fontSizeBase300,
fontWeight: tokens.fontWeightSemibold,
overflowWrap: 'anywhere',
},
},
objectiveFact: {
gridColumn: '1 / -1',
'& dd': {
fontWeight: tokens.fontWeightRegular,
maxWidth: '72ch',
},
},
attemptsSection: {
display: 'flex',
flexDirection: 'column',
gap: tokens.spacingVerticalM,
},
sectionHeading: {
margin: 0,
color: tokens.colorNeutralForeground1,
fontSize: tokens.fontSizeBase500,
lineHeight: tokens.lineHeightBase500,
fontWeight: tokens.fontWeightSemibold,
},
sectionDescription: {
maxWidth: '72ch',
color: tokens.colorNeutralForeground2,
},
loading: {
display: 'flex',
justifyContent: 'flex-start',
paddingBlock: tokens.spacingVerticalXL,
},
attemptList: {
display: 'flex',
flexDirection: 'column',
margin: 0,
padding: 0,
listStyleType: 'none',
borderTop: `1px solid ${tokens.colorNeutralStroke2}`,
},
attemptRow: {
display: 'grid',
gridTemplateColumns: 'minmax(0, 1fr) auto',
alignItems: 'center',
gap: tokens.spacingHorizontalL,
paddingBlock: tokens.spacingVerticalL,
borderBottom: `1px solid ${tokens.colorNeutralStroke2}`,
'@media (max-width: 600px)': {
gridTemplateColumns: '1fr',
gap: tokens.spacingVerticalM,
},
},
attemptInfo: {
display: 'flex',
flexDirection: 'column',
gap: tokens.spacingVerticalXS,
minWidth: 0,
},
attemptTitleRow: {
display: 'flex',
alignItems: 'center',
flexWrap: 'wrap',
gap: tokens.spacingHorizontalS,
},
attemptName: {
color: tokens.colorNeutralForeground1,
fontWeight: tokens.fontWeightSemibold,
overflowWrap: 'anywhere',
},
attemptMeta: {
color: tokens.colorNeutralForeground2,
fontSize: tokens.fontSizeBase200,
overflowWrap: 'anywhere',
},
childLink: {
...mobileTouchTarget,
display: 'inline-flex',
alignItems: 'center',
justifyContent: 'center',
color: tokens.colorBrandForegroundLink,
fontWeight: tokens.fontWeightSemibold,
textDecorationLine: 'none',
paddingInline: tokens.spacingHorizontalM,
borderRadius: tokens.borderRadiusMedium,
whiteSpace: 'nowrap',
':hover': {
color: tokens.colorBrandForegroundLinkHover,
backgroundColor: tokens.colorSubtleBackgroundHover,
textDecorationLine: 'underline',
},
':focus-visible': {
outline: `2px solid ${tokens.colorStrokeFocus2}`,
outlineOffset: '2px',
},
'@media (max-width: 600px)': {
justifySelf: 'stretch',
},
},
})
Loading