From 332c82bab0c9756ea025398bd6d4d1823484281e Mon Sep 17 00:00:00 2001 From: jbolor21 <86250273+jbolor21@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:21:54 -0700 Subject: [PATCH 1/8] initial commit adding scores per message --- frontend/src/App.test.tsx | 5 ++ frontend/src/App.tsx | 6 ++ frontend/src/components/Chat/ChatWindow.tsx | 5 ++ .../src/components/Chat/MessageList.styles.ts | 40 +++++++++ .../src/components/Chat/MessageList.test.tsx | 48 +++++++++++ frontend/src/components/Chat/MessageList.tsx | 64 ++++++++++++++- .../components/Chat/ObjectiveHeader.styles.ts | 41 ++++++++++ .../components/Chat/ObjectiveHeader.test.tsx | 81 +++++++++++++++++++ .../src/components/Chat/ObjectiveHeader.tsx | 66 +++++++++++++++ .../components/History/AttackHistory.test.tsx | 2 + .../components/History/AttackTable.test.tsx | 3 + frontend/src/components/Home/Home.test.tsx | 1 + frontend/src/types/index.ts | 3 + frontend/src/utils/messageMapper.test.ts | 51 ++++++++++++ frontend/src/utils/messageMapper.ts | 19 +++++ 15 files changed, 433 insertions(+), 2 deletions(-) create mode 100644 frontend/src/components/Chat/ObjectiveHeader.styles.ts create mode 100644 frontend/src/components/Chat/ObjectiveHeader.test.tsx create mode 100644 frontend/src/components/Chat/ObjectiveHeader.tsx diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 188850bca1..bf2ef8e537 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -120,6 +120,7 @@ jest.mock("./components/Chat/ChatWindow", () => { conversationId, activeConversationId, attackTarget, + objective, onConversationCreated, onSelectConversation, labels, @@ -130,6 +131,7 @@ jest.mock("./components/Chat/ChatWindow", () => { conversationId: string | null; activeConversationId: string | null; attackTarget?: { identifier_hash?: string | null } | null; + objective?: string; onConversationCreated: (attackResultId: string, conversationId: string) => void; onSelectConversation: (convId: string) => void; labels: Record; @@ -141,6 +143,7 @@ jest.mock("./components/Chat/ChatWindow", () => { {activeConversationId ?? "none"} {activeTarget ? "yes" : "no"} {attackTarget?.identifier_hash ?? "none"} + {objective ?? ""} {labels.operator ?? ""} {JSON.stringify(labels)} + + +
+ Score details +
+ Value + {score.score_value} +
+
+ Type + {score.score_type} +
+
+ Scorer + {score.scorer_type} +
+ {categories.length > 0 && ( +
+ Category + {categories.join(', ')} +
+ )} + {score.score_rationale && ( +
+ Rationale + {score.score_rationale} +
+ )} +
+ + + ) +} + /** * If the trimmed text is a JSON object or array, return a 2-space pretty-printed * version of it; otherwise return null. Used to render structured assistant @@ -455,7 +512,10 @@ export default function MessageList({ messages, onCopyToInput, onCopyToNewConver
{timestamp} - {message.role} +
+ {message.role} + {message.score && } +
diff --git a/frontend/src/components/Chat/ObjectiveHeader.styles.ts b/frontend/src/components/Chat/ObjectiveHeader.styles.ts new file mode 100644 index 0000000000..0fadfb0566 --- /dev/null +++ b/frontend/src/components/Chat/ObjectiveHeader.styles.ts @@ -0,0 +1,41 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +export const useObjectiveHeaderStyles = makeStyles({ + root: { + flexShrink: 0, + display: 'flex', + flexDirection: 'row', + alignItems: 'baseline', + columnGap: tokens.spacingHorizontalS, + padding: `${tokens.spacingVerticalS} ${tokens.spacingHorizontalL}`, + backgroundColor: tokens.colorNeutralBackground2, + borderBottom: `1px solid ${tokens.colorNeutralStroke1}`, + borderLeft: `3px solid ${tokens.colorBrandStroke1}`, + }, + label: { + flexShrink: 0, + }, + content: { + flexGrow: 1, + minWidth: 0, + color: tokens.colorNeutralForeground1, + fontSize: tokens.fontSizeBase300, + }, + contentCollapsed: { + whiteSpace: 'nowrap', + overflow: 'hidden', + textOverflow: 'ellipsis', + }, + contentExpanded: { + whiteSpace: 'pre-wrap', + wordBreak: 'break-word', + maxHeight: '30vh', + overflowY: 'auto', + }, + toggle: { + flexShrink: 0, + minWidth: 'auto', + whiteSpace: 'nowrap', + color: tokens.colorBrandForeground1, + }, +}) diff --git a/frontend/src/components/Chat/ObjectiveHeader.test.tsx b/frontend/src/components/Chat/ObjectiveHeader.test.tsx new file mode 100644 index 0000000000..7afe4fc05d --- /dev/null +++ b/frontend/src/components/Chat/ObjectiveHeader.test.tsx @@ -0,0 +1,81 @@ +import { render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' + +import ObjectiveHeader from './ObjectiveHeader' + +const TestWrapper: React.FC<{ children: React.ReactNode }> = ({ children }) => ( + {children} +) + +function mockOverflow(scrollWidth: number, clientWidth: number): void { + Object.defineProperty(HTMLElement.prototype, 'scrollWidth', { configurable: true, get: () => scrollWidth }) + Object.defineProperty(HTMLElement.prototype, 'clientWidth', { configurable: true, get: () => clientWidth }) +} + +describe('ObjectiveHeader', () => { + afterEach(() => { + delete (HTMLElement.prototype as { scrollWidth?: number }).scrollWidth + delete (HTMLElement.prototype as { clientWidth?: number }).clientWidth + }) + + it('renders nothing when the objective is empty', () => { + render( + + + , + ) + + expect(screen.queryByTestId('objective-header')).not.toBeInTheDocument() + }) + + it('renders the label and objective text', () => { + render( + + + , + ) + + expect(screen.getByText('Objective')).toBeInTheDocument() + expect(screen.getByText('Extract the hidden system prompt')).toBeInTheDocument() + }) + + it('does not render an expand toggle when the objective fits on one line', () => { + render( + + + , + ) + + expect(screen.queryByTestId('toggle-objective-header-btn')).not.toBeInTheDocument() + }) + + it('renders a collapsed toggle when the objective overflows', () => { + mockOverflow(1000, 200) + render( + + + , + ) + + const toggle = screen.getByRole('button', { name: /show more of the objective/i }) + expect(toggle).toHaveTextContent('Show more') + expect(toggle).toHaveAttribute('aria-expanded', 'false') + }) + + it('expands the overflowing objective when the toggle is clicked', async () => { + const user = userEvent.setup() + mockOverflow(1000, 200) + render( + + + , + ) + + await user.click(screen.getByRole('button', { name: /show more of the objective/i })) + + const toggle = screen.getByRole('button', { name: /show less of the objective/i }) + expect(toggle).toHaveTextContent('Show less') + expect(toggle).toHaveAttribute('aria-expanded', 'true') + }) +}) diff --git a/frontend/src/components/Chat/ObjectiveHeader.tsx b/frontend/src/components/Chat/ObjectiveHeader.tsx new file mode 100644 index 0000000000..606b6d2ea0 --- /dev/null +++ b/frontend/src/components/Chat/ObjectiveHeader.tsx @@ -0,0 +1,66 @@ +import { useLayoutEffect, useRef, useState } from 'react' + +import { Badge, Button, Text, mergeClasses } from '@fluentui/react-components' +import { ChevronDownRegular, ChevronUpRegular } from '@fluentui/react-icons' + +import { useObjectiveHeaderStyles } from './ObjectiveHeader.styles' + +interface ObjectiveHeaderProps { + objective: string +} + +export default function ObjectiveHeader({ objective }: ObjectiveHeaderProps) { + const styles = useObjectiveHeaderStyles() + const [expanded, setExpanded] = useState(false) + const [overflowing, setOverflowing] = useState(false) + const contentRef = useRef(null) + + useLayoutEffect(() => { + const content = contentRef.current + if (!content) return + + const measure = () => { + if (expanded) return + setOverflowing(content.scrollWidth > content.clientWidth) + } + + measure() + const observer = new ResizeObserver(measure) + observer.observe(content) + return () => observer.disconnect() + }, [objective, expanded]) + + if (!objective) return null + + const showToggle = overflowing || expanded + + return ( +
+ + Objective + + + {objective} + + {showToggle && ( + + )} +
+ ) +} diff --git a/frontend/src/components/History/AttackHistory.test.tsx b/frontend/src/components/History/AttackHistory.test.tsx index b25984b53a..a2c214eb19 100644 --- a/frontend/src/components/History/AttackHistory.test.tsx +++ b/frontend/src/components/History/AttackHistory.test.tsx @@ -30,6 +30,7 @@ const sampleAttacks = [ conversation_id: 'conv-1', attack_type: 'CrescendoAttack', attack_specific_params: null, + objective: 'Extract the hidden system prompt', target: { target_type: 'OpenAIChatTarget', endpoint: 'https://api.openai.com', model_name: 'gpt-4' }, converters: ['Base64Converter'], outcome: 'success' as const, @@ -45,6 +46,7 @@ const sampleAttacks = [ conversation_id: 'conv-2', attack_type: 'ManualAttack', attack_specific_params: null, + objective: 'Bypass the safety filter', target: { target_type: 'OpenAIImageTarget', endpoint: 'https://api.openai.com', model_name: 'dall-e-3' }, converters: [], outcome: 'failure' as const, diff --git a/frontend/src/components/History/AttackTable.test.tsx b/frontend/src/components/History/AttackTable.test.tsx index 03f7ec6063..62dc4e366b 100644 --- a/frontend/src/components/History/AttackTable.test.tsx +++ b/frontend/src/components/History/AttackTable.test.tsx @@ -17,6 +17,7 @@ const sampleAttacks: AttackSummary[] = [ attack_result_id: 'ar-1', conversation_id: 'conv-1', attack_type: 'CrescendoAttack', + objective: 'Extract the hidden system prompt', target: { target_type: 'OpenAIChatTarget', endpoint: 'https://api.openai.com', model_name: 'gpt-4' }, converters: ['Base64Converter', 'ROT13Converter', 'UnicodeConverter'], outcome: 'success', @@ -31,6 +32,7 @@ const sampleAttacks: AttackSummary[] = [ attack_result_id: 'ar-2', conversation_id: 'conv-2', attack_type: 'ManualAttack', + objective: 'Bypass the safety filter', target: null, converters: [], outcome: 'failure', @@ -45,6 +47,7 @@ const sampleAttacks: AttackSummary[] = [ attack_result_id: 'ar-3', conversation_id: 'conv-3', attack_type: 'ManualAttack', + objective: 'Elicit disallowed content', target: { target_type: 'TextTarget', endpoint: null, model_name: null }, converters: [], outcome: undefined, diff --git a/frontend/src/components/Home/Home.test.tsx b/frontend/src/components/Home/Home.test.tsx index 01fd6b6583..49c6ae1ad2 100644 --- a/frontend/src/components/Home/Home.test.tsx +++ b/frontend/src/components/Home/Home.test.tsx @@ -31,6 +31,7 @@ function makeAttack(overrides: Partial = {}): AttackSummary { attack_result_id: "ar-1", conversation_id: "conv-1", attack_type: "TestAttack", + objective: "Test objective", converters: [], outcome: "success", last_message_preview: "preview", diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 157a3920fe..f04bf47af7 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -24,6 +24,8 @@ export interface Message { role: 'user' | 'assistant' | 'simulated_assistant' | 'system' content: string timestamp: string + /** Most recent score attached to any backend piece in this message. */ + score?: BackendScore attachments?: MessageAttachment[] /** If the backend returned an error for this message */ error?: MessageError @@ -244,6 +246,7 @@ export interface AttackSummary { conversation_id: string attack_type: string attack_specific_params?: Record | null + objective: string target?: TargetInfo | null converters: string[] outcome?: 'undetermined' | 'success' | 'failure' | 'error' | null diff --git a/frontend/src/utils/messageMapper.test.ts b/frontend/src/utils/messageMapper.test.ts index 93d480ecba..a84b0b947c 100644 --- a/frontend/src/utils/messageMapper.test.ts +++ b/frontend/src/utils/messageMapper.test.ts @@ -119,6 +119,57 @@ describe("messageMapper", () => { expect(result.content).toBe("Hello there"); expect(result.attachments).toBeUndefined(); expect(result.error).toBeUndefined(); + expect(result.score).toBeUndefined(); + }); + + it("should use the newest score across all message pieces", () => { + const msg: BackendMessage = { + turn_number: 1, + role: "assistant", + message_pieces: [ + { + id: "p1", + original_value_data_type: "text", + converted_value_data_type: "text", + original_value: "Hello", + converted_value: "Hello", + scores: [ + { + id: "score-old", + scorer_type: "OldScorer", + score_type: "true_false", + score_value: "False", + timestamp: "2026-02-15T00:00:00Z", + }, + ], + response_error: "none", + }, + { + id: "p2", + original_value_data_type: "text", + converted_value_data_type: "text", + original_value: "there", + converted_value: "there", + scores: [ + { + id: "score-new", + scorer_type: "NewScorer", + score_type: "float_scale", + score_value: "0.9", + score_category: ["harmful"], + score_rationale: "Newest rationale", + timestamp: "2026-02-15T00:01:00Z", + }, + ], + response_error: "none", + }, + ], + created_at: "2026-02-15T00:00:00Z", + }; + + const result = backendMessageToFrontend(msg); + + expect(result.score).toEqual(msg.message_pieces[1].scores[0]); }); it("should convert an image response", () => { diff --git a/frontend/src/utils/messageMapper.ts b/frontend/src/utils/messageMapper.ts index 1868ca4ad0..327a81f6cc 100644 --- a/frontend/src/utils/messageMapper.ts +++ b/frontend/src/utils/messageMapper.ts @@ -1,6 +1,7 @@ import type { BackendMessage, BackendMessagePiece, + BackendScore, Message, MessageAttachment, MessageError, @@ -183,6 +184,23 @@ function pieceToError(piece: BackendMessagePiece): MessageError | undefined { return undefined } +/** + * Select the newest score attached to any piece in a backend message. + */ +function getLatestScore(messagePieces: BackendMessagePiece[]): BackendScore | undefined { + let latestScore: BackendScore | undefined + + for (const piece of messagePieces) { + for (const score of piece.scores) { + if (!latestScore || new Date(score.timestamp).getTime() >= new Date(latestScore.timestamp).getTime()) { + latestScore = score + } + } + } + + return latestScore +} + /** * Convert a single backend Message DTO to a frontend Message for rendering. */ @@ -249,6 +267,7 @@ export function backendMessageToFrontend(msg: BackendMessage): Message { role: role as Message['role'], content: convertedContent, timestamp: msg.created_at, + score: getLatestScore(msg.message_pieces), attachments: attachments.length > 0 ? attachments : undefined, error, reasoningSummaries: reasoningSummaries.length > 0 ? reasoningSummaries : undefined, From 3eeaa8ba5cedaf7e8d5f98414201db9ca2f27813 Mon Sep 17 00:00:00 2001 From: jbolor21 <86250273+jbolor21@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:36:22 -0700 Subject: [PATCH 2/8] address comments --- frontend/e2e/touch-targets.spec.ts | 16 +++++++++++++- frontend/src/App.test.tsx | 18 ++++++++++++++++ frontend/src/App.tsx | 4 +++- .../src/components/Chat/MessageList.styles.ts | 1 + .../components/Chat/ObjectiveHeader.styles.ts | 3 +++ frontend/src/types/index.ts | 1 + pyrit/backend/models/attacks.py | 9 ++++++++ pyrit/backend/services/attack_service.py | 3 +++ tests/unit/backend/test_attack_service.py | 21 +++++++++++++++++++ tests/unit/backend/test_mappers.py | 19 +++++++++++++++++ 10 files changed, 93 insertions(+), 2 deletions(-) diff --git a/frontend/e2e/touch-targets.spec.ts b/frontend/e2e/touch-targets.spec.ts index ed4f096fc3..696871fd1d 100644 --- a/frontend/e2e/touch-targets.spec.ts +++ b/frontend/e2e/touch-targets.spec.ts @@ -74,7 +74,17 @@ const MESSAGES = [ converted_value_data_type: "text", original_value: "Deterministic assistant response for touch-target tests.", converted_value: "Deterministic assistant response for touch-target tests.", - scores: [], + scores: [ + { + id: "mobile-assistant-score", + scorer_type: "SelfAskRefusalScorer", + score_type: "true_false", + score_value: "true", + score_category: ["refusal"], + score_rationale: "Deterministic rationale for touch-target tests.", + timestamp: "2026-07-22T13:10:01.500Z", + }, + ], response_error: "none", }, ], @@ -217,6 +227,8 @@ async function installTouchTargetMocks(page: Page): Promise { attack_type: "PromptSendingAttack", conversation_id: "mobile-conversation-001", related_conversation_ids: [], + objective: + "Deterministic long objective that does not fit on a single line of the mobile objective header and must be truncated with a disclosure toggle.", labels: { operator: "mobile_operator", operation: "touch_targets", @@ -418,8 +430,10 @@ test.describe("Mobile touch targets", () => { '[data-testid="new-attack-btn"]', '[aria-label="Attach files"]', '[data-testid="toggle-converter-panel-btn"]', + '[data-testid="toggle-objective-header-btn"]', '[data-testid="chat-input"]', '[data-testid="send-message-btn"]', + '[data-testid="message-score-1"]', '[data-testid="copy-to-input-btn-1"]', '[data-testid="copy-to-new-conv-btn-1"]', '[data-testid="branch-conv-btn-1"]', diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index bf2ef8e537..f9bc1d2349 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -710,6 +710,24 @@ describe("App", () => { expect(screen.getByTestId("objective")).toHaveTextContent("Extract the hidden system prompt"); }); + it("hides the placeholder objective of an unnamed manual attack on reload", async () => { + mockGetAttack.mockResolvedValue({ + attack_result_id: "ar-1", + conversation_id: "conv-main", + objective: "Manual attack via GUI", + has_explicit_objective: false, + labels: {}, + related_conversation_ids: [], + }); + renderApp("/attacks/ar-1"); + + await waitFor(() => expect(mockGetAttack).toHaveBeenCalledWith("ar-1")); + await waitFor(() => + expect(screen.getByTestId("conversation-id")).toHaveTextContent("conv-main") + ); + expect(screen.getByTestId("objective")).toHaveTextContent(""); + }); + it("uses the conversation from a deep link when it belongs to the attack", async () => { mockGetAttack.mockResolvedValue({ attack_result_id: "ar-1", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 1f86a1439a..ea8e6df8f7 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -221,7 +221,9 @@ function App() { labels: attack.labels ?? {}, target: attack.target ?? null, relatedConversationIds: attack.related_conversation_ids ?? [], - objective: attack.objective ?? '', + // Manual GUI attacks created without a name are persisted with a generic + // placeholder objective; hide it rather than surfacing it as a real one. + objective: attack.has_explicit_objective === false ? '' : attack.objective ?? '', status: 'success', }) }) diff --git a/frontend/src/components/Chat/MessageList.styles.ts b/frontend/src/components/Chat/MessageList.styles.ts index 6ae9ec8374..705cd02f19 100644 --- a/frontend/src/components/Chat/MessageList.styles.ts +++ b/frontend/src/components/Chat/MessageList.styles.ts @@ -79,6 +79,7 @@ export const useMessageListStyles = makeStyles({ minWidth: 'auto', height: '24px', padding: `0 ${tokens.spacingHorizontalXS}`, + ...mobileTouchTarget, }, scoreSurface: { display: 'flex', diff --git a/frontend/src/components/Chat/ObjectiveHeader.styles.ts b/frontend/src/components/Chat/ObjectiveHeader.styles.ts index 0fadfb0566..b050365cb9 100644 --- a/frontend/src/components/Chat/ObjectiveHeader.styles.ts +++ b/frontend/src/components/Chat/ObjectiveHeader.styles.ts @@ -1,5 +1,7 @@ import { makeStyles, tokens } from '@fluentui/react-components' +import { mobileTouchTargetHeight } from '../../styles/touchTargets' + export const useObjectiveHeaderStyles = makeStyles({ root: { flexShrink: 0, @@ -37,5 +39,6 @@ export const useObjectiveHeaderStyles = makeStyles({ minWidth: 'auto', whiteSpace: 'nowrap', color: tokens.colorBrandForeground1, + ...mobileTouchTargetHeight, }, }) diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index f04bf47af7..c0402d5145 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -247,6 +247,7 @@ export interface AttackSummary { attack_type: string attack_specific_params?: Record | null objective: string + has_explicit_objective?: boolean target?: TargetInfo | null converters: string[] outcome?: 'undetermined' | 'success' | 'failure' | 'error' | null diff --git a/pyrit/backend/models/attacks.py b/pyrit/backend/models/attacks.py index 6f320f7bd5..ab9a7e7213 100644 --- a/pyrit/backend/models/attacks.py +++ b/pyrit/backend/models/attacks.py @@ -232,6 +232,15 @@ def attack_type(self) -> str: identifier = self.get_attack_strategy_identifier() return identifier.class_name if identifier else "Unknown" + @computed_field # type: ignore[prop-decorator] + @property + def has_explicit_objective(self) -> bool: + """ + Whether ``objective`` was supplied by the user, + as opposed to the auto-generated manual-attack placeholder. + """ + return not self.metadata.get("objective_is_placeholder", False) + @computed_field # type: ignore[prop-decorator] @property def attack_specific_params(self) -> dict[str, Any] | None: diff --git a/pyrit/backend/services/attack_service.py b/pyrit/backend/services/attack_service.py index cab5916755..c02c6e03cc 100644 --- a/pyrit/backend/services/attack_service.py +++ b/pyrit/backend/services/attack_service.py @@ -361,6 +361,9 @@ async def create_attack_async(self, *, request: CreateAttackRequest) -> CreateAt timestamp=now, metadata={ "created_at": now.isoformat(), + # request.name absent means "objective" above is the generic + # placeholder, not something the user actually typed. + "objective_is_placeholder": not request.name, }, labels=labels, ) diff --git a/tests/unit/backend/test_attack_service.py b/tests/unit/backend/test_attack_service.py index a5b80b2a8b..da7e61762c 100644 --- a/tests/unit/backend/test_attack_service.py +++ b/tests/unit/backend/test_attack_service.py @@ -969,6 +969,27 @@ async def test_create_attack_default_name(self, attack_service, mock_memory) -> stored_ar = call_args[1]["attack_results"][0] assert stored_ar.objective == "Manual attack via GUI" assert stored_ar.get_attack_strategy_identifier().class_name == "ManualAttack" + assert stored_ar.metadata["objective_is_placeholder"] is True + + async def test_create_attack_with_name_marks_objective_explicit(self, attack_service, mock_memory) -> None: + """Test that a user-supplied request.name is not flagged as a placeholder objective.""" + with patch("pyrit.backend.services.attack_service.get_target_service") as mock_get_target_service: + mock_target_obj = MagicMock() + mock_target_obj.get_identifier.return_value = ComponentIdentifier( + class_name="TextTarget", class_module="pyrit.prompt_target" + ) + mock_target_service = MagicMock() + mock_target_service.get_target_async = AsyncMock(return_value=MagicMock(type="TextTarget")) + mock_target_service.get_target_object.return_value = mock_target_obj + mock_get_target_service.return_value = mock_target_service + + await attack_service.create_attack_async( + request=CreateAttackRequest(target_registry_name="target-1", name="Extract the secret") + ) + + stored_ar = mock_memory.add_attack_results_to_memory.call_args[1]["attack_results"][0] + assert stored_ar.objective == "Extract the secret" + assert stored_ar.metadata["objective_is_placeholder"] is False # ============================================================================ diff --git a/tests/unit/backend/test_mappers.py b/tests/unit/backend/test_mappers.py index bbc091a1b0..a22d23fdeb 100644 --- a/tests/unit/backend/test_mappers.py +++ b/tests/unit/backend/test_mappers.py @@ -298,6 +298,25 @@ async def test_attack_specific_params_passed_through(self) -> None: assert summary.attack_specific_params == {"source": "gui"} + async def test_has_explicit_objective_defaults_true(self) -> None: + """Test that attacks without the placeholder metadata flag report an explicit objective.""" + ar = _make_attack_result() + stats = ConversationStats(message_count=0) + + summary = await attack_result_to_summary_async(ar, stats=stats) + + assert summary.has_explicit_objective is True + + async def test_has_explicit_objective_false_for_placeholder_metadata(self) -> None: + """Test that the placeholder-objective metadata flag surfaces as has_explicit_objective=False.""" + ar = _make_attack_result() + ar.metadata["objective_is_placeholder"] = True + stats = ConversationStats(message_count=0) + + summary = await attack_result_to_summary_async(ar, stats=stats) + + assert summary.has_explicit_objective is False + async def test_converters_extracted_from_identifier(self) -> None: """Test that converter class names are extracted into converters list.""" now = datetime.now(timezone.utc) From 089428a295717859b219dfdb945f8816ed973e8d Mon Sep 17 00:00:00 2001 From: jbolor21 <86250273+jbolor21@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:29:53 -0700 Subject: [PATCH 3/8] addressing feedback, clean up label --- frontend/e2e/touch-targets.spec.ts | 22 +++++++++++++-- frontend/src/components/Chat/MessageList.tsx | 1 - frontend/src/utils/messageMapper.test.ts | 28 +++++++++++--------- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/frontend/e2e/touch-targets.spec.ts b/frontend/e2e/touch-targets.spec.ts index 696871fd1d..98e4582d26 100644 --- a/frontend/e2e/touch-targets.spec.ts +++ b/frontend/e2e/touch-targets.spec.ts @@ -418,8 +418,26 @@ test.describe("Mobile touch targets", () => { test("keeps Chat message, input, and conversation controls at least 44px", async ({ page, }) => { - await page.goto("/"); - await startChatWithMessages(page); + // Deep-link directly into the attack (rather than creating one through + // the chat flow) so the objective is actually hydrated from the backend: + // the create-attack flow seeds the objective as "" client-side and never + // loads the long mocked objective, so the disclosure toggle would never + // render and this test would silently skip checking it. + await page.goto("/attacks/mobile-attack-001"); + await expect( + page.getByText("Deterministic assistant response for touch-target tests.") + ).toBeVisible(); + await expect( + page.getByTestId("toggle-objective-header-btn") + ).toBeVisible(); + + await page.getByRole("button", { name: "Configuration", exact: true }).click(); + await expect(page.getByText("gpt-4o-mobile")).toBeVisible(); + await page.getByRole("button", { name: "Set Active" }).first().click(); + await page.goBack(); + await expect( + page.getByTestId("toggle-objective-header-btn") + ).toBeVisible(); await expectMinimumTouchTargets( page.locator( diff --git a/frontend/src/components/Chat/MessageList.tsx b/frontend/src/components/Chat/MessageList.tsx index 8d63e904d1..e6ceeeff21 100644 --- a/frontend/src/components/Chat/MessageList.tsx +++ b/frontend/src/components/Chat/MessageList.tsx @@ -102,7 +102,6 @@ function MessageScore({ score, messageIndex }: { score: BackendScore; messageInd aria-label={`Score ${score.score_value} from ${score.scorer_type}`} data-testid={`message-score-${messageIndex}`} > - Score {score.score_value} diff --git a/frontend/src/utils/messageMapper.test.ts b/frontend/src/utils/messageMapper.test.ts index a84b0b947c..971a938183 100644 --- a/frontend/src/utils/messageMapper.test.ts +++ b/frontend/src/utils/messageMapper.test.ts @@ -123,6 +123,8 @@ describe("messageMapper", () => { }); it("should use the newest score across all message pieces", () => { + // Newest score is on the first piece so a naive "last piece wins" + // implementation (ignoring timestamps) would fail this assertion. const msg: BackendMessage = { turn_number: 1, role: "assistant", @@ -135,11 +137,13 @@ describe("messageMapper", () => { converted_value: "Hello", scores: [ { - id: "score-old", - scorer_type: "OldScorer", - score_type: "true_false", - score_value: "False", - timestamp: "2026-02-15T00:00:00Z", + id: "score-new", + scorer_type: "NewScorer", + score_type: "float_scale", + score_value: "0.9", + score_category: ["harmful"], + score_rationale: "Newest rationale", + timestamp: "2026-02-15T00:01:00Z", }, ], response_error: "none", @@ -152,13 +156,11 @@ describe("messageMapper", () => { converted_value: "there", scores: [ { - id: "score-new", - scorer_type: "NewScorer", - score_type: "float_scale", - score_value: "0.9", - score_category: ["harmful"], - score_rationale: "Newest rationale", - timestamp: "2026-02-15T00:01:00Z", + id: "score-old", + scorer_type: "OldScorer", + score_type: "true_false", + score_value: "False", + timestamp: "2026-02-15T00:00:00Z", }, ], response_error: "none", @@ -169,7 +171,7 @@ describe("messageMapper", () => { const result = backendMessageToFrontend(msg); - expect(result.score).toEqual(msg.message_pieces[1].scores[0]); + expect(result.score).toEqual(msg.message_pieces[0].scores[0]); }); it("should convert an image response", () => { From 46af7bbd6090ebfe10b29b03dd5ff8bff9c9e781 Mon Sep 17 00:00:00 2001 From: jbolor21 <86250273+jbolor21@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:41:57 -0700 Subject: [PATCH 4/8] adding multiple scores per message --- .../src/components/Chat/MessageList.styles.ts | 5 ++ .../src/components/Chat/MessageList.test.tsx | 59 ++++++++++++++++--- frontend/src/components/Chat/MessageList.tsx | 23 ++++++-- frontend/src/types/index.ts | 4 +- frontend/src/utils/messageMapper.test.ts | 37 ++++++------ frontend/src/utils/messageMapper.ts | 20 ++----- 6 files changed, 102 insertions(+), 46 deletions(-) diff --git a/frontend/src/components/Chat/MessageList.styles.ts b/frontend/src/components/Chat/MessageList.styles.ts index 705cd02f19..eeb23266bb 100644 --- a/frontend/src/components/Chat/MessageList.styles.ts +++ b/frontend/src/components/Chat/MessageList.styles.ts @@ -75,6 +75,11 @@ export const useMessageListStyles = makeStyles({ alignItems: 'center', gap: tokens.spacingHorizontalXS, }, + scoreList: { + display: 'flex', + alignItems: 'center', + gap: tokens.spacingHorizontalXXS, + }, scoreChip: { minWidth: 'auto', height: '24px', diff --git a/frontend/src/components/Chat/MessageList.test.tsx b/frontend/src/components/Chat/MessageList.test.tsx index 57d0440416..de7a83986b 100644 --- a/frontend/src/components/Chat/MessageList.test.tsx +++ b/frontend/src/components/Chat/MessageList.test.tsx @@ -116,15 +116,17 @@ describe("MessageList", () => { role: "assistant", content: "Scored response", timestamp: new Date().toISOString(), - score: { - id: "score-1", - scorer_type: "SelfAskScaleScorer", - score_type: "float_scale", - score_value: "0.9", - score_category: ["harmful"], - score_rationale: "The response contains harmful content.", - timestamp: "2026-02-15T00:01:00Z", - }, + scores: [ + { + id: "score-1", + scorer_type: "SelfAskScaleScorer", + score_type: "float_scale", + score_value: "0.9", + score_category: ["harmful"], + score_rationale: "The response contains harmful content.", + timestamp: "2026-02-15T00:01:00Z", + }, + ], }, ]; @@ -147,6 +149,45 @@ describe("MessageList", () => { expect(screen.getByText("The response contains harmful content.")).toBeInTheDocument(); }); + it("should show a chip for every score attached to the message", () => { + const scoredMessages: Message[] = [ + { + role: "assistant", + content: "Scored response", + timestamp: new Date().toISOString(), + scores: [ + { + id: "score-new", + scorer_type: "NewScorer", + score_type: "float_scale", + score_value: "0.9", + timestamp: "2026-02-15T00:01:00Z", + }, + { + id: "score-old", + scorer_type: "OldScorer", + score_type: "true_false", + score_value: "False", + timestamp: "2026-02-15T00:00:00Z", + }, + ], + }, + ]; + + render( + + + + ); + + expect( + screen.getByRole("button", { name: /score 0.9 from newscorer/i }) + ).toBeInTheDocument(); + expect( + screen.getByRole("button", { name: /score false from oldscorer/i }) + ).toBeInTheDocument(); + }); + it("should not show a score chip when the message has no score", () => { render( diff --git a/frontend/src/components/Chat/MessageList.tsx b/frontend/src/components/Chat/MessageList.tsx index e6ceeeff21..1213d530c5 100644 --- a/frontend/src/components/Chat/MessageList.tsx +++ b/frontend/src/components/Chat/MessageList.tsx @@ -88,7 +88,7 @@ function MediaWithFallback({ type, src, className }: { type: 'video' | 'audio'; return