diff --git a/doc/code/framework.md b/doc/code/framework.md index a3938ef2dc..0929a71f77 100644 --- a/doc/code/framework.md +++ b/doc/code/framework.md @@ -314,8 +314,8 @@ The below talks about responsibilities of most modules in the PyRIT library **Responsibility**: Reshape prompts and conversations so components and targets can interoperate. There are two distinct modules: -- **`prompt_normalizer`** applies converters and dispatches individual prompts to a `PromptTarget` (handling batching and memory persistence). It is the single component that writes each request and response to memory; targets never persist on their own. `NormalizerRequest` and `ConverterConfiguration` describe what to send and which converters to apply. -- **`message_normalizer`** reshapes multi-message conversation payloads into the structure a given model expects — for example, handling system-message behavior (keep / squash / ignore), history squashing, and tokenizer chat templates. +- **`prompt_normalizer`** applies converters and dispatches individual prompts to a `PromptTarget` (handling batching and memory persistence). It is the single component that writes each request and response to memory; targets never persist on their own. `NormalizerRequest` and `ConverterConfiguration` describe what to send and which converters to apply. Prepended history remains role-structured in memory; when a target cannot edit history, the prompt normalizer passes its formatter to the target for the first live send. +- **`message_normalizer`** reshapes multi-message conversation payloads into the structure a given model expects — for example, handling system-message behavior (keep / squash / ignore), history squashing, prepended-history adaptation, and tokenizer chat templates. These target-specific views are ephemeral and do not replace the logical conversation in memory. ## [Output](./output/0_output) diff --git a/pyrit/executor/attack/component/conversation_manager.py b/pyrit/executor/attack/component/conversation_manager.py index d92f99bbde..5c064549aa 100644 --- a/pyrit/executor/attack/component/conversation_manager.py +++ b/pyrit/executor/attack/component/conversation_manager.py @@ -13,7 +13,7 @@ PrependedConversationConfig, ) from pyrit.memory import CentralMemory -from pyrit.message_normalizer import ConversationContextNormalizer, GenericSystemSquashNormalizer +from pyrit.message_normalizer import ConversationContextNormalizer from pyrit.models import ( ChatMessageRole, ComponentIdentifier, @@ -275,17 +275,12 @@ async def initialize_context_async( This is the primary method for setting up an attack context. It: 1. Merges memory_labels from attack strategy with context labels - 2. Processes prepended_conversation based on target type and config + 2. Persists prepended_conversation structurally with role-scoped converters 3. Updates context.executed_turns for multi-turn attacks - 4. Sets context.next_message if there's an unanswered user message - For chat-capable PromptTarget: - - Adds prepended messages to memory with simulated_assistant role - - All messages get new UUIDs - - For non-chat PromptTarget: - - Normalizes the prepended conversation to a string and prepends it to - ``context.next_message`` (using ``config.message_normalizer`` when provided). + For all PromptTarget types, prepended messages are added to memory with + simulated_assistant roles and new UUIDs. Targets without editable history receive + a one-shot formatter that combines this structured history with the first live request. Args: context: The attack context to initialize. @@ -300,8 +295,7 @@ async def initialize_context_async( ConversationState with turn_count and last_assistant_message_scores. Raises: - ValueError: If conversation_id is empty, or if prepended_conversation - requires a chat-capable PromptTarget but target is not one. + ValueError: If conversation_id is empty. """ if not conversation_id: raise ValueError("conversation_id cannot be empty") @@ -316,19 +310,7 @@ async def initialize_context_async( logger.debug(f"No prepended conversation for context initialization: {conversation_id}") return state - # Targets that don't natively support editable history cannot consume a - # prepended multi-message conversation as-is — route them to the - # single-string fallback path via capability-based routing. - is_chat_target = target.configuration.includes(capability=CapabilityName.EDITABLE_HISTORY) - if not is_chat_target: - return await self._handle_non_chat_target_async( - context=context, - prepended_conversation=prepended_conversation, - config=prepended_conversation_config, - ) - - # Process prepended conversation for objective target - return await self._process_prepended_for_chat_target_async( + return await self._process_prepended_conversation_async( context=context, prepended_conversation=prepended_conversation, conversation_id=conversation_id, @@ -336,76 +318,9 @@ async def initialize_context_async( prepended_conversation_config=prepended_conversation_config, max_turns=max_turns, target_identifier=target.get_identifier(), + target=target, ) - async def _handle_non_chat_target_async( - self, - *, - context: AttackContext[Any], - prepended_conversation: list[Message], - config: PrependedConversationConfig | None, - ) -> ConversationState: - """ - Handle prepended conversation for non-chat targets. - - Args: - context: The attack context. - prepended_conversation: Messages to prepend. - config: Configuration for non-chat target behavior. - - Returns: - Empty ConversationState (non-chat targets don't track turns). - """ - if config is None: - config = PrependedConversationConfig() - - normalizer = config.get_message_normalizer() - messages_to_normalize = prepended_conversation - if isinstance(normalizer, ConversationContextNormalizer): - messages_to_normalize = await GenericSystemSquashNormalizer().normalize_async(prepended_conversation) - - normalized_context = await normalizer.normalize_string_async(messages_to_normalize) - - next_message = context.next_message - if next_message is None: - next_message = Message.from_prompt(prompt=context.objective, role="user") - context.next_message = next_message - - if normalized_context: - # Find an existing text piece to prepend to - text_piece = None - for piece in next_message.message_pieces: - if piece.original_value_data_type == "text": - text_piece = piece - break - - if text_piece: - # Prepend context to the existing text piece - context_prefix = f"{normalized_context}\n\n" - if text_piece.original_value != normalized_context and not text_piece.original_value.startswith( - context_prefix - ): - text_piece.original_value = f"{context_prefix}{text_piece.original_value}" - if text_piece.converted_value != normalized_context and not text_piece.converted_value.startswith( - context_prefix - ): - text_piece.converted_value = f"{context_prefix}{text_piece.converted_value}" - else: - # No text piece found (multimodal message), add a new text piece at the beginning - context_piece = MessagePiece( - id=uuid.uuid4(), - role="user", - original_value=normalized_context, - converted_value=normalized_context, - original_value_data_type="text", - converted_value_data_type="text", - ) - # Create a new message with the context piece prepended - context.next_message = Message(message_pieces=[context_piece] + list(next_message.message_pieces)) - - logger.debug(f"Normalized prepended conversation for non-chat target: {len(normalized_context)} characters") - return ConversationState() - async def add_prepended_conversation_to_memory_async( self, *, @@ -415,9 +330,10 @@ async def add_prepended_conversation_to_memory_async( prepended_conversation_config: PrependedConversationConfig | None = None, max_turns: int | None = None, target_identifier: ComponentIdentifier | None = None, + target: PromptTarget | None = None, ) -> int: """ - Add prepended conversation messages to memory for a chat target. + Add prepended conversation messages to memory for a target. This is a lower-level method that handles adding messages to memory without modifying any attack context state. It can be called directly by attacks @@ -437,6 +353,8 @@ async def add_prepended_conversation_to_memory_async( max_turns: If provided, validates that turn count doesn't exceed this limit. target_identifier (ComponentIdentifier | None): The target the conversation is held with, if known. Recorded once per conversation. + target (PromptTarget | None): Target that will receive the first live request. When it + lacks editable history, its target-normalization path receives the configured formatter. Returns: The number of turns (assistant messages) added. @@ -449,13 +367,19 @@ async def add_prepended_conversation_to_memory_async( if not valid_messages: return 0 + if target and target_identifier is None: + target_identifier = target.get_identifier() + self._memory.add_conversation_to_memory( conversation=Conversation(conversation_id=conversation_id, target_identifier=target_identifier) ) - # Get roles that should have converters applied - apply_to_roles = ( - prepended_conversation_config.apply_converters_to_roles if prepended_conversation_config else None + # Assistant history represents simulated target output, so the absent-config + # path must use the same safe role default as an explicit default config. + config = prepended_conversation_config or PrependedConversationConfig() + apply_to_roles = config.apply_converters_to_roles + requires_prepended_adaptation = bool( + target and not target.configuration.includes(capability=CapabilityName.EDITABLE_HISTORY) ) turn_count = 0 @@ -485,14 +409,25 @@ async def add_prepended_conversation_to_memory_async( request_converters=request_converters, apply_to_roles=apply_to_roles, ) + if requires_prepended_adaptation: + self._validate_flattenable_converter_output( + source_message=message, + converted_message=message_copy, + ) # Add to memory self._memory.add_message_to_memory(request=message_copy) logger.debug(f"Added prepended message {i + 1}/{len(valid_messages)} to memory") + if requires_prepended_adaptation: + self._prompt_normalizer.register_prepended_conversation_normalizer( + conversation_id=conversation_id, + message_normalizer=config.get_message_normalizer(), + ) + return turn_count - async def _process_prepended_for_chat_target_async( + async def _process_prepended_conversation_async( self, *, context: AttackContext[Any], @@ -502,9 +437,10 @@ async def _process_prepended_for_chat_target_async( prepended_conversation_config: PrependedConversationConfig | None, max_turns: int | None, target_identifier: ComponentIdentifier | None = None, + target: PromptTarget, ) -> ConversationState: """ - Process prepended conversation for a chat target. + Process prepended conversation for a target. Adds messages to memory with: - New UUIDs for all pieces @@ -520,6 +456,7 @@ async def _process_prepended_for_chat_target_async( max_turns: Maximum turns for validation. target_identifier (ComponentIdentifier | None): The objective target the conversation is held with, if known. + target: The objective target that will receive the conversation. Returns: ConversationState with turn_count and scores. @@ -540,6 +477,7 @@ async def _process_prepended_for_chat_target_async( prepended_conversation_config=prepended_conversation_config, max_turns=max_turns, target_identifier=target_identifier, + target=target, ) # Update context for multi-turn attacks to reflect prepended_conversation @@ -570,12 +508,41 @@ async def _process_prepended_for_chat_target_async( return state + @staticmethod + def _validate_flattenable_converter_output( + *, + source_message: Message, + converted_message: Message, + ) -> None: + """ + Reject non-text output produced by this prepended conversion pass. + + Raises: + ValueError: If an applied converter produced non-text prepended history. + """ + output_types = { + converted_piece.converted_value_data_type + for source_piece, converted_piece in zip( + source_message.message_pieces, + converted_message.message_pieces, + strict=True, + ) + if len(converted_piece.converter_identifiers) > len(source_piece.converter_identifiers) + and converted_piece.converted_value_data_type != "text" + } + if output_types: + raise ValueError( + "Cannot flatten prepended conversation for a target without editable history after " + f"request converters produced non-text output types {sorted(output_types)}. Prepended " + "conversion must produce text." + ) + async def _apply_converters_async( self, *, message: Message, request_converters: list[ConverterConfiguration], - apply_to_roles: list[ChatMessageRole] | None, + apply_to_roles: list[ChatMessageRole], ) -> None: """ Apply converters to message pieces. @@ -583,16 +550,15 @@ async def _apply_converters_async( Args: message: The message containing pieces to convert. request_converters: Converter configurations to apply. - apply_to_roles: If provided, only apply to pieces with these roles. - If None, apply to all roles. + apply_to_roles: Only apply to pieces with these roles. """ - for piece in message.message_pieces: - # Filter by role if specified - if apply_to_roles is not None and piece.api_role not in apply_to_roles: - continue - - temp_message = Message(message_pieces=[piece]) - await self._prompt_normalizer.convert_values_async( - message=temp_message, - converter_configurations=request_converters, - ) + if message.api_role not in apply_to_roles: + return + + # Apply to the complete message so ConverterConfiguration.indexes_to_apply remains relative + # to the original piece list. Converting one temporary piece at a time would reset every + # selected piece to index zero. + await self._prompt_normalizer.convert_values_async( + message=message, + converter_configurations=request_converters, + ) diff --git a/pyrit/executor/attack/component/prepended_conversation_config.py b/pyrit/executor/attack/component/prepended_conversation_config.py index a0daedfd6e..0a511f8489 100644 --- a/pyrit/executor/attack/component/prepended_conversation_config.py +++ b/pyrit/executor/attack/component/prepended_conversation_config.py @@ -4,13 +4,15 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import get_args +from typing import TYPE_CHECKING from pyrit.message_normalizer import ( ConversationContextNormalizer, MessageStringNormalizer, ) -from pyrit.models import ChatMessageRole + +if TYPE_CHECKING: + from pyrit.models import ChatMessageRole @dataclass @@ -21,21 +23,23 @@ class PrependedConversationConfig: This class provides control over: - Which message roles should have request converters applied - - How to normalize conversation history for non-chat objective targets + - How targets without editable history format prepended messages on the first live send - Non-chat objective targets always normalize the prepended conversation into the - first turn (via ``message_normalizer``; default: ConversationContextNormalizer). + Prepended messages remain role-structured in memory. Request converters are applied to + configured roles before a target without editable history renders that history into the + first live request (via ``message_normalizer``; default: ConversationContextNormalizer). + Those converters must produce text because string normalization cannot preserve converted + image, audio, or other non-text output. """ - # Roles for which request converters should be applied to prepended messages. - # By default, converters are applied to all roles. - # Example: ["user"] to apply converters only to user messages. - apply_converters_to_roles: list[ChatMessageRole] = field(default_factory=lambda: list(get_args(ChatMessageRole))) + # Request converters default to prepended user messages only. Assistant history is + # simulated target output and must be explicitly opted in with ["assistant"]. + apply_converters_to_roles: list[ChatMessageRole] = field(default_factory=lambda: ["user"]) # Optional normalizer to format conversation history into a single text block. # Must implement MessageStringNormalizer (e.g., TokenizerTemplateNormalizer or ConversationContextNormalizer). - # When None and normalization is needed (e.g., for non-chat targets), a default - # ConversationContextNormalizer is used that produces "Turn N: User/Assistant" format. + # When None and adaptation is needed, a default ConversationContextNormalizer is used + # that produces "Turn N: User/Assistant" format. message_normalizer: MessageStringNormalizer | None = None def get_message_normalizer(self) -> MessageStringNormalizer: diff --git a/pyrit/executor/attack/multi_turn/crescendo.py b/pyrit/executor/attack/multi_turn/crescendo.py index 584065aa87..f1feee5d64 100644 --- a/pyrit/executor/attack/multi_turn/crescendo.py +++ b/pyrit/executor/attack/multi_turn/crescendo.py @@ -181,7 +181,7 @@ def __init__( max_turns (int): Maximum number of turns allowed. prepended_conversation_config (PrependedConversationConfiguration | None): Configuration for how to process prepended conversations. Controls converter - application by role, message normalization, and non-chat target behavior. + application by role and first-send formatting for targets without editable history. Raises: ValueError: If objective_target does not natively support editable history. diff --git a/pyrit/executor/attack/multi_turn/red_teaming.py b/pyrit/executor/attack/multi_turn/red_teaming.py index 402eb1303b..c05d2c8994 100644 --- a/pyrit/executor/attack/multi_turn/red_teaming.py +++ b/pyrit/executor/attack/multi_turn/red_teaming.py @@ -171,7 +171,7 @@ def __init__( # Initialize utilities self._prompt_normalizer = prompt_normalizer or PromptNormalizer() - self._conversation_manager = ConversationManager() + self._conversation_manager = ConversationManager(prompt_normalizer=self._prompt_normalizer) # set the maximum number of turns for the attack if max_turns <= 0: diff --git a/pyrit/executor/attack/multi_turn/tree_of_attacks.py b/pyrit/executor/attack/multi_turn/tree_of_attacks.py index 639f0e2427..9cd0550231 100644 --- a/pyrit/executor/attack/multi_turn/tree_of_attacks.py +++ b/pyrit/executor/attack/multi_turn/tree_of_attacks.py @@ -464,6 +464,7 @@ async def initialize_with_prepended_conversation_async( request_converters=self._request_converters, prepended_conversation_config=prepended_conversation_config, target_identifier=self._objective_target.get_identifier(), + target=self._objective_target, ) # Build context string for adversarial chat system prompt (like Crescendo) @@ -1373,7 +1374,7 @@ def __init__( batch_size (int): Number of nodes to process in parallel per batch. Defaults to 10. prepended_conversation_config (PrependedConversationConfig | None): Configuration for how to process prepended conversations. Controls converter - application by role, message normalization, and non-chat target behavior. + application by role and first-send formatting for targets without editable history. Raises: ValueError: If attack_scoring_config uses a non-FloatScaleThresholdScorer objective scorer, diff --git a/pyrit/executor/attack/single_turn/prompt_sending.py b/pyrit/executor/attack/single_turn/prompt_sending.py index 508b72d924..1736bfc19d 100644 --- a/pyrit/executor/attack/single_turn/prompt_sending.py +++ b/pyrit/executor/attack/single_turn/prompt_sending.py @@ -76,7 +76,9 @@ def __init__( a params type that rejects certain fields. prepended_conversation_config (PrependedConversationConfiguration | None): Configuration for how to process prepended conversations. Controls converter - application by role, message normalization, and non-chat target behavior. + application by role and first-send formatting for targets without editable history. + Request converters apply to prepended user messages by default; include + ``"assistant"`` explicitly to transform simulated assistant history. Raises: ValueError: If the objective scorer is not a true/false scorer. diff --git a/pyrit/message_normalizer/__init__.py b/pyrit/message_normalizer/__init__.py index 79df1cb50d..2ef7e1bc39 100644 --- a/pyrit/message_normalizer/__init__.py +++ b/pyrit/message_normalizer/__init__.py @@ -14,6 +14,7 @@ MessageListNormalizer, MessageStringNormalizer, ) +from pyrit.message_normalizer.prepended_conversation_normalizer import PrependedConversationNormalizer from pyrit.message_normalizer.tokenizer_template_normalizer import TokenizerTemplateNormalizer __all__ = [ @@ -22,6 +23,7 @@ "GenericSystemSquashNormalizer", "HistorySquashNormalizer", "JsonSchemaNormalizer", + "PrependedConversationNormalizer", "TokenizerTemplateNormalizer", "ConversationContextNormalizer", "ChatMessageNormalizer", diff --git a/pyrit/message_normalizer/prepended_conversation_normalizer.py b/pyrit/message_normalizer/prepended_conversation_normalizer.py new file mode 100644 index 0000000000..3640095a4b --- /dev/null +++ b/pyrit/message_normalizer/prepended_conversation_normalizer.py @@ -0,0 +1,154 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import uuid + +from pyrit.message_normalizer.conversation_context_normalizer import ConversationContextNormalizer +from pyrit.message_normalizer.generic_system_squash import GenericSystemSquashNormalizer +from pyrit.message_normalizer.message_normalizer import MessageListNormalizer, MessageStringNormalizer +from pyrit.models import Message, MessagePiece + + +class PrependedConversationNormalizer(MessageListNormalizer[Message]): + """ + Combine prepended history with the first live request for targets without editable history. + + The history remains structured in memory. This normalizer creates an ephemeral target view + that preserves the live request's modalities while prefixing independently rendered original + and converted history. + """ + + def __init__(self, *, message_normalizer: MessageStringNormalizer) -> None: + """ + Initialize the adapter. + + Args: + message_normalizer: Formatter used to render prepended history. + """ + self._message_normalizer = message_normalizer + + async def normalize_async(self, messages: list[Message]) -> list[Message]: + """ + Normalize prepended history into the final request message. + + Args: + messages: Prepended history followed by the first live request. + + Returns: + A single request message containing the rendered history. + """ + if len(messages) < 2: + return copy.deepcopy(messages) + + prepended_messages = messages[:-1] + self._validate_flattenable_converter_output(messages=prepended_messages) + + original_context = await self._normalize_context_async( + messages=self._build_original_view(messages=prepended_messages) + ) + converted_context = original_context + if self._contains_converted_values(messages=prepended_messages): + converted_context = await self._normalize_context_async( + messages=self._build_converted_view(messages=prepended_messages) + ) + + request = copy.deepcopy(messages[-1]) + self._prepend_context( + message=request, + original_context=original_context, + converted_context=converted_context, + ) + return [request] + + async def _normalize_context_async(self, *, messages: list[Message]) -> str: + messages_to_normalize = messages + if isinstance(self._message_normalizer, ConversationContextNormalizer): + messages_to_normalize = await GenericSystemSquashNormalizer().normalize_async(messages) + return await self._message_normalizer.normalize_string_async(messages_to_normalize) + + @staticmethod + def _build_original_view(*, messages: list[Message]) -> list[Message]: + original_messages = copy.deepcopy(messages) + for message in original_messages: + for piece in message.message_pieces: + piece.converted_value = piece.original_value + piece.converted_value_data_type = piece.original_value_data_type + return original_messages + + @staticmethod + def _build_converted_view(*, messages: list[Message]) -> list[Message]: + converted_messages = copy.deepcopy(messages) + for message in converted_messages: + for piece in message.message_pieces: + piece.original_value = piece.converted_value + piece.original_value_data_type = piece.converted_value_data_type + return converted_messages + + @staticmethod + def _contains_converted_values(*, messages: list[Message]) -> bool: + return any( + piece.converter_identifiers + or piece.original_value != piece.converted_value + or piece.original_value_data_type != piece.converted_value_data_type + for message in messages + for piece in message.message_pieces + ) + + @staticmethod + def _validate_flattenable_converter_output(*, messages: list[Message]) -> None: + output_types = { + piece.converted_value_data_type + for message in messages + for piece in message.message_pieces + if piece.converted_value_data_type != "text" + and piece.converted_value_data_type != piece.original_value_data_type + } + if output_types: + raise ValueError( + "Cannot flatten prepended conversation after request converters produced " + f"non-text output types {sorted(output_types)}. Prepended conversion must produce " + "text for a target without editable history." + ) + + @staticmethod + def _prepend_context(*, message: Message, original_context: str, converted_context: str) -> None: + text_piece = next( + ( + piece + for piece in message.message_pieces + if piece.original_value_data_type == "text" and piece.converted_value_data_type == "text" + ), + None, + ) + if text_piece: + text_piece.original_value = PrependedConversationNormalizer._prepend_context_value( + context=original_context, + value=text_piece.original_value, + ) + text_piece.converted_value = PrependedConversationNormalizer._prepend_context_value( + context=converted_context, + value=text_piece.converted_value, + ) + return + + template_piece = message.get_piece() + message.message_pieces.insert( + 0, + MessagePiece( + id=uuid.uuid4(), + role=template_piece.role, + original_value=original_context, + converted_value=converted_context, + original_value_data_type="text", + converted_value_data_type="text", + conversation_id=template_piece.conversation_id, + sequence=template_piece.sequence, + ), + ) + + @staticmethod + def _prepend_context_value(*, context: str, value: str) -> str: + if not context or value == context or value.startswith(f"{context}\n\n"): + return value + return f"{context}\n\n{value}" diff --git a/pyrit/prompt_normalizer/prompt_normalizer.py b/pyrit/prompt_normalizer/prompt_normalizer.py index cf7b458f94..24d995812e 100644 --- a/pyrit/prompt_normalizer/prompt_normalizer.py +++ b/pyrit/prompt_normalizer/prompt_normalizer.py @@ -19,6 +19,7 @@ get_execution_context, ) from pyrit.memory import CentralMemory, MemoryInterface, set_message_piece_sha256_async +from pyrit.message_normalizer import MessageStringNormalizer from pyrit.models import ( ComponentIdentifier, Conversation, @@ -62,6 +63,22 @@ def __init__(self, start_token: str = "⟪", end_token: str = "⟫") -> None: self._start_token = start_token self._end_token = end_token self.id = str(uuid4()) + self._prepended_conversation_normalizers: dict[str, MessageStringNormalizer] = {} + + def register_prepended_conversation_normalizer( + self, + *, + conversation_id: str, + message_normalizer: MessageStringNormalizer, + ) -> None: + """ + Register the formatter used to deliver structured prepended history on the next send. + + Args: + conversation_id: Conversation whose next request should include prepended history. + message_normalizer: Formatter the target should use for that history. + """ + self._prepended_conversation_normalizers[conversation_id] = message_normalizer async def send_prompt_async( self, @@ -108,15 +125,27 @@ async def send_prompt_async( for piece in request.message_pieces: piece.conversation_id = conversation_id - # Apply request converters - await self.convert_values_async(converter_configurations=request_converter_configurations, message=request) + prepended_conversation_normalizer = self._prepended_conversation_normalizers.pop( + request.conversation_id, + None, + ) + await self.convert_values_async( + converter_configurations=request_converter_configurations, + message=request, + ) await self._calc_hash_async(request=request) responses = None try: - responses = await target.send_prompt_async(message=request) + if prepended_conversation_normalizer: + responses = await target.send_prompt_async( + message=request, + prepended_conversation_normalizer=prepended_conversation_normalizer, + ) + else: + responses = await target.send_prompt_async(message=request) self.memory.add_message_to_memory(request=request) except EmptyResponseException: # Empty responses are retried, but we don't want them to stop execution diff --git a/pyrit/prompt_target/common/prompt_target.py b/pyrit/prompt_target/common/prompt_target.py index aaf918f4cf..99b8f5c330 100644 --- a/pyrit/prompt_target/common/prompt_target.py +++ b/pyrit/prompt_target/common/prompt_target.py @@ -6,6 +6,7 @@ from typing import Any, ClassVar, Literal, final from pyrit.memory import CentralMemory, MemoryInterface +from pyrit.message_normalizer import MessageStringNormalizer, PrependedConversationNormalizer from pyrit.models import ( ComponentIdentifier, Conversation, @@ -132,7 +133,12 @@ def __init__( logging.basicConfig(level=logging.INFO) @final - async def send_prompt_async(self, *, message: Message) -> list[Message]: + async def send_prompt_async( + self, + *, + message: Message, + prepended_conversation_normalizer: MessageStringNormalizer | None = None, + ) -> list[Message]: """ Validate, normalize, and send a prompt to the target. @@ -149,6 +155,8 @@ async def send_prompt_async(self, *, message: Message) -> list[Message]: Args: message (Message): The message to send. + prepended_conversation_normalizer (MessageStringNormalizer | None): Optional one-shot + formatter for structured prepended history on a target without editable history. Returns: list[Message]: Response messages from the target. @@ -157,7 +165,10 @@ async def send_prompt_async(self, *, message: Message) -> list[Message]: ValueError: If the message or normalized conversation are empty. """ message.validate() - normalized_conversation = await self._get_normalized_conversation_async(message=message) + normalized_conversation = await self._get_normalized_conversation_async( + message=message, + prepended_conversation_normalizer=prepended_conversation_normalizer, + ) if not normalized_conversation: raise ValueError("Normalization pipeline returned an empty conversation. Cannot send an empty request.") self._validate_request(normalized_conversation=normalized_conversation) @@ -220,7 +231,12 @@ def _validate_request(self, *, normalized_conversation: list[Message]) -> None: if not self.configuration.includes(capability=CapabilityName.MULTI_TURN) and len(normalized_conversation) > 1: raise ValueError(f"This target only supports a single turn conversation. {custom_configuration_message}") - async def _get_normalized_conversation_async(self, *, message: Message) -> list[Message]: + async def _get_normalized_conversation_async( + self, + *, + message: Message, + prepended_conversation_normalizer: MessageStringNormalizer | None = None, + ) -> list[Message]: """ Fetch the conversation from memory, append the current message, and run the normalization pipeline. @@ -235,6 +251,9 @@ async def _get_normalized_conversation_async(self, *, message: Message) -> list[ Args: message (Message): The current message to append. + prepended_conversation_normalizer (MessageStringNormalizer | None): Optional formatter + that combines the existing prepended history with this request before the standard + capability pipeline runs. Returns: list[Message]: The normalized conversation (possibly with system prompt squashed, @@ -245,6 +264,12 @@ async def _get_normalized_conversation_async(self, *, message: Message) -> list[ list(self._memory.get_conversation_messages(conversation_id=conversation_id)) if conversation_id else [] ) conversation.append(message) + if prepended_conversation_normalizer and not self.configuration.includes( + capability=CapabilityName.EDITABLE_HISTORY + ): + conversation = await PrependedConversationNormalizer( + message_normalizer=prepended_conversation_normalizer + ).normalize_async(conversation) normalized = await self.configuration.normalize_async(messages=conversation) if normalized: # Normalizers may create new Message objects (via Message.from_prompt) with diff --git a/pyrit/scenario/scenarios/airt/jailbreak.py b/pyrit/scenario/scenarios/airt/jailbreak.py index 710c7eba7c..9bfcc63f19 100644 --- a/pyrit/scenario/scenarios/airt/jailbreak.py +++ b/pyrit/scenario/scenarios/airt/jailbreak.py @@ -107,23 +107,14 @@ def _extra_default_factories() -> dict[str, AttackTechniqueFactory]: @cache def _build_jailbreak_technique() -> type[ScenarioTechnique]: """ - Build the Jailbreak technique class dynamically from every registered factory plus the - scenario-local defaults. - - The technique axis is the set of *attack techniques* a jailbreak is delivered through: the two - default deliveries (``prompt_sending`` and ``jailbreak_system_prompt``) plus whatever techniques - are registered (``role_play_*``, ``many_shot``, ``tap``, …). Jailbreak templates are a separate - selector (``num_jailbreaks`` / ``jailbreak_names``), so only the two deliveries are on by default - — crossing every template with every registered technique explodes quickly. + Build the Jailbreak technique class from its two scenario-owned delivery methods. Returns: type[ScenarioTechnique]: The dynamically generated technique enum class. """ - registry = AttackTechniqueRegistry.get_registry_singleton() - factories = list(registry.get_factories_or_raise().values()) + list(_extra_default_factories().values()) return AttackTechniqueRegistry.build_technique_class_from_factories( # type: ignore[return-value, ty:invalid-return-type] class_name="JailbreakTechnique", - factories=factories, + factories=list(_extra_default_factories().values()), default_names=set(_DEFAULT_TECHNIQUES), ) @@ -136,24 +127,23 @@ class Jailbreak(Scenario): selectors: - **dataset** — the harmful objectives (HarmBench). - - **techniques** — the *attack techniques* each jailbreak is delivered through. Two deliveries - are on by default: ``prompt_sending`` (the template rendered inline into the user message) and + - **techniques** — two delivery methods for each jailbreak: ``prompt_sending`` (the template + rendered inline into the user message) and ``jailbreak_system_prompt`` (the template set as the system prompt with the objective sent as - the user turn). The registry techniques (``role_play_*``, ``many_shot``, ``tap``, …) are - opt-in. + the user turn). - **jailbreaks** — which jailbreak templates to run (a random ``num_jailbreaks`` sample or an explicit ``jailbreak_names`` set). ``prompt_sending`` applies each template as a ``TextJailbreakConverter`` on the outgoing request, - so the objective is rendered inline into the template's ``{{prompt}}`` slot; this keeps that - delivery target-agnostic and lets it compose with every technique. ``jailbreak_system_prompt`` - instead sets the template as a native system prompt and sends the objective as its own user turn, - so it is only built for targets that natively support editable history and system prompts (it is - skipped for incapable targets, or raises if it is the only selected technique). Responses are - scored to determine whether the jailbreak succeeded (non-refusal). + so the objective is rendered inline into the template's ``{{prompt}}`` slot. + ``jailbreak_system_prompt`` instead sets the template as a native system prompt and sends the + objective as its own user turn, so it is only built for targets that natively support editable + history and system prompts (it is skipped for incapable targets, or raises if it is the only + selected technique). Responses are scored to determine whether the jailbreak succeeded + (non-refusal). """ - VERSION: int = 3 + VERSION: int = 4 #: Baseline (an un-jailbroken prompt-send over the objectives) is included by default: a model #: that complies with the bare objective is itself interesting signal. Callers opt out per run @@ -232,6 +222,30 @@ def __init__( scenario_result_id=scenario_result_id, ) + def _resolve_scenario_techniques(self, *, scenario_techniques: Any) -> list[ScenarioTechnique]: + """ + Resolve techniques while rejecting stale or incompatible enum members. + + Args: + scenario_techniques (Any): Requested Jailbreak technique members. + + Returns: + list[ScenarioTechnique]: Compatible concrete techniques. + + Raises: + ValueError: If a caller supplies members from an older or different + technique enum. + """ + if scenario_techniques: + incompatible = [item for item in scenario_techniques if not isinstance(item, self._technique_class)] + if incompatible: + values = [getattr(item, "value", repr(item)) for item in incompatible] + raise ValueError( + "Jailbreak received stale or incompatible techniques " + f"{values}. Select 'prompt_sending' or 'jailbreak_system_prompt'." + ) + return super()._resolve_scenario_techniques(scenario_techniques=scenario_techniques) + def _resolve_templates(self) -> list[str]: """ Resolve the jailbreak templates for this run, replaying the persisted set on resume. @@ -286,13 +300,12 @@ async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list """ Build one atomic attack per (technique x jailbreak template x dataset x attempt). - ``prompt_sending`` (and any opt-in registry techniques) deliver each jailbreak template as a - ``TextJailbreakConverter`` appended to that technique's request converters, so the objective - is rendered inline into the template's ``{{prompt}}`` slot on the wire — target-agnostic and - composable with every technique. ``jailbreak_system_prompt`` instead delivers the template as - a native system prompt (no converter) with the objective sent as its own user turn, so it is - only built when the objective target natively supports editable history and system prompts. - Results group by jailbreak template so per-template ASR rolls up naturally. + ``prompt_sending`` delivers each jailbreak template as a ``TextJailbreakConverter`` so the + objective is rendered inline into the template's ``{{prompt}}`` slot on the wire. + ``jailbreak_system_prompt`` instead delivers the template as a native system prompt (no + converter) with the objective sent as its own user turn, so it is only built when the + objective target natively supports editable history and system prompts. Results group by + jailbreak template so per-template ASR rolls up naturally. Args: context (ScenarioContext): The resolved runtime inputs for this run. @@ -314,17 +327,20 @@ async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list num_attempts = self.params.get("num_jailbreak_attempts", 1) technique_factories = resolve_technique_factories(context=context, extra_factories=_extra_default_factories()) + selected_names = {technique.value for technique in context.scenario_techniques} + missing = selected_names - set(technique_factories) + if missing: + raise ValueError( + "Jailbreak selected techniques that are no longer available: " + f"{sorted(missing)}. Refresh the plan and select a supported delivery method." + ) - # ``jailbreak_system_prompt`` is delivered separately (native system prompt, no converter); - # every other technique goes through the inline converter path. + prompt_sending_factory = technique_factories.get(_PROMPT_SENDING) system_selected = _JAILBREAK_SYSTEM_PROMPT in technique_factories - converter_factories = { - name: factory for name, factory in technique_factories.items() if name != _JAILBREAK_SYSTEM_PROMPT - } build_system_delivery = system_selected and self._target_supports_system_delivery(self._objective_target) if system_selected and not build_system_delivery: - if not converter_factories: + if prompt_sending_factory is None: raise ValueError( "The 'jailbreak_system_prompt' technique needs a target that natively supports " "editable history and system prompts. Choose a capable target or a different technique." @@ -353,22 +369,21 @@ async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list for template_file_name in self._resolved_jailbreaks: template_stem = Path(template_file_name).stem - if converter_factories: + if prompt_sending_factory is not None: jailbreak_converter = TextJailbreakConverter( jailbreak_template=TextJailBreak(template_file_name=template_file_name) ) - # Within the extra-converter stack, apply the jailbreak first (wrap the raw objective - # in the template), then any per-technique converters the caller layered on via - # ``--techniques :converter.*``. (A technique's own built-in converters, if any, - # still run ahead of this extra stack inside the factory.) + # Apply the jailbreak before any caller-supplied prompt_sending converters. technique_converters = { - technique_name: [jailbreak_converter, *self._technique_converters.get(technique_name, [])] - for technique_name in converter_factories + _PROMPT_SENDING: [ + jailbreak_converter, + *self._technique_converters.get(_PROMPT_SENDING, []), + ] } atomic_attacks.extend( self._build_delivery_attacks( builder=builder, - technique_factories=converter_factories, + technique_factories={_PROMPT_SENDING: prompt_sending_factory}, technique_converters=technique_converters, dataset_groups=context.seed_groups_by_dataset, template_stem=template_stem, diff --git a/tests/unit/executor/attack/component/test_conversation_manager.py b/tests/unit/executor/attack/component/test_conversation_manager.py index 550e4b631d..8ddca07ef0 100644 --- a/tests/unit/executor/attack/component/test_conversation_manager.py +++ b/tests/unit/executor/attack/component/test_conversation_manager.py @@ -17,12 +17,14 @@ - get_prepended_turn_count: Counts assistant messages in a conversation """ +import base64 import uuid from unittest.mock import AsyncMock, MagicMock import pytest from unit.mocks import get_mock_scorer_identifier +from pyrit.converter import Base64Converter, Converter, ConverterResult from pyrit.executor.attack import ConversationManager, ConversationState from pyrit.executor.attack.component import PrependedConversationConfig from pyrit.executor.attack.component.conversation_manager import ( @@ -34,7 +36,7 @@ from pyrit.executor.attack.core import AttackContext from pyrit.executor.attack.core.attack_parameters import AttackParameters from pyrit.message_normalizer import ConversationContextNormalizer -from pyrit.models import ComponentIdentifier, Message, MessagePiece, Score +from pyrit.models import ComponentIdentifier, Message, MessagePiece, PromptDataType, Score from pyrit.prompt_normalizer import ConverterConfiguration, PromptNormalizer from pyrit.prompt_target import PromptTarget @@ -59,6 +61,16 @@ class _TestAttackContext(AttackContext): last_score: Score | None = None +class _ImageOutputConverter(Converter): + """A deterministic text-to-image converter for prepended-history adaptation tests.""" + + SUPPORTED_INPUT_TYPES: tuple[PromptDataType, ...] = ("text",) + SUPPORTED_OUTPUT_TYPES: tuple[PromptDataType, ...] = ("image_path",) + + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: + return ConverterResult(output_text="converted.png", output_type="image_path") + + # ============================================================================= # Fixtures # ============================================================================= @@ -710,105 +722,41 @@ async def test_converts_assistant_to_simulated_assistant( assert stored[0].get_piece().role == "simulated_assistant" assert stored[0].get_piece().api_role == "assistant" - async def test_normalizes_for_non_chat_target_by_default( + async def test_stores_prepended_conversation_for_non_editable_target( self, attack_identifier: ComponentIdentifier, mock_prompt_target: MagicMock, sample_conversation: list[Message], ) -> None: - """Test that prepended conversation is normalized for non-chat targets by default.""" manager = ConversationManager() conversation_id = str(uuid.uuid4()) context = _TestAttackContext(params=AttackParameters(objective="Test objective")) context.prepended_conversation = sample_conversation - context.next_message = None - - # By default, should normalize (not raise) - matching PrependedConversationConfig field default - await manager.initialize_context_async( - context=context, - target=mock_prompt_target, - conversation_id=conversation_id, - ) - # next_message should now contain the normalized prepended context - assert context.next_message is not None - text_value = context.next_message.get_piece().original_value - assert len(text_value) > 0 - - async def test_normalizes_for_non_chat_target_when_configured( - self, - attack_identifier: ComponentIdentifier, - mock_prompt_target: MagicMock, - sample_conversation: list[Message], - ) -> None: - """Test that non-chat target normalizes prepended conversation when configured.""" - manager = ConversationManager() - conversation_id = str(uuid.uuid4()) - context = _TestAttackContext(params=AttackParameters(objective="Test objective")) - context.prepended_conversation = sample_conversation - context.next_message = Message.from_prompt(prompt="Next message", role="user") - - config = PrependedConversationConfig() - - await manager.initialize_context_async( + state = await manager.initialize_context_async( context=context, target=mock_prompt_target, conversation_id=conversation_id, - prepended_conversation_config=config, ) - # next_message should now contain the prepended context - assert context.next_message is not None - text_value = context.next_message.get_piece().original_value - assert "Next message" in text_value - assert "Hello" in text_value or "doing well" in text_value - - @pytest.mark.parametrize( - "prepended_conversation_config", - [ - None, - PrependedConversationConfig(message_normalizer=ConversationContextNormalizer()), - ], - ) - async def test_system_prompt_for_non_chat_target_preserves_instruction_and_objective( - self, - attack_identifier: ComponentIdentifier, - mock_prompt_target: MagicMock, - prepended_conversation_config: PrependedConversationConfig | None, - ) -> None: - manager = ConversationManager() - context = _TestAttackContext(params=AttackParameters(objective="Explain saponification")) - context.prepended_conversation = [Message.from_system_prompt("You are a chemistry tutor")] - - await manager.initialize_context_async( - context=context, - target=mock_prompt_target, - conversation_id=str(uuid.uuid4()), - prepended_conversation_config=prepended_conversation_config, - ) - - assert context.next_message is not None - assert context.next_message.get_value() == "Turn 1:\nuser: You are a chemistry tutor\n\nExplain saponification" - - await manager.initialize_context_async( - context=context, - target=mock_prompt_target, - conversation_id=str(uuid.uuid4()), - prepended_conversation_config=prepended_conversation_config, - ) - - assert context.next_message.get_value() == "Turn 1:\nuser: You are a chemistry tutor\n\nExplain saponification" + stored = manager.get_conversation(conversation_id) + assert len(stored) == 2 + assert [message.api_role for message in stored] == ["user", "assistant"] + assert stored[1].get_piece().role == "simulated_assistant" + assert state.turn_count == 1 + assert context.next_message is None - async def test_system_prompt_for_non_chat_target_preserves_supplied_next_message( + async def test_non_editable_target_does_not_rewrite_supplied_next_message( self, attack_identifier: ComponentIdentifier, mock_prompt_target: MagicMock, ) -> None: manager = ConversationManager() + next_message = Message.from_prompt(prompt="Caller-supplied question", role="user") context = _TestAttackContext( params=AttackParameters( objective="Unused objective", - next_message=Message.from_prompt(prompt="Caller-supplied question", role="user"), + next_message=next_message, ) ) context.prepended_conversation = [Message.from_system_prompt("Follow the policy")] @@ -819,48 +767,35 @@ async def test_system_prompt_for_non_chat_target_preserves_supplied_next_message conversation_id=str(uuid.uuid4()), ) - assert context.next_message is not None - assert context.next_message.get_value() == "Turn 1:\nuser: Follow the policy\n\nCaller-supplied question" + assert context.next_message is next_message + assert context.next_message.get_value() == "Caller-supplied question" - async def test_system_prompt_for_non_chat_target_preserves_multimodal_next_message( + async def test_non_editable_target_registers_custom_first_send_formatter( self, attack_identifier: ComponentIdentifier, + mock_prompt_normalizer: MagicMock, mock_prompt_target: MagicMock, + sample_conversation: list[Message], ) -> None: - manager = ConversationManager() - image_piece = MessagePiece( - role="user", - original_value="diagram.png", - original_value_data_type="image_path", - ) - context = _TestAttackContext( - params=AttackParameters( - objective="Unused objective", - next_message=Message(message_pieces=[image_piece]), - ) - ) - context.prepended_conversation = [Message.from_system_prompt("Describe images precisely")] + manager = ConversationManager(prompt_normalizer=mock_prompt_normalizer) + conversation_id = str(uuid.uuid4()) + context = _TestAttackContext(params=AttackParameters(objective="Test objective")) + context.prepended_conversation = sample_conversation + message_normalizer = MagicMock(spec=ConversationContextNormalizer) + config = PrependedConversationConfig(message_normalizer=message_normalizer) await manager.initialize_context_async( context=context, target=mock_prompt_target, - conversation_id=str(uuid.uuid4()), + conversation_id=conversation_id, + prepended_conversation_config=config, ) - assert context.next_message is not None - assert len(context.next_message.message_pieces) == 2 - assert context.next_message.message_pieces[0].converted_value == "Turn 1:\nuser: Describe images precisely" - assert context.next_message.message_pieces[1].converted_value == "diagram.png" - assert context.next_message.message_pieces[1].original_value_data_type == "image_path" - - await manager.initialize_context_async( - context=context, - target=mock_prompt_target, - conversation_id=str(uuid.uuid4()), + mock_prompt_normalizer.register_prepended_conversation_normalizer.assert_called_once_with( + conversation_id=conversation_id, + message_normalizer=message_normalizer, ) - - assert len(context.next_message.message_pieces) == 2 - assert context.next_message.message_pieces[0].converted_value == "Turn 1:\nuser: Describe images precisely" + message_normalizer.normalize_string_async.assert_not_called() async def test_returns_turn_count_for_multi_turn_attacks( self, @@ -1064,127 +999,136 @@ async def test_prepended_conversation_ignores_true_scores( class TestPrependedConversationConfigSettings: """Tests for PrependedConversationConfig settings in initialize_context_async.""" - # ------------------------------------------------------------------------- - # non_chat_target_behavior Tests - # ------------------------------------------------------------------------- - - async def test_non_chat_target_behavior_normalize_is_default( + async def test_non_editable_target_converts_selected_roles_before_storage( self, attack_identifier: ComponentIdentifier, mock_prompt_target: MagicMock, sample_conversation: list[Message], ) -> None: - """Test that non-chat targets normalize by default (no config), matching dataclass field default.""" manager = ConversationManager() - conversation_id = str(uuid.uuid4()) context = _TestAttackContext(params=AttackParameters(objective="Test objective")) context.prepended_conversation = sample_conversation - context.next_message = None + context.next_message = Message.from_prompt(prompt="live request", role="user") + converter_config = ConverterConfiguration.from_converters(converters=[Base64Converter()]) - # Should normalize by default (matching PrependedConversationConfig field default) + conversation_id = str(uuid.uuid4()) await manager.initialize_context_async( context=context, target=mock_prompt_target, conversation_id=conversation_id, + request_converters=converter_config, ) - # next_message should contain normalized context - assert context.next_message is not None - text_value = context.next_message.get_piece().original_value - assert len(text_value) > 0 + stored = manager.get_conversation(conversation_id) + encoded_user = base64.b64encode(b"Hello, how are you?").decode() + assert stored[0].get_piece().converted_value == encoded_user + assert stored[1].get_piece().converted_value == "I'm doing well, thank you!" + assert context.next_message.get_piece().converted_value == "live request" - async def test_non_chat_target_behavior_normalize_first_turn_creates_next_message( + async def test_non_editable_target_converts_assistant_history_only_when_opted_in( self, attack_identifier: ComponentIdentifier, mock_prompt_target: MagicMock, sample_conversation: list[Message], ) -> None: - """Test that normalize_first_turn creates next_message when none exists.""" manager = ConversationManager() - conversation_id = str(uuid.uuid4()) context = _TestAttackContext(params=AttackParameters(objective="Test objective")) context.prepended_conversation = sample_conversation - context.next_message = None - - config = PrependedConversationConfig() + context.next_message = Message.from_prompt(prompt="live request", role="user") + converter_config = ConverterConfiguration.from_converters(converters=[Base64Converter()]) + config = PrependedConversationConfig(apply_converters_to_roles=["assistant"]) + conversation_id = str(uuid.uuid4()) await manager.initialize_context_async( context=context, target=mock_prompt_target, conversation_id=conversation_id, + request_converters=converter_config, prepended_conversation_config=config, ) - # Should have created a next_message with the normalized context - assert context.next_message is not None - text_value = context.next_message.get_piece().original_value - assert len(text_value) > 0 + stored = manager.get_conversation(conversation_id) + encoded_assistant = base64.b64encode(b"I'm doing well, thank you!").decode() + assert stored[0].get_piece().converted_value == "Hello, how are you?" + assert stored[1].get_piece().converted_value == encoded_assistant - async def test_non_chat_target_behavior_normalize_first_turn_prepends_to_existing_message( + async def test_non_editable_target_rejects_non_text_output_from_current_converter( self, attack_identifier: ComponentIdentifier, mock_prompt_target: MagicMock, sample_conversation: list[Message], ) -> None: - """Test that normalize_first_turn prepends context to existing next_message.""" manager = ConversationManager() - conversation_id = str(uuid.uuid4()) context = _TestAttackContext(params=AttackParameters(objective="Test objective")) context.prepended_conversation = sample_conversation - context.next_message = Message.from_prompt(prompt="My question", role="user") - - config = PrependedConversationConfig() + converter_config = ConverterConfiguration.from_converters(converters=[_ImageOutputConverter()]) - await manager.initialize_context_async( - context=context, - target=mock_prompt_target, - conversation_id=conversation_id, - prepended_conversation_config=config, - ) + with pytest.raises(ValueError, match="non-text output types.*image_path"): + await manager.initialize_context_async( + context=context, + target=mock_prompt_target, + conversation_id=str(uuid.uuid4()), + request_converters=converter_config, + ) - # Should have prepended context to existing message - text_value = context.next_message.get_piece().original_value - assert "My question" in text_value - # Context should come before the original question - question_index = text_value.find("My question") - assert question_index > 0 # Context should be prepended + assert sample_conversation[0].get_piece().converted_value_data_type == "text" - async def test_non_chat_target_behavior_normalize_returns_empty_state( + async def test_non_editable_target_preserves_converter_piece_indexes( self, attack_identifier: ComponentIdentifier, mock_prompt_target: MagicMock, - sample_conversation: list[Message], ) -> None: - """Test that normalize_first_turn returns empty ConversationState (no turn tracking).""" manager = ConversationManager() - conversation_id = str(uuid.uuid4()) context = _TestAttackContext(params=AttackParameters(objective="Test objective")) - context.prepended_conversation = sample_conversation - - config = PrependedConversationConfig() + context.prepended_conversation = [ + Message( + message_pieces=[ + MessagePiece( + role="user", + original_value="first piece", + conversation_id="seed", + sequence=0, + ), + MessagePiece( + role="user", + original_value="second piece", + conversation_id="seed", + sequence=0, + ), + ] + ) + ] + context.next_message = Message.from_prompt(prompt="live request", role="user") + converter_config = [ + ConverterConfiguration( + converters=[Base64Converter()], + indexes_to_apply=[0], + ) + ] - state = await manager.initialize_context_async( + conversation_id = str(uuid.uuid4()) + await manager.initialize_context_async( context=context, target=mock_prompt_target, conversation_id=conversation_id, - prepended_conversation_config=config, + request_converters=converter_config, ) - # Non-chat targets don't track turns - assert state.turn_count == 0 - assert state.last_assistant_message_scores == [] + stored_pieces = manager.get_conversation(conversation_id)[0].message_pieces + assert stored_pieces[0].converted_value == base64.b64encode(b"first piece").decode() + assert stored_pieces[1].converted_value == "second piece" # ------------------------------------------------------------------------- # apply_converters_to_roles Tests # ------------------------------------------------------------------------- - async def test_apply_converters_to_roles_default_applies_to_all( + async def test_apply_converters_to_roles_default_applies_to_user_only( self, attack_identifier: ComponentIdentifier, mock_chat_target: MagicMock, sample_conversation: list[Message], ) -> None: - """Test that converters are applied to all roles by default.""" + """Test that converters are applied only to user history by default.""" mock_normalizer = MagicMock(spec=PromptNormalizer) mock_normalizer.convert_values_async = AsyncMock() manager = ConversationManager(prompt_normalizer=mock_normalizer) @@ -1201,8 +1145,7 @@ async def test_apply_converters_to_roles_default_applies_to_all( request_converters=converter_config, ) - # convert_values_async should be called for each message (both user and assistant) - assert mock_normalizer.convert_values_async.call_count == 2 + mock_normalizer.convert_values_async.assert_awaited_once() async def test_apply_converters_to_roles_user_only( self, @@ -1295,66 +1238,25 @@ async def test_apply_converters_to_roles_empty_list_skips_all( async def test_message_normalizer_default_uses_conversation_context_normalizer( self, attack_identifier: ComponentIdentifier, + mock_prompt_normalizer: MagicMock, mock_prompt_target: MagicMock, sample_conversation: list[Message], ) -> None: - """Test that default normalizer produces Turn N format.""" - manager = ConversationManager() - conversation_id = str(uuid.uuid4()) - context = _TestAttackContext(params=AttackParameters(objective="Test objective")) - context.prepended_conversation = sample_conversation - context.next_message = None - - config = PrependedConversationConfig() - - await manager.initialize_context_async( - context=context, - target=mock_prompt_target, - conversation_id=conversation_id, - prepended_conversation_config=config, - ) - - # Default ConversationContextNormalizer produces "Turn N:" format - assert context.next_message is not None - text_value = context.next_message.get_piece().original_value - assert "Turn 1" in text_value or "turn 1" in text_value.lower() - - async def test_message_normalizer_custom_normalizer_is_used( - self, - attack_identifier: ComponentIdentifier, - mock_prompt_target: MagicMock, - sample_conversation: list[Message], - ) -> None: - """Test that custom message_normalizer is used when provided.""" - from pyrit.message_normalizer import MessageStringNormalizer - - # Create a mock normalizer that returns a specific format - mock_normalizer = MagicMock(spec=MessageStringNormalizer) - mock_normalizer.normalize_string_async = AsyncMock(return_value="CUSTOM_FORMAT: test content") - - manager = ConversationManager() + """Test that the default formatter is registered for target-side adaptation.""" + manager = ConversationManager(prompt_normalizer=mock_prompt_normalizer) conversation_id = str(uuid.uuid4()) context = _TestAttackContext(params=AttackParameters(objective="Test objective")) context.prepended_conversation = sample_conversation - context.next_message = None - - config = PrependedConversationConfig( - message_normalizer=mock_normalizer, - ) await manager.initialize_context_async( context=context, target=mock_prompt_target, conversation_id=conversation_id, - prepended_conversation_config=config, ) - # Verify custom normalizer was called - mock_normalizer.normalize_string_async.assert_called_once() - # Verify the custom format is in the message - assert context.next_message is not None - text_value = context.next_message.get_piece().original_value - assert "CUSTOM_FORMAT: test content" in text_value + registered = mock_prompt_normalizer.register_prepended_conversation_normalizer.call_args.kwargs + assert registered["conversation_id"] == conversation_id + assert isinstance(registered["message_normalizer"], ConversationContextNormalizer) # ------------------------------------------------------------------------- # Chat Target Behavior (Config has no effect) diff --git a/tests/unit/executor/attack/component/test_prepended_conversation_config.py b/tests/unit/executor/attack/component/test_prepended_conversation_config.py index b1c34a6770..546a7f1dbf 100644 --- a/tests/unit/executor/attack/component/test_prepended_conversation_config.py +++ b/tests/unit/executor/attack/component/test_prepended_conversation_config.py @@ -1,17 +1,15 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from typing import get_args from unittest.mock import MagicMock from pyrit.executor.attack.component.prepended_conversation_config import PrependedConversationConfig from pyrit.message_normalizer import ConversationContextNormalizer -from pyrit.models import ChatMessageRole -def test_default_init_apply_converters_to_all_roles(): +def test_default_init_apply_converters_to_user_role(): config = PrependedConversationConfig() - assert config.apply_converters_to_roles == list(get_args(ChatMessageRole)) + assert config.apply_converters_to_roles == ["user"] def test_default_init_message_normalizer_is_none(): diff --git a/tests/unit/executor/attack/multi_turn/test_red_teaming.py b/tests/unit/executor/attack/multi_turn/test_red_teaming.py index 4603152590..51b0f326b7 100644 --- a/tests/unit/executor/attack/multi_turn/test_red_teaming.py +++ b/tests/unit/executor/attack/multi_turn/test_red_teaming.py @@ -298,6 +298,7 @@ def test_init_with_all_custom_configurations( assert attack._request_converters == converter_config.request_converters assert attack._response_converters == converter_config.response_converters assert attack._prompt_normalizer == mock_prompt_normalizer + assert attack._conversation_manager._prompt_normalizer is mock_prompt_normalizer assert attack._max_turns == 20 def test_init_without_objective_scorer_raises_error( diff --git a/tests/unit/executor/attack/single_turn/test_prompt_sending.py b/tests/unit/executor/attack/single_turn/test_prompt_sending.py index 5d4fda2209..76d0b8cf62 100644 --- a/tests/unit/executor/attack/single_turn/test_prompt_sending.py +++ b/tests/unit/executor/attack/single_turn/test_prompt_sending.py @@ -1,20 +1,23 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import base64 import uuid from unittest.mock import AsyncMock, MagicMock, patch import pytest -from unit.mocks import get_mock_scorer_identifier, get_mock_target_identifier +from unit.mocks import MockPromptTarget, get_mock_scorer_identifier, get_mock_target_identifier from pyrit.converter import Base64Converter, StringJoinConverter from pyrit.executor.attack import ( AttackConverterConfig, AttackParameters, AttackScoringConfig, + PrependedConversationConfig, PromptSendingAttack, SingleTurnAttackContext, ) +from pyrit.memory import CentralMemory from pyrit.models import ( AttackOutcome, AttackResult, @@ -27,6 +30,8 @@ ) from pyrit.prompt_normalizer import ConverterConfiguration, PromptNormalizer from pyrit.prompt_target import PromptTarget +from pyrit.prompt_target.common.target_capabilities import TargetCapabilities +from pyrit.prompt_target.common.target_configuration import TargetConfiguration from pyrit.score import Scorer, TrueFalseScorer @@ -280,6 +285,86 @@ async def test_setup_updates_conversation_state_with_converters(self, mock_targe memory_labels={}, ) + async def test_default_converter_scoping_preserves_simulated_assistant_history(self): + target = MockPromptTarget() + converter_config = AttackConverterConfig( + request_converters=ConverterConfiguration.from_converters(converters=[Base64Converter()]) + ) + attack = PromptSendingAttack(objective_target=target, attack_converter_config=converter_config) + prepended_user = "prepended user request" + simulated_response = "simulated assistant response" + final_request = "live final request" + + result = await attack.execute_async( + objective="Test objective", + prepended_conversation=[ + Message.from_prompt(prompt=prepended_user, role="user"), + Message.from_prompt(prompt=simulated_response, role="assistant"), + ], + next_message=Message.from_prompt(prompt=final_request, role="user"), + ) + + pieces = CentralMemory.get_memory_instance().get_message_pieces(conversation_id=result.conversation_id) + assistant_piece = next(piece for piece in pieces if piece.original_value == simulated_response) + final_piece = next(piece for piece in pieces if piece.original_value == final_request) + + assert assistant_piece.role == "simulated_assistant" + assert assistant_piece.converted_value == assistant_piece.original_value + assert assistant_piece.converter_identifiers == [] + assert final_piece.converted_value != final_piece.original_value + assert [identifier.class_name for identifier in final_piece.converter_identifiers] == ["Base64Converter"] + + async def test_explicit_assistant_role_opt_in_converts_simulated_history(self): + target = MockPromptTarget() + converter_config = AttackConverterConfig( + request_converters=ConverterConfiguration.from_converters(converters=[Base64Converter()]) + ) + attack = PromptSendingAttack( + objective_target=target, + attack_converter_config=converter_config, + prepended_conversation_config=PrependedConversationConfig(apply_converters_to_roles=["assistant"]), + ) + simulated_response = "assistant history explicitly converted" + + result = await attack.execute_async( + objective="Test objective", + prepended_conversation=[Message.from_prompt(prompt=simulated_response, role="assistant")], + next_message=Message.from_prompt(prompt="live request", role="user"), + ) + + pieces = CentralMemory.get_memory_instance().get_message_pieces(conversation_id=result.conversation_id) + assistant_piece = next(piece for piece in pieces if piece.original_value == simulated_response) + + assert assistant_piece.role == "simulated_assistant" + assert assistant_piece.converted_value != assistant_piece.original_value + assert [identifier.class_name for identifier in assistant_piece.converter_identifiers] == ["Base64Converter"] + + async def test_non_chat_target_converts_history_by_role_before_flattening(self): + target = MockPromptTarget() + target._configuration = TargetConfiguration(capabilities=TargetCapabilities()) + converter_config = AttackConverterConfig( + request_converters=ConverterConfiguration.from_converters(converters=[Base64Converter()]) + ) + attack = PromptSendingAttack(objective_target=target, attack_converter_config=converter_config) + prepended_user = "prepended user request" + simulated_response = "simulated assistant response" + final_request = "live final request" + + await attack.execute_async( + objective="Test objective", + prepended_conversation=[ + Message.from_prompt(prompt=prepended_user, role="user"), + Message.from_prompt(prompt=simulated_response, role="assistant"), + ], + next_message=Message.from_prompt(prompt=final_request, role="user"), + ) + + encoded_user = base64.b64encode(prepended_user.encode()).decode() + encoded_final_request = base64.b64encode(final_request.encode()).decode() + assert target.prompt_sent == [ + f"Turn 1:\nuser: {encoded_user}\nassistant: {simulated_response}\n\n{encoded_final_request}" + ] + @pytest.mark.usefixtures("patch_central_database") class TestPromptPreparation: diff --git a/tests/unit/prompt_normalizer/test_prompt_normalizer.py b/tests/unit/prompt_normalizer/test_prompt_normalizer.py index 025d6d6c77..01ad9bf435 100644 --- a/tests/unit/prompt_normalizer/test_prompt_normalizer.py +++ b/tests/unit/prompt_normalizer/test_prompt_normalizer.py @@ -25,6 +25,7 @@ get_execution_context, ) from pyrit.memory import CentralMemory +from pyrit.message_normalizer import ConversationContextNormalizer from pyrit.models import ( Message, MessagePiece, @@ -115,6 +116,39 @@ async def test_send_prompt_async_multiple_converters(mock_memory_instance, seed_ assert prompt_target.prompt_sent == ["S_G_V_s_b_G_8_="] +async def test_send_prompt_async_passes_registered_prepended_formatter_once(mock_memory_instance): + prompt_target = MagicMock(spec=PromptTarget) + prompt_target.get_identifier.return_value = get_mock_target_identifier("MockTarget") + prompt_target.send_prompt_async = AsyncMock( + side_effect=[ + [MessagePiece(role="assistant", original_value="first").to_message()], + [MessagePiece(role="assistant", original_value="second").to_message()], + ] + ) + normalizer = PromptNormalizer() + formatter = ConversationContextNormalizer() + conversation_id = "prepended-conversation" + normalizer.register_prepended_conversation_normalizer( + conversation_id=conversation_id, + message_normalizer=formatter, + ) + + await normalizer.send_prompt_async( + message=Message.from_prompt(prompt="first request", role="user"), + target=prompt_target, + conversation_id=conversation_id, + ) + await normalizer.send_prompt_async( + message=Message.from_prompt(prompt="second request", role="user"), + target=prompt_target, + conversation_id=conversation_id, + ) + + first_call, second_call = prompt_target.send_prompt_async.await_args_list + assert first_call.kwargs["prepended_conversation_normalizer"] is formatter + assert "prepended_conversation_normalizer" not in second_call.kwargs + + async def test_send_prompt_async_no_response_adds_memory(mock_memory_instance, seed_group): prompt_target = MagicMock() prompt_target.send_prompt_async = AsyncMock(return_value=None) diff --git a/tests/unit/prompt_target/target/test_normalize_async_integration.py b/tests/unit/prompt_target/target/test_normalize_async_integration.py index 407f1222a2..438d0ab8f8 100644 --- a/tests/unit/prompt_target/target/test_normalize_async_integration.py +++ b/tests/unit/prompt_target/target/test_normalize_async_integration.py @@ -13,9 +13,11 @@ import pytest from openai.types.chat import ChatCompletion from openai.types.responses import ResponseOutputMessage, ResponseOutputText +from unit.mocks import MockPromptTarget from pyrit.memory.memory_interface import MemoryInterface -from pyrit.models import Message, MessagePiece +from pyrit.message_normalizer import ConversationContextNormalizer, MessageStringNormalizer +from pyrit.models import ComponentIdentifier, Message, MessagePiece from pyrit.prompt_target import AzureMLChatTarget, OpenAIChatTarget from pyrit.prompt_target.common.target_capabilities import ( CapabilityHandlingPolicy, @@ -489,3 +491,180 @@ async def test_get_normalized_conversation_passthrough_when_no_adaptation_needed assert result[0].get_value() == "be nice" assert result[1].get_piece().api_role == "user" assert result[1].get_value() == "hello" + + +# --------------------------------------------------------------------------- +# Prepended history adaptation +# --------------------------------------------------------------------------- + + +@pytest.mark.usefixtures("patch_central_database") +async def test_non_editable_target_adapts_prepended_history_without_mutating_memory(): + target = MockPromptTarget() + target._configuration = TargetConfiguration(capabilities=TargetCapabilities()) + + prepended_user = _make_message(role="user", content="original history") + prepended_user.get_piece().converted_value = "converted history" + prepended_user.get_piece().converter_identifiers = [ + ComponentIdentifier(class_name="TestConverter", class_module="tests") + ] + prepended_assistant = _make_message(role="simulated_assistant", content="assistant history") + live_request = _make_message(role="user", content="original live") + live_request.get_piece().converted_value = "converted live" + memory_messages: MutableSequence[Message] = [prepended_user, prepended_assistant] + + mock_memory = MagicMock(spec=MemoryInterface) + mock_memory.get_conversation_messages.return_value = memory_messages + target._memory = mock_memory + + result = await target._get_normalized_conversation_async( + message=live_request, + prepended_conversation_normalizer=ConversationContextNormalizer(), + ) + + assert len(result) == 1 + assert result[0].get_piece().original_value == ( + "Turn 1:\nuser: original history\nassistant: assistant history\n\noriginal live" + ) + assert result[0].get_piece().converted_value == ( + "Turn 1:\nuser: converted history\nassistant: assistant history\n\nconverted live" + ) + assert len(memory_messages) == 2 + assert memory_messages[0].get_piece().original_value == "original history" + + +@pytest.mark.usefixtures("patch_central_database") +async def test_non_editable_target_preserves_system_history_and_multimodal_live_request(): + target = MockPromptTarget() + target._configuration = TargetConfiguration( + capabilities=TargetCapabilities( + supports_multi_message_pieces=True, + input_modalities=frozenset({frozenset({"text", "image_path"})}), + ) + ) + system_message = _make_message(role="system", content="Describe images precisely") + live_request = Message( + message_pieces=[ + MessagePiece( + role="user", + original_value="diagram.png", + converted_value="diagram.png", + original_value_data_type="image_path", + converted_value_data_type="image_path", + conversation_id="conv1", + ) + ] + ) + mock_memory = MagicMock(spec=MemoryInterface) + mock_memory.get_conversation_messages.return_value = [system_message] + target._memory = mock_memory + + result = await target._get_normalized_conversation_async( + message=live_request, + prepended_conversation_normalizer=ConversationContextNormalizer(), + ) + + assert len(result) == 1 + assert len(result[0].message_pieces) == 2 + assert result[0].message_pieces[0].converted_value == "Turn 1:\nuser: Describe images precisely" + assert result[0].message_pieces[1].converted_value == "diagram.png" + assert result[0].message_pieces[1].converted_value_data_type == "image_path" + + +@pytest.mark.usefixtures("patch_central_database") +async def test_prepended_history_adapter_is_used_only_when_explicitly_passed(): + target = MockPromptTarget() + target._configuration = TargetConfiguration( + capabilities=TargetCapabilities( + supports_multi_turn=True, + supports_system_prompt=True, + ) + ) + prepended = _make_message(role="user", content="prepended") + prior_live = _make_message(role="user", content="first live") + prior_response = _make_message(role="assistant", content="first response") + second_live = _make_message(role="user", content="second live") + + mock_memory = MagicMock(spec=MemoryInterface) + mock_memory.get_conversation_messages.side_effect = [ + [prepended], + [prepended, prior_live, prior_response], + ] + target._memory = mock_memory + + await target.send_prompt_async( + message=prior_live, + prepended_conversation_normalizer=ConversationContextNormalizer(), + ) + await target.send_prompt_async(message=second_live) + + assert target.prompt_sent == ["Turn 1:\nuser: prepended\n\nfirst live", "second live"] + + +@pytest.mark.usefixtures("patch_central_database") +async def test_non_editable_target_uses_custom_prepended_formatter(): + target = MockPromptTarget() + target._configuration = TargetConfiguration(capabilities=TargetCapabilities()) + mock_memory = MagicMock(spec=MemoryInterface) + mock_memory.get_conversation_messages.return_value = [_make_message(role="user", content="prepended")] + target._memory = mock_memory + formatter = MagicMock(spec=MessageStringNormalizer) + formatter.normalize_string_async = AsyncMock(return_value="CUSTOM HISTORY") + + result = await target._get_normalized_conversation_async( + message=_make_message(role="user", content="live"), + prepended_conversation_normalizer=formatter, + ) + + assert result[0].get_value() == "CUSTOM HISTORY\n\nlive" + formatter.normalize_string_async.assert_awaited_once() + + +@pytest.mark.usefixtures("patch_central_database") +async def test_non_editable_target_rejects_non_text_converted_prepended_history(): + target = MockPromptTarget() + target._configuration = TargetConfiguration(capabilities=TargetCapabilities()) + prepended = _make_message(role="user", content="original") + prepended.get_piece().converted_value = "converted.png" + prepended.get_piece().converted_value_data_type = "image_path" + prepended.get_piece().converter_identifiers = [ + ComponentIdentifier(class_name="ImageConverter", class_module="tests") + ] + mock_memory = MagicMock(spec=MemoryInterface) + mock_memory.get_conversation_messages.return_value = [prepended] + target._memory = mock_memory + + with pytest.raises(ValueError, match="non-text output types.*image_path"): + await target._get_normalized_conversation_async( + message=_make_message(role="user", content="live"), + prepended_conversation_normalizer=ConversationContextNormalizer(), + ) + + +@pytest.mark.usefixtures("patch_central_database") +async def test_non_editable_target_allows_preexisting_non_text_history_with_converter_provenance(): + target = MockPromptTarget() + target._configuration = TargetConfiguration(capabilities=TargetCapabilities()) + prepended = Message( + message_pieces=[ + MessagePiece( + role="user", + original_value="existing.png", + converted_value="existing.png", + original_value_data_type="image_path", + converted_value_data_type="image_path", + conversation_id="conv1", + converter_identifiers=[ComponentIdentifier(class_name="PriorConverter", class_module="tests")], + ) + ] + ) + mock_memory = MagicMock(spec=MemoryInterface) + mock_memory.get_conversation_messages.return_value = [prepended] + target._memory = mock_memory + + result = await target._get_normalized_conversation_async( + message=_make_message(role="user", content="live"), + prepended_conversation_normalizer=ConversationContextNormalizer(), + ) + + assert result[0].get_value() == "Turn 1:\nuser: [Image_path]\n\nlive" diff --git a/tests/unit/scenario/airt/test_jailbreak.py b/tests/unit/scenario/airt/test_jailbreak.py index 5cce529015..b055e087aa 100644 --- a/tests/unit/scenario/airt/test_jailbreak.py +++ b/tests/unit/scenario/airt/test_jailbreak.py @@ -16,6 +16,7 @@ from pyrit.prompt_target import PromptTarget from pyrit.registry import TargetRegistry from pyrit.registry.components.attack_technique_registry import AttackTechniqueRegistry +from pyrit.registry.components.scenario_registry import ScenarioRegistry from pyrit.scenario.core import BaselineAttackPolicy from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory from pyrit.scenario.scenarios.airt.jailbreak import ( @@ -41,13 +42,7 @@ def _technique_class(): @pytest.fixture(autouse=True) def reset_technique_registry(): - """Populate the attack-technique registry so the dynamic technique class can be built. - - Mirrors the RapidResponse test setup: reset the registries, register a mock adversarial - target (so factory construction does not fall back to a real target), and register the core - technique factories. The build cache is cleared around each test so the class reflects the - freshly-registered factories. - """ + """Populate the attack-technique registry used by the shared matrix factory resolver.""" AttackTechniqueRegistry.reset_registry_singleton() TargetRegistry.reset_registry_singleton() _build_jailbreak_technique.cache_clear() @@ -309,9 +304,8 @@ async def test_jailbreak_delivered_as_request_converter( """The crux: the jailbreak template reaches the target as a ``TextJailbreakConverter`` on the technique's outgoing requests (not as prepended framing on the seed group). - Delivery via ``factory.create(extra_request_converters=...)`` is what keeps the scenario - target-agnostic and composable with every technique. Also assert the seed groups carry no - prepended jailbreak framing. + Delivery via ``factory.create(extra_request_converters=...)`` keeps the prompt-sending path + target-agnostic. Also assert the seed groups carry no prepended jailbreak framing. """ captured: list[Any] = [] original_create = AttackTechniqueFactory.create @@ -328,9 +322,7 @@ def _spy_create(self, **kwargs): assert captured, "Expected factory.create to be called" converters = [c for extra in captured if extra for cc in extra for c in cc.converters] - assert any(isinstance(c, TextJailbreakConverter) for c in converters), ( - "Expected a TextJailbreakConverter to be threaded to factory.create" - ) + assert sum(isinstance(c, TextJailbreakConverter) for c in converters) == 1 # The objective seed groups themselves carry no jailbreak framing (converter delivery only). for attack in scenario._atomic_attacks: @@ -376,49 +368,35 @@ def _spy_create(self, **kwargs): "Jailbreak converter must be applied before caller-supplied converters" ) - async def test_simulated_conversation_techniques_produce_attacks_with_jailbreak( + async def test_stale_incompatible_technique_is_rejected( self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - """Regression: simulated-conversation techniques (``role_play_*``, ``crescendo_*``) must still - produce atomic attacks when crossed with a jailbreak template, and each must receive the - jailbreak converter. - - Converter delivery leaves the objective seed group unframed, so it stays compatible with the - simulated-conversation seed technique. (Delivering the jailbreak as a system-role framing seed - instead collided with that technique's seed range and silently produced zero attacks.) - """ - technique_class = _build_jailbreak_technique() - techniques = [ - technique_class("role_play_movie_script"), - technique_class("crescendo_simulated"), - ] - captured: list[Any] = [] - original_create = AttackTechniqueFactory.create - - def _spy_create(self, **kwargs): - captured.append(kwargs.get("extra_request_converters")) - return original_create(self, **kwargs) - + registry_factories = list(AttackTechniqueRegistry.get_registry_singleton().get_factories_or_raise().values()) + legacy_class = AttackTechniqueRegistry.build_technique_class_from_factories( + class_name="LegacyJailbreakTechnique", + factories=registry_factories, + ) with _patch_seed_groups(mock_memory_seed_groups): - with patch.object(AttackTechniqueFactory, "create", _spy_create): - scenario = Jailbreak(objective_scorer=mock_objective_scorer) - scenario.set_params_from_args( - args=_default_args( - mock_objective_target, scenario_techniques=techniques, jailbreak_names=["aim.yaml"] - ) + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args( + args=_default_args( + mock_objective_target, + scenario_techniques=[legacy_class("tap")], + jailbreak_names=["aim.yaml"], ) + ) + with pytest.raises(ValueError, match="stale or incompatible"): await scenario.initialize_async() - names = {a.atomic_attack_name for a in scenario._atomic_attacks} - assert "role_play_movie_script_aim_harmbench" in names - assert "crescendo_simulated_aim_harmbench" in names - # Every build of a simulated-conversation technique must still carry the jailbreak - # converter. Assert both techniques captured a non-empty converter stack (so the check - # can't pass vacuously on a dropped/None stack) and each contains the jailbreak converter. - populated = [extra for extra in captured if extra] - assert len(populated) == 2, "Expected both simulated-conversation techniques to receive converters" - assert all( - any(isinstance(c, TextJailbreakConverter) for cc in extra for c in cc.converters) for extra in populated - ), "Each simulated-conversation technique must receive the jailbreak converter" + + async def test_missing_runtime_factory_is_rejected( + self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups + ): + with _patch_seed_groups(mock_memory_seed_groups): + with patch("pyrit.scenario.scenarios.airt.jailbreak.resolve_technique_factories", return_value={}): + scenario = Jailbreak(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args(args=_default_args(mock_objective_target, jailbreak_names=["aim.yaml"])) + with pytest.raises(ValueError, match="no longer available.*prompt_sending"): + await scenario.initialize_async() async def test_all_templates_produce_attacks( self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups @@ -708,15 +686,17 @@ def test_default_techniques_are_the_two_deliveries(self): assert default_values == set(_DEFAULT_TECHNIQUES) assert default_values == {_PROMPT_SENDING, _JAILBREAK_SYSTEM_PROMPT} - def test_registry_techniques_are_available(self): + def test_only_scenario_delivery_techniques_are_available(self): technique_class = _technique_class() available = {t.value for t in technique_class.get_all_techniques()} - assert {_PROMPT_SENDING, _JAILBREAK_SYSTEM_PROMPT}.issubset(available) - # The "normal ones available like from rapid response" are exposed as opt-in techniques. - assert {"role_play_movie_script", "many_shot", "tap"}.issubset(available) + assert available == {_PROMPT_SENDING, _JAILBREAK_SYSTEM_PROMPT} + + def test_registry_metadata_lists_only_scenario_deliveries(self): + metadata = ScenarioRegistry()._build_metadata("airt.jailbreak", Jailbreak) + assert set(metadata.all_techniques) == {_PROMPT_SENDING, _JAILBREAK_SYSTEM_PROMPT} - def test_scenario_version_is_three(self): - assert Jailbreak.VERSION == 3 + def test_scenario_version_is_four(self): + assert Jailbreak.VERSION == 4 def test_default_dataset_is_harmbench(self): assert Jailbreak.required_datasets() == ["harmbench"]