From d146aa188e2bcebe920fd1e4c2b0f534312e7ca4 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Sun, 16 Aug 2026 01:35:26 -0700 Subject: [PATCH] FIX Honor attack score feedback configuration Handle zero-vector cosine similarity deterministically and migrate deprecated shell examples to scenario-results. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- doc/scanner/2_pyrit_shell.md | 33 ++++++++----------- pyrit/analytics/conversation_analytics.py | 2 ++ pyrit/executor/attack/multi_turn/crescendo.py | 21 ++++++------ .../attack/multi_turn/tree_of_attacks.py | 10 +++++- .../analytics/test_conversation_analytics.py | 15 ++++++++- .../attack/multi_turn/test_crescendo.py | 23 +++++++++++++ .../attack/multi_turn/test_tree_of_attacks.py | 23 +++++++++++++ 7 files changed, 96 insertions(+), 31 deletions(-) diff --git a/doc/scanner/2_pyrit_shell.md b/doc/scanner/2_pyrit_shell.md index 7a098fbae3..2ae15d5583 100644 --- a/doc/scanner/2_pyrit_shell.md +++ b/doc/scanner/2_pyrit_shell.md @@ -42,8 +42,8 @@ Once starting the shell, you will see the list of commands you have access to. S | `list-targets` | List all available targets from the registry | | `list-converters` | List all registered converter instances | | `run [options]` | Run a scenario with optional parameters | -| `scenario-history` | List all previous scenario runs in this session | -| `print-scenario [N]` | Print detailed results for scenario run(s) | +| `scenario-history [N]` | List recent scenario runs and their IDs | +| `scenario-results [options]` | Inspect overview or attack-level results for a scenario run | | `help [command]` | Show help for a command | | `clear` | Clear the screen | | `exit` (or `quit`, `q`) | Exit the shell | @@ -118,14 +118,11 @@ Track and review all scenario runs in your session: # Show all runs from this session pyrit> scenario-history -# Print details of the most recent run -pyrit> print-scenario +# Print an overview for a run using its scenario result ID +pyrit> scenario-results 2a1f91a0-28bf-4f48-bd54-8f12451cf7af -# Print details of a specific run (by number from history) -pyrit> print-scenario 1 - -# Print all runs -pyrit> print-scenario +# Inspect attack-level results for that run +pyrit> scenario-results 2a1f91a0-28bf-4f48-bd54-8f12451cf7af --view attacks ``` Example output: @@ -135,14 +132,12 @@ pyrit> scenario-history Scenario Run History: ================================================================================ -1) foundry.red_team_agent --initializers target --techniques base64 -2) garak.encoding --initializers target --techniques rot13 -3) foundry.red_team_agent --initializers target -t jailbreak + 1) [COMPLETED] foundry.red_team_agent (id: 2a1f91a0-28bf-4f48-bd54-8f12451cf7af) — 12 attacks, 33.3% success — 2026-08-16T01:15:00+00:00 + 2) [COMPLETED] garak.encoding (id: 9b27f101-1440-4454-9609-b307230f36a9) — 8 attacks, 25.0% success — 2026-08-16T01:10:00+00:00 + 3) [COMPLETED] foundry.red_team_agent (id: 40624391-c9d3-492d-b3e4-2c15202ade62) — 12 attacks, 16.7% success — 2026-08-16T01:05:00+00:00 ================================================================================ Total runs: 3 - -Use 'print-scenario ' to view detailed results for a specific run. ``` ## Interactive Exploration @@ -161,8 +156,8 @@ pyrit> run garak.encoding --techniques morse_code # Review and compare pyrit> scenario-history -pyrit> print-scenario 1 -pyrit> print-scenario 2 +pyrit> scenario-results 2a1f91a0-28bf-4f48-bd54-8f12451cf7af +pyrit> scenario-results 9b27f101-1440-4454-9609-b307230f36a9 ``` ## Shell Benefits @@ -190,10 +185,10 @@ pyrit> print-scenario 2 pyrit> scenario-history ``` -4. **Print specific results** to compare outcomes: +4. **Inspect specific results** to compare outcomes: ```bash - pyrit> print-scenario 1 # baseline run - pyrit> print-scenario 3 # modified run + pyrit> scenario-results 2a1f91a0-28bf-4f48-bd54-8f12451cf7af # baseline run + pyrit> scenario-results 40624391-c9d3-492d-b3e4-2c15202ade62 # modified run ``` ## Exit the Shell diff --git a/pyrit/analytics/conversation_analytics.py b/pyrit/analytics/conversation_analytics.py index 12e2d8b63a..b85d7af8e9 100644 --- a/pyrit/analytics/conversation_analytics.py +++ b/pyrit/analytics/conversation_analytics.py @@ -106,5 +106,7 @@ def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float: dot_product = np.dot(a, b) norms = np.linalg.norm(a) * np.linalg.norm(b) + if norms == 0: + return 0.0 return float(dot_product / norms) diff --git a/pyrit/executor/attack/multi_turn/crescendo.py b/pyrit/executor/attack/multi_turn/crescendo.py index 584065aa87..fe5b896cfc 100644 --- a/pyrit/executor/attack/multi_turn/crescendo.py +++ b/pyrit/executor/attack/multi_turn/crescendo.py @@ -565,20 +565,21 @@ def _build_adversarial_prompt( f"{refused_text}\n", ] ) - elif context.last_score and context.last_response: + elif context.last_response: # Get the last assistant response directly from the response object last_message_value = context.last_response.get_value() if last_message_value: - score_value = normalize_score_to_float(context.last_score) - prompt_parts.extend( - [ - f"\nThe target responded to the last question with: {last_message_value}", - "\n--------------------------------", - f"\nThis response received a score of: {score_value:.2f} on a scale of 0.0 to 1.0", - f"Rationale: {context.last_score.score_rationale}\n", - ] - ) + prompt_parts.append(f"\nThe target responded to the last question with: {last_message_value}") + if self._use_score_as_feedback and context.last_score: + score_value = normalize_score_to_float(context.last_score) + prompt_parts.extend( + [ + "\n--------------------------------", + f"\nThis response received a score of: {score_value:.2f} on a scale of 0.0 to 1.0", + f"Rationale: {context.last_score.score_rationale}\n", + ] + ) return " ".join(prompt_parts) diff --git a/pyrit/executor/attack/multi_turn/tree_of_attacks.py b/pyrit/executor/attack/multi_turn/tree_of_attacks.py index 38b44db43b..8e2f12aeb5 100644 --- a/pyrit/executor/attack/multi_turn/tree_of_attacks.py +++ b/pyrit/executor/attack/multi_turn/tree_of_attacks.py @@ -338,6 +338,7 @@ def __init__( attack_id: ComponentIdentifier, attack_strategy_name: str, modality_router: _ModalityFeedbackRouter, + use_score_as_feedback: bool = True, memory_labels: dict[str, str] | None = None, parent_id: str | None = None, prompt_normalizer: PromptNormalizer | None = None, @@ -364,6 +365,8 @@ def __init__( whether prior media should travel back to the adversarial chat or forward to the objective target, and fills adversarial-placeholder pieces in seed messages. Typically shared across all nodes of the same attack. + use_score_as_feedback (bool): Whether subsequent adversarial prompts include + the objective score. Defaults to True. memory_labels (dict[str, str] | None): Labels for memory storage. parent_id (str | None): ID of the parent node, if this is a child node prompt_normalizer (PromptNormalizer | None): Normalizer for handling prompts and responses. @@ -386,6 +389,7 @@ def __init__( self._attack_strategy_name = attack_strategy_name self._memory_labels = memory_labels or {} self._modality_router = modality_router + self._use_score_as_feedback = use_score_as_feedback # Initialize utilities self._memory = CentralMemory.get_memory_instance() @@ -876,6 +880,7 @@ def duplicate(self) -> _TreeOfAttacksNode: attack_id=self._attack_id, attack_strategy_name=self._attack_strategy_name, modality_router=self._modality_router, + use_score_as_feedback=self._use_score_as_feedback, memory_labels=self._memory_labels, desired_response_prefix=self._desired_response_prefix, parent_id=self.node_id, @@ -1170,7 +1175,9 @@ async def _generate_subsequent_turn_prompt_async(self, objective: str) -> str: logger.debug(f"Node {self.node_id}: Using response {target_response_piece.id} for next prompt") # Get score for the response - score = await self._get_response_score_async(str(target_response_piece.id)) + score = ( + await self._get_response_score_async(str(target_response_piece.id)) if self._use_score_as_feedback else "" + ) # Generate prompt using template return self._adversarial_chat_prompt_template.render_template_value( @@ -2090,6 +2097,7 @@ def _create_attack_node( attack_id=self.get_identifier(), attack_strategy_name=self.__class__.__name__, modality_router=self._modality_router, + use_score_as_feedback=self._attack_scoring_config.use_score_as_feedback, memory_labels=context.memory_labels, desired_response_prefix=self._configuration.desired_response_prefix, parent_id=parent_id, diff --git a/tests/unit/analytics/test_conversation_analytics.py b/tests/unit/analytics/test_conversation_analytics.py index f31fcc4876..15efcec1cf 100644 --- a/tests/unit/analytics/test_conversation_analytics.py +++ b/tests/unit/analytics/test_conversation_analytics.py @@ -4,9 +4,10 @@ from collections.abc import Sequence from unittest.mock import MagicMock +import numpy as np import pytest -from pyrit.analytics.conversation_analytics import ConversationAnalytics +from pyrit.analytics.conversation_analytics import ConversationAnalytics, cosine_similarity from pyrit.memory.memory_interface import MemoryInterface from pyrit.memory.memory_models import EmbeddingDataEntry from pyrit.models import MessagePiece, flatten_to_message_pieces @@ -70,3 +71,15 @@ def test_get_similar_chat_messages_by_embedding(mock_memory_interface, sample_me assert len(similar_messages) == 1 assert similar_messages[0].score >= 0.99 assert similar_messages[0].metric == "cosine_similarity" + + +@pytest.mark.parametrize( + "a,b", + [ + (np.array([0.0, 0.0]), np.array([1.0, 2.0])), + (np.array([1.0, 2.0]), np.array([0.0, 0.0])), + (np.array([0.0, 0.0]), np.array([0.0, 0.0])), + ], +) +def test_cosine_similarity_zero_vector_returns_zero(a: np.ndarray, b: np.ndarray) -> None: + assert cosine_similarity(a, b) == 0.0 diff --git a/tests/unit/executor/attack/multi_turn/test_crescendo.py b/tests/unit/executor/attack/multi_turn/test_crescendo.py index e962dddf41..98a1060ad7 100644 --- a/tests/unit/executor/attack/multi_turn/test_crescendo.py +++ b/tests/unit/executor/attack/multi_turn/test_crescendo.py @@ -923,6 +923,29 @@ async def test_build_adversarial_prompt_with_objective_score( assert "0.30" in result # Score value assert failure_objective_score.score_rationale in result + def test_build_adversarial_prompt_without_score_feedback( + self, + mock_objective_target: MagicMock, + mock_adversarial_chat: MagicMock, + basic_context: CrescendoAttackContext, + sample_response: Message, + failure_objective_score: Score, + ): + """The response remains available when objective-score feedback is disabled.""" + attack = CrescendoAttack( + objective_target=mock_objective_target, + attack_adversarial_config=AttackAdversarialConfig(target=mock_adversarial_chat), + attack_scoring_config=AttackScoringConfig(use_score_as_feedback=False), + ) + basic_context.last_response = sample_response + basic_context.last_score = failure_objective_score + + result = attack._build_adversarial_prompt(context=basic_context, refused_text="") + + assert "Test response" in result + assert "received a score of" not in result + assert failure_objective_score.score_rationale not in result + async def test_generate_next_prompt_raises_when_adversarial_chat_returns_no_response( self, mock_objective_target: MagicMock, diff --git a/tests/unit/executor/attack/multi_turn/test_tree_of_attacks.py b/tests/unit/executor/attack/multi_turn/test_tree_of_attacks.py index 239f7c444c..1c25f74edc 100644 --- a/tests/unit/executor/attack/multi_turn/test_tree_of_attacks.py +++ b/tests/unit/executor/attack/multi_turn/test_tree_of_attacks.py @@ -1581,6 +1581,29 @@ def test_node_initialization(self, node_components): assert node.auxiliary_scores == {} assert node.error_message is None + async def test_subsequent_prompt_omits_score_when_feedback_disabled(self, node_components): + """A disabled score-feedback setting preserves response context without exposing the score.""" + node = _TreeOfAttacksNode(**node_components, use_score_as_feedback=False) + response = MagicMock() + response.get_piece.return_value = MessagePiece( + role="assistant", + original_value="target response", + converted_value="target response", + conversation_id=node.objective_target_conversation_id, + ) + + with ( + patch.object(node._memory, "get_conversation_messages", return_value=[response]), + patch.object(node, "_get_response_score_async", new_callable=AsyncMock) as get_score, + ): + result = await node._generate_subsequent_turn_prompt_async("test objective") + + assert result == "rendered template" + get_score.assert_not_awaited() + render_kwargs = node_components["adversarial_chat_prompt_template"].render_template_value.call_args.kwargs + assert render_kwargs["target_response"] == "target response" + assert render_kwargs["score"] == "" + def test_node_duplicate_creates_child(self, node_components): """Test that duplicate() creates a proper child node.""" parent_node = _TreeOfAttacksNode(**node_components)