Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions .github/instructions/scenarios.instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,11 +145,59 @@ get atomic-attack construction **for free** — no override needed.

The default implementation:
1. Calls `self._get_attack_technique_factories()` to get name→factory mapping
(defaults to reading every `AttackTechniqueFactory` registered in the
`AttackTechniqueRegistry` singleton)
2. Iterates over every (technique × dataset) pair from `self._dataset_config`
3. Calls `factory.create()` with `objective_target` and conditional scorer override
4. Uses `self._build_display_group()` for user-facing grouping
5. Builds `AtomicAttack` with unique `atomic_attack_name` = `"{technique}_{dataset}"`

### AttackTechniqueFactory

Techniques are described by `AttackTechniqueFactory` instances rather than a separate spec
dataclass. The canonical catalog lives in
`pyrit.setup.initializers.components.scenario_techniques` (`build_scenario_technique_factories()`)
and is loaded into the registry by `ScenarioTechniqueInitializer`.

```python
from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory

AttackTechniqueFactory(
name="prompt_sending", # REQUIRED — must match the strategy enum value
attack_class=PromptSendingAttack,
strategy_tags=["core", "single_turn", "default"],
attack_kwargs={"max_turns": 5},
adversarial_chat=None, # mutually exclusive with adversarial_config
adversarial_config=None,
seed_technique=None,
uses_adversarial=None, # None = auto-derive from attack signature/seeds
scorer_override_policy=ScorerOverridePolicy.WARN,
)
```

Key points:
- `name` is required and must match the strategy enum value the scenario looks up.
- `strategy_tags` on the factory drives `TagQuery` filters used by
`AttackTechniqueRegistry.build_strategy_class_from_factories(...)`. This is **distinct**
from the per-entry `tags` argument passed to `registry.register_technique(...)`.
- `uses_adversarial` is auto-derived from the attack class signature (presence of
`attack_adversarial_config`) and seed shape; pass `False` explicitly to opt out, or
`True` to force opt-in.
- `kwargs` are validated against the attack class constructor signature at
factory-construction time, so typos fail loudly and early.

### Registering factories

```python
registry = AttackTechniqueRegistry.get_registry_singleton()
registry.register_from_factories(build_scenario_technique_factories())
```

`register_from_factories` reads `factory.strategy_tags` to populate the per-entry tags used
by the registry. Tests that exercise scenarios should reset both `AttackTechniqueRegistry`
and `TargetRegistry` and re-register a mock `adversarial_chat` so the catalog builder
resolves without falling back to `OpenAIChatTarget`.

### Customization hooks (no need to override `_get_atomic_attacks_async`):
- **`_get_attack_technique_factories()`** — override to add/remove/replace factories
- **`_build_display_group()`** — override to change grouping (default: by technique)
Expand Down
12 changes: 6 additions & 6 deletions doc/scanner/airt.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,12 @@
"pyrit_scan airt.rapid_response \\\n",
" --initializers target load_default_datasets \\\n",
" --target openai_chat \\\n",
" --strategies prompt_sending \\\n",
" --strategies role_play \\\n",
" --dataset-names airt_hate \\ \n",
" --max-dataset-size 1\n",
"```\n",
"\n",
"**Available strategies:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, prompt_sending, role_play, many_shot, tap"
"**Available strategies:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, role_play, many_shot, tap"
]
},
{
Expand Down Expand Up @@ -111,7 +111,7 @@
"scenario = RapidResponse()\n",
"await scenario.initialize_async( # type: ignore\n",
" objective_target=objective_target,\n",
" scenario_strategies=[RapidResponseStrategy.prompt_sending],\n",
" scenario_strategies=[RapidResponseStrategy.role_play],\n",
" dataset_config=dataset_config,\n",
")\n",
"\n",
Expand Down Expand Up @@ -369,11 +369,11 @@
"pyrit_scan airt.cyber \\\n",
" --initializers target load_default_datasets \\\n",
" --target openai_chat \\\n",
" --strategies single_turn \\\n",
" --strategies multi_turn \\\n",
" --max-dataset-size 1\n",
"```\n",
"\n",
"**Available strategies:** ALL, SINGLE_TURN, MULTI_TURN"
"**Available strategies:** ALL, MULTI_TURN, red_teaming"
]
},
{
Expand Down Expand Up @@ -405,7 +405,7 @@
"scenario = Cyber()\n",
"await scenario.initialize_async( # type: ignore\n",
" objective_target=objective_target,\n",
" scenario_strategies=[CyberStrategy.SINGLE_TURN],\n",
" scenario_strategies=[CyberStrategy.MULTI_TURN],\n",
" dataset_config=dataset_config,\n",
")\n",
"\n",
Expand Down
12 changes: 6 additions & 6 deletions doc/scanner/airt.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,12 @@
# pyrit_scan airt.rapid_response \
# --initializers target load_default_datasets \
# --target openai_chat \
# --strategies prompt_sending \
# --strategies role_play \
# --dataset-names airt_hate \
# --max-dataset-size 1
# ```
#
# **Available strategies:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, prompt_sending, role_play, many_shot, tap
# **Available strategies:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, role_play, many_shot, tap

# %%
from pyrit.scenario.scenarios.airt import RapidResponse, RapidResponseStrategy
Expand All @@ -57,7 +57,7 @@
scenario = RapidResponse()
await scenario.initialize_async( # type: ignore
objective_target=objective_target,
scenario_strategies=[RapidResponseStrategy.prompt_sending],
scenario_strategies=[RapidResponseStrategy.role_play],
dataset_config=dataset_config,
)

Expand Down Expand Up @@ -125,11 +125,11 @@
# pyrit_scan airt.cyber \
# --initializers target load_default_datasets \
# --target openai_chat \
# --strategies single_turn \
# --strategies multi_turn \
# --max-dataset-size 1
# ```
#
# **Available strategies:** ALL, SINGLE_TURN, MULTI_TURN
# **Available strategies:** ALL, MULTI_TURN, red_teaming

# %%
from pyrit.scenario.scenarios.airt import Cyber, CyberStrategy
Expand All @@ -139,7 +139,7 @@
scenario = Cyber()
await scenario.initialize_async( # type: ignore
objective_target=objective_target,
scenario_strategies=[CyberStrategy.SINGLE_TURN],
scenario_strategies=[CyberStrategy.MULTI_TURN],
dataset_config=dataset_config,
)

Expand Down
2 changes: 1 addition & 1 deletion pyrit/cli/pyrit_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@

# Run rapid response with specific datasets and concurrency
pyrit_scan airt.rapid_response --target openai_chat
--strategies prompt_sending --dataset-names airt_hate
--strategies role_play --dataset-names airt_hate
--max-dataset-size 5 --max-concurrency 4

# Run multi-turn red team agent with labels for tracking
Expand Down
2 changes: 0 additions & 2 deletions pyrit/registry/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
)
from pyrit.registry.object_registries import (
AttackTechniqueRegistry,
AttackTechniqueSpec,
BaseInstanceRegistry,
ConverterRegistry,
RegistryEntry,
Expand Down Expand Up @@ -49,6 +48,5 @@
"ScenarioRegistry",
"ScorerRegistry",
"TargetRegistry",
"AttackTechniqueSpec",
"TagQuery",
]
2 changes: 0 additions & 2 deletions pyrit/registry/object_registries/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

from pyrit.registry.object_registries.attack_technique_registry import (
AttackTechniqueRegistry,
AttackTechniqueSpec,
)
from pyrit.registry.object_registries.base_instance_registry import (
BaseInstanceRegistry,
Expand Down Expand Up @@ -42,5 +41,4 @@
"ConverterRegistry",
"ScorerRegistry",
"TargetRegistry",
"AttackTechniqueSpec",
]
Loading
Loading