FEAT: Backfill class-level metadata for all remote seed datasets

pyrit/datasets/seed_datasets/remote/aegis_ai_content_safety_dataset.py

@@ -9,7 +9,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -60,6 +60,11 @@ class _AegisContentSafetyDataset(_RemoteDatasetLoader):
         "Violence",
     ]
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "huge"  # 19093 annotated human-LLM interactions
+    tags: frozenset[str] = frozenset({"default", "safety"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/aya_redteaming_dataset.py

@@ -8,7 +8,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -39,6 +39,11 @@ class _AyaRedteamingDataset(_RemoteDatasetLoader):
         "Tagalog": "tgl",
     }
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "medium"  # 987 prompts across multiple languages
+    tags: frozenset[str] = frozenset({"safety", "multilingual"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/babelscape_alert_dataset.py

@@ -7,7 +7,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -23,6 +23,11 @@ class _BabelscapeAlertDataset(_RemoteDatasetLoader):
     Reference: [@tedeschi2024alert]
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "huge"  # 30968 prompts (default config)
+    tags: frozenset[str] = frozenset({"default", "safety", "jailbreak"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/beaver_tails_dataset.py

@@ -6,7 +6,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -31,6 +31,11 @@ class _BeaverTailsDataset(_RemoteDatasetLoader):
 
     HF_DATASET_NAME: str = "PKU-Alignment/BeaverTails"
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "huge"  # 166382 annotated prompt-response entries (default config)
+    tags: frozenset[str] = frozenset({"default", "safety"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/cbt_bench_dataset.py

@@ -7,7 +7,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -28,6 +28,11 @@ class _CBTBenchDataset(_RemoteDatasetLoader):
         - [@zhang2024cbtbench]
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "small"  # 20 core_fine_seed therapy seeds (default config)
+    tags: frozenset[str] = frozenset({"safety", "medical"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/ccp_sensitive_prompts_dataset.py

@@ -6,7 +6,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +21,11 @@ class _CCPSensitivePromptsDataset(_RemoteDatasetLoader):
     Reference: [@promptfoo2025ccp]
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "large"  # 1360 censorship-sensitive prompts (single-language Mandarin)
+    tags: frozenset[str] = frozenset({"safety", "multilingual"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/comic_jailbreak_dataset.py

@@ -12,7 +12,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import Seed, SeedDataset, SeedObjective, SeedPrompt
+from pyrit.models import Modality, Seed, SeedDataset, SeedObjective, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -92,8 +92,8 @@ class _ComicJailbreakDataset(_RemoteDatasetLoader):
         "sexual",
         "privacy",
     )
-    modalities: tuple[str, ...] = ("text", "image")
-    size: str = "large"  # 300 goals × 5 templates
+    modalities: tuple[Modality, ...] = (Modality.TEXT, Modality.IMAGE)
+    size: str = "large"  # 3501 image-text jailbreak prompts
     tags: frozenset[str] = frozenset({"safety", "multimodal"})
 
     def __init__(

pyrit/datasets/seed_datasets/remote/darkbench_dataset.py

@@ -4,7 +4,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 
 class _DarkBenchDataset(_RemoteDatasetLoader):
@@ -23,6 +23,11 @@ class _DarkBenchDataset(_RemoteDatasetLoader):
         - https://openreview.net/forum?id=odjMSBSWRt
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "large"  # 660 prompts across 6 dark-pattern categories
+    tags: frozenset[str] = frozenset({"default", "safety"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/equitymedqa_dataset.py

@@ -8,7 +8,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -24,6 +24,11 @@ class _EquityMedQADataset(_RemoteDatasetLoader):
     Reference: [@pfohl2024equitymedqa]
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "huge"  # 5565 prompts across 11 medical-bias subsets
+    tags: frozenset[str] = frozenset({"safety", "bias", "medical"})
+
     DATA_SUBSETS: list[str] = [
         "cc_llm",
         "cc_manual",

pyrit/datasets/seed_datasets/remote/forbidden_questions_dataset.py

@@ -6,7 +6,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -25,6 +25,11 @@ class _ForbiddenQuestionsDataset(_RemoteDatasetLoader):
     Website: https://jailbreak-llms.xinyueshen.me/
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "medium"  # 390 questions (13 scenarios x 30 questions)
+    tags: frozenset[str] = frozenset({"default", "safety", "jailbreak"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/harmbench_dataset.py

@@ -6,7 +6,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedObjective
+from pyrit.models import Modality, SeedDataset, SeedObjective
 
 
 class _HarmBenchDataset(_RemoteDatasetLoader):
@@ -21,8 +21,8 @@ class _HarmBenchDataset(_RemoteDatasetLoader):
 
     # Metadata
     harm_categories: list[str] = ["cybercrime", "illegal", "harmful", "chemical_biological", "harassment"]
-    modalities: list[str] = ["text"]
-    size: str = "large"  # 504 seeds
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "medium"  # 400 harmful behaviors
     tags: set[str] = {"default", "safety"}
 
     def __init__(

pyrit/datasets/seed_datasets/remote/harmbench_multimodal_dataset.py

@@ -12,7 +12,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -43,6 +43,11 @@ class _HarmBenchMultimodalDataset(_RemoteDatasetLoader):
     Paper: [@mazeika2024harmbench]
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT, Modality.IMAGE)
+    size: str = "medium"  # 220 harmful multimodal behaviors
+    tags: frozenset[str] = frozenset({"safety", "jailbreak", "multimodal"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/harmful_qa_dataset.py

@@ -6,7 +6,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -28,6 +28,11 @@ class _HarmfulQADataset(_RemoteDatasetLoader):
 
     HF_DATASET_NAME: str = "declare-lab/HarmfulQA"
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "large"  # 1960 harmful questions by academic topic
+    tags: frozenset[str] = frozenset({"default", "safety", "jailbreak"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/jbb_behaviors_dataset.py

@@ -6,7 +6,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -24,6 +24,11 @@ class _JBBBehaviorsDataset(_RemoteDatasetLoader):
     before using these prompts against production LLMs.
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "small"  # 100 harmful behaviors across 10 categories
+    tags: frozenset[str] = frozenset({"safety", "jailbreak"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/librai_do_not_answer_dataset.py

@@ -6,7 +6,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -22,6 +22,11 @@ class _LibrAIDoNotAnswerDataset(_RemoteDatasetLoader):
     GitHub: https://github.com/libr-ai/do-not-answer
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "large"  # 939 prompts across multiple risk areas
+    tags: frozenset[str] = frozenset({"default", "safety", "refusal"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/llm_latent_adversarial_training_dataset.py

@@ -6,7 +6,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +21,11 @@ class _LLMLatentAdversarialTrainingDataset(_RemoteDatasetLoader):
     Reference: [@sheshadri2024lat]
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "large"  # 4948 harmful prompts
+    tags: frozenset[str] = frozenset({"default", "safety", "jailbreak"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/medsafetybench_dataset.py

@@ -7,7 +7,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -23,6 +23,11 @@ class _MedSafetyBenchDataset(_RemoteDatasetLoader):
     Paper: [@han2024medsafetybench]
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "huge"  # 76174 medical-safety prompts (combined train+test+generated)
+    tags: frozenset[str] = frozenset({"safety", "medical"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/mlcommons_ailuminate_dataset.py

@@ -7,7 +7,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -23,6 +23,11 @@ class _MLCommonsAILuminateDataset(_RemoteDatasetLoader):
     Paper: [@vidgen2024ailuminate]
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "large"  # 1,200 hazard-category prompts
+    tags: frozenset[str] = frozenset({"default", "safety"})
+
     HAZARD_CATEGORIES_MAP = {
         "vcr": "Violent Crimes",
         "src": "Sex-Related Crimes",

pyrit/datasets/seed_datasets/remote/multilingual_vulnerability_dataset.py

@@ -6,7 +6,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -22,6 +22,11 @@ class _MultilingualVulnerabilityDataset(_RemoteDatasetLoader):
     GitHub: https://github.com/CarsonDon/Multilingual-Vuln-LLMs
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "small"  # 70 multilingual vulnerability prompts
+    tags: frozenset[str] = frozenset({"default", "safety", "multilingual"})
+
     def __init__(
         self,
         *,