From ef824da8efedee27f70167adfa5caa33836df800 Mon Sep 17 00:00:00 2001 From: emmanuelgjr Date: Fri, 9 Oct 2026 16:14:32 -0400 Subject: [PATCH] datasets: validators for rag and incident; point crossframework at the crosswalk - rag_dataset/validate.py and incident_dataset/validate.py check every entries/*.json against data_validation/schemas/{rag,incident}.schema.json (draft-07, with date formats checked), DSGAI IDs against the shared taxonomy, ID-matches-filename and duplicate IDs. An empty entries/ passes, so incident_dataset is green until its first entry. - RAG-0001.json moves to rag_dataset/entries/ and gains a $schema line, like the other datasets' entries; its content is unchanged. - rag and incident READMEs: the "TODO: define schema" placeholders now point at the schema, the entries/ layout and validate.py; incident entries are JSON only, since Markdown cannot be validated. - crossframework_mapping_dataset never held data and framework mappings now live in the GenAI Crosswalk repo, so its README and the datasets table point there instead of inviting contributions. Co-Authored-By: Claude Opus 5.5 --- datasets/README.md | 2 +- .../crossframework_mapping_dataset/README.md | 56 ++----------- datasets/incident_dataset/README.md | 11 ++- datasets/incident_dataset/validate.py | 83 +++++++++++++++++++ datasets/rag_dataset/README.md | 9 +- .../rag_dataset/{ => entries}/RAG-0001.json | 1 + datasets/rag_dataset/validate.py | 83 +++++++++++++++++++ 7 files changed, 188 insertions(+), 57 deletions(-) create mode 100644 datasets/incident_dataset/validate.py rename datasets/rag_dataset/{ => entries}/RAG-0001.json (97%) create mode 100644 datasets/rag_dataset/validate.py diff --git a/datasets/README.md b/datasets/README.md index 4b24509..c02e25f 100644 --- a/datasets/README.md +++ b/datasets/README.md @@ -8,7 +8,7 @@ Community-contributed datasets supporting research, benchmarking, and practical | `incident_dataset` | Anonymized GenAI data security incidents (DSGAI01–DSGAI21) | Accepting contributions | | `promptinj_dataextraction_testcases` | Adversarial prompts and extraction techniques for red-teaming | Accepting contributions | | `rag_dataset` | RAG poisoning and retrieval integrity test sets | Accepting contributions | -| `crossframework_mapping_dataset` | Machine-readable DSGAI-to-framework control mappings | Accepting contributions | +| `crossframework_mapping_dataset` | Machine-readable DSGAI-to-framework control mappings | Superseded by the [GenAI Crosswalk](https://github.com/GenAI-Security-Project/crosswalk) | | `agentdataflow_toolexchange_traces` | Sanitized agentic AI tool call and delegation chain traces | Accepting contributions | ## How to Contribute diff --git a/datasets/crossframework_mapping_dataset/README.md b/datasets/crossframework_mapping_dataset/README.md index eb47a00..4fb7a41 100644 --- a/datasets/crossframework_mapping_dataset/README.md +++ b/datasets/crossframework_mapping_dataset/README.md @@ -1,56 +1,10 @@ # Cross-Framework Control Mapping Dataset -Machine-readable mappings of DSGAI risks to controls across major cybersecurity and AI governance frameworks. +**Status:** Superseded by the [GenAI Crosswalk](https://github.com/GenAI-Security-Project/crosswalk). Do not add mappings here. -**Status:** Accepting contributions — this dataset is being built from scratch with the community. +Machine-readable mappings of DSGAI risks to the controls of security and AI governance frameworks now live in the dedicated crosswalk repository, which covers the frameworks this folder was planned for (NIST CSF 2.0, NIST AI RMF, MITRE ATLAS and ATT&CK, ISO/IEC 42001 and 27001, CIS Controls, the OWASP Top 10s, the EU AI Act, GDPR and more). -## Scope +- Browse the mappings: https://genai-security-project.github.io/crosswalk/ +- Propose a mapping: open an issue or pull request in https://github.com/GenAI-Security-Project/crosswalk -Structured, machine-readable mappings that connect each DSGAI entry (DSGAI01–DSGAI21) to the specific controls, requirements, and guidelines in recognized security and AI frameworks. Unlike the narrative mappings in the `/mappings` directory, this dataset is designed for **automated consumption** — enabling compliance gap analysis tools, GRC platforms, and audit automation. - -Target frameworks for mapping: - -- **NIST Cybersecurity Framework (CSF) 2.0** — Functions, categories, and subcategories -- **NIST AI RMF 1.0 (AI 100-1)** — Map, Measure, Manage, Govern functions -- **MITRE ATLAS** — Techniques and mitigations -- **MITRE ATT&CK** — Relevant enterprise techniques -- **ISO/IEC 42001** — AI management system controls -- **ISO/IEC 27001** — Annex A controls -- **CIS Controls v8** — Safeguards -- **OWASP Top 10 for LLM Applications 2025** — Entries and mitigations -- **OWASP Top 10 for Agentic Applications 2026** — Entries and mitigations -- **EU AI Act** — Relevant articles and requirements -- **GDPR** — Relevant articles -- **SOC 2** — Trust Services Criteria - -## Data Format - -Mappings should be provided in JSON or CSV with the following structure: - -```json -{ - "dsgai_id": "DSGAI01", - "dsgai_name": "Sensitive Data Leakage", - "framework": "NIST CSF 2.0", - "control_id": "PR.DS-01", - "control_name": "Data-at-rest is protected", - "relationship": "mitigates", - "notes": "Applies to training data, embeddings, and vector store contents at rest", - "mitigation_tier": "Tier 1" -} -``` - -Fields: - -- **dsgai_id** — DSGAI entry identifier (DSGAI01–DSGAI21) -- **dsgai_name** — DSGAI entry name -- **framework** — Target framework name and version -- **control_id** — The specific control, requirement, or article identifier in the target framework -- **control_name** — Human-readable name of the control -- **relationship** — `mitigates`, `detects`, `governs`, `partially_addresses`, or `related` -- **notes** — GenAI-specific context for how this control applies -- **mitigation_tier** — Which DSGAI mitigation tier (Tier 1 / Tier 2 / Tier 3) the control aligns with, if applicable - -## Contributing - -Add mappings as JSON or CSV files organized by framework (e.g., `nist_csf2_mapping.json`, `mitre_atlas_mapping.csv`). Submit a pull request. See the [main datasets README](../README.md) for general contribution guidelines. +This folder never held data. It is kept so existing links still resolve; [`data_validation/schemas/crossframework_mapping.schema.json`](../../data_validation/schemas/crossframework_mapping.schema.json) is kept for the same reason. diff --git a/datasets/incident_dataset/README.md b/datasets/incident_dataset/README.md index 00d51dc..4b66a7b 100644 --- a/datasets/incident_dataset/README.md +++ b/datasets/incident_dataset/README.md @@ -12,9 +12,14 @@ This dataset is modeled on existing incident databases (e.g., AIAAIC, AI Inciden ## Data Format - +Each entry is one JSON file in `entries/`, named after its `incident_id` (e.g. `entries/INC-0001.json`), that validates against [`data_validation/schemas/incident.schema.json`](../../data_validation/schemas/incident.schema.json). Start the file with `"$schema": "../../../data_validation/schemas/incident.schema.json"`. Check your entry with: -Contributions should include where possible: +```bash +cd datasets/incident_dataset +python validate.py +``` + +Required fields are `incident_id`, `date`, `summary`, `dsgai_mapping`, `attack_vector`, `data_types_affected` and `impact`. Contributions should include where possible: - **Incident ID** — Unique identifier (assigned on merge) - **Date** — When the incident occurred or was disclosed @@ -42,4 +47,4 @@ Self-reported incidents from your own organization are welcome and encouraged ## Contributing -Add entries as individual JSON or Markdown files and submit a pull request. See the [main datasets README](../README.md) for general contribution guidelines. +Add each incident as its own JSON file in `entries/` and submit a pull request. See the [main datasets README](../README.md) for general contribution guidelines. diff --git a/datasets/incident_dataset/validate.py b/datasets/incident_dataset/validate.py new file mode 100644 index 0000000..e5576ec --- /dev/null +++ b/datasets/incident_dataset/validate.py @@ -0,0 +1,83 @@ +"""Validate every entry in ./entries/ against the incident schema. + +Usage: python validate.py +Exit code 0 on success (including when there are no entries yet), 1 on any +validation failure, 2 when jsonschema is missing. +Requires: jsonschema (pip install jsonschema) + +The schema is data_validation/schemas/incident.schema.json, the only schema for +this dataset. Beyond the schema this checks that incident_id matches the +file name, that no two entries share an incident_id, and that every +dsgai_mapping value is in datasets/_shared/dsgai_taxonomy.json. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +try: + from jsonschema import Draft7Validator, FormatChecker +except ImportError: + sys.stderr.write("jsonschema not installed. Run: pip install jsonschema\n") + sys.exit(2) + +ROOT = Path(__file__).resolve().parent +SCHEMA_PATH = ROOT.parent.parent / "data_validation" / "schemas" / "incident.schema.json" +ENTRIES_DIR = ROOT / "entries" +TAXONOMY_PATH = ROOT.parent / "_shared" / "dsgai_taxonomy.json" +ID_FIELD = "incident_id" + + +def load_taxonomy_ids() -> set[str]: + data = json.loads(TAXONOMY_PATH.read_text(encoding="utf-8")) + return {e["id"] for e in data.get("entries", [])} + + +def main() -> int: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + validator = Draft7Validator(schema, format_checker=FormatChecker()) + taxonomy_ids = load_taxonomy_ids() + + entry_files = sorted(ENTRIES_DIR.glob("*.json")) if ENTRIES_DIR.is_dir() else [] + if not entry_files: + print(f"OK: no entries yet in {ENTRIES_DIR.name}/.") + return 0 + + errors: list[str] = [] + seen: dict[str, str] = {} + for path in entry_files: + try: + entry = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + errors.append(f"{path.name}: invalid JSON — {e}") + continue + + for err in validator.iter_errors(entry): + loc = "/".join(str(p) for p in err.absolute_path) or "" + errors.append(f"{path.name}: {loc}: {err.message}") + + for dsgai_id in entry.get("dsgai_mapping", []): + if dsgai_id not in taxonomy_ids: + errors.append(f"{path.name}: dsgai_mapping {dsgai_id!r} not in taxonomy") + + entry_id = entry.get(ID_FIELD) + if isinstance(entry_id, str): + if entry_id != path.stem: + errors.append(f"{path.name}: {ID_FIELD} {entry_id!r} does not match the file name") + if entry_id in seen: + errors.append(f"{path.name}: {ID_FIELD} {entry_id!r} is also used by {seen[entry_id]}") + seen.setdefault(entry_id, path.name) + + if errors: + for line in errors: + print(line) + print(f"\nFAIL: {len(errors)} issue(s) across {len(entry_files)} entries.") + return 1 + + print(f"OK: {len(entry_files)} entries validated against schema.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/datasets/rag_dataset/README.md b/datasets/rag_dataset/README.md index c87bffb..9305e72 100644 --- a/datasets/rag_dataset/README.md +++ b/datasets/rag_dataset/README.md @@ -18,7 +18,12 @@ Categories include: ## Data Format - +Each entry is one JSON file in [`entries/`](entries/), named after its `document_id` (e.g. `entries/RAG-0002.json`), that validates against [`data_validation/schemas/rag.schema.json`](../../data_validation/schemas/rag.schema.json). Start the file with `"$schema": "../../../data_validation/schemas/rag.schema.json"` and see [`entries/RAG-0001.json`](entries/RAG-0001.json) for a complete example. Check your entry with: + +```bash +cd datasets/rag_dataset +python validate.py +``` Contributions should include: @@ -37,4 +42,4 @@ All documents in this dataset must be **synthetic or publicly sourced**. Do not ## Contributing -Add documents and test sets as individual files or structured JSON/YAML and submit a pull request. See the [main datasets README](../README.md) for general contribution guidelines. +Add each document or test case as its own JSON file in `entries/` and submit a pull request. See the [main datasets README](../README.md) for general contribution guidelines. diff --git a/datasets/rag_dataset/RAG-0001.json b/datasets/rag_dataset/entries/RAG-0001.json similarity index 97% rename from datasets/rag_dataset/RAG-0001.json rename to datasets/rag_dataset/entries/RAG-0001.json index 09f7346..8fc4c91 100644 --- a/datasets/rag_dataset/RAG-0001.json +++ b/datasets/rag_dataset/entries/RAG-0001.json @@ -1,4 +1,5 @@ { + "$schema": "../../../data_validation/schemas/rag.schema.json", "document_id": "RAG-0001", "category": "poisoned_document", "dsgai_mapping": [ diff --git a/datasets/rag_dataset/validate.py b/datasets/rag_dataset/validate.py new file mode 100644 index 0000000..757c387 --- /dev/null +++ b/datasets/rag_dataset/validate.py @@ -0,0 +1,83 @@ +"""Validate every entry in ./entries/ against the RAG schema. + +Usage: python validate.py +Exit code 0 on success (including when there are no entries yet), 1 on any +validation failure, 2 when jsonschema is missing. +Requires: jsonschema (pip install jsonschema) + +The schema is data_validation/schemas/rag.schema.json, the only schema for +this dataset. Beyond the schema this checks that document_id matches the +file name, that no two entries share a document_id, and that every +dsgai_mapping value is in datasets/_shared/dsgai_taxonomy.json. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +try: + from jsonschema import Draft7Validator, FormatChecker +except ImportError: + sys.stderr.write("jsonschema not installed. Run: pip install jsonschema\n") + sys.exit(2) + +ROOT = Path(__file__).resolve().parent +SCHEMA_PATH = ROOT.parent.parent / "data_validation" / "schemas" / "rag.schema.json" +ENTRIES_DIR = ROOT / "entries" +TAXONOMY_PATH = ROOT.parent / "_shared" / "dsgai_taxonomy.json" +ID_FIELD = "document_id" + + +def load_taxonomy_ids() -> set[str]: + data = json.loads(TAXONOMY_PATH.read_text(encoding="utf-8")) + return {e["id"] for e in data.get("entries", [])} + + +def main() -> int: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + validator = Draft7Validator(schema, format_checker=FormatChecker()) + taxonomy_ids = load_taxonomy_ids() + + entry_files = sorted(ENTRIES_DIR.glob("*.json")) if ENTRIES_DIR.is_dir() else [] + if not entry_files: + print(f"OK: no entries yet in {ENTRIES_DIR.name}/.") + return 0 + + errors: list[str] = [] + seen: dict[str, str] = {} + for path in entry_files: + try: + entry = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + errors.append(f"{path.name}: invalid JSON — {e}") + continue + + for err in validator.iter_errors(entry): + loc = "/".join(str(p) for p in err.absolute_path) or "" + errors.append(f"{path.name}: {loc}: {err.message}") + + for dsgai_id in entry.get("dsgai_mapping", []): + if dsgai_id not in taxonomy_ids: + errors.append(f"{path.name}: dsgai_mapping {dsgai_id!r} not in taxonomy") + + entry_id = entry.get(ID_FIELD) + if isinstance(entry_id, str): + if entry_id != path.stem: + errors.append(f"{path.name}: {ID_FIELD} {entry_id!r} does not match the file name") + if entry_id in seen: + errors.append(f"{path.name}: {ID_FIELD} {entry_id!r} is also used by {seen[entry_id]}") + seen.setdefault(entry_id, path.name) + + if errors: + for line in errors: + print(line) + print(f"\nFAIL: {len(errors)} issue(s) across {len(entry_files)} entries.") + return 1 + + print(f"OK: {len(entry_files)} entries validated against schema.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())