datasets: Croissant 1.1 + RAI metadata, versioned signed releases, citation

.gitattributes

@@ -0,0 +1,5 @@
+# Dataset files are hashed into each dataset's croissant.json and into release
+# checksums, so their bytes must be the same on every platform: always LF.
+datasets/**/*.csv   text eol=lf
+datasets/**/*.json  text eol=lf
+data_validation/reference_data/*.csv  text eol=lf

.github/workflows/datasets-release.yml

@@ -0,0 +1,90 @@
+name: datasets-release
+
+# Cuts a versioned datasets release when a tag datasets-v<version> is pushed
+# (the version must equal CITATION.cff's). It re-runs every check, builds one
+# zip per dataset plus an all-datasets zip, writes SHA256SUMS, attests build
+# provenance for every file with Sigstore (verify with `gh attestation
+# verify <file> --repo GenAI-Security-Project/GenAI-Data-Security-Initiative`),
+# and publishes a GitHub release. With the repository enabled in Zenodo's
+# GitHub integration, Zenodo archives each release and mints a DOI.
+on:
+  push:
+    tags:
+      - 'datasets-v*'
+
+permissions:
+  contents: write
+  id-token: write
+  attestations: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.11'
+
+      - name: Install requirements
+        run: python -m pip install --quiet -r data_validation/requirements.txt -r data_validation/requirements-croissant.txt
+
+      - name: Tag matches CITATION.cff
+        id: version
+        run: |
+          tag_version="${GITHUB_REF_NAME#datasets-v}"
+          cff_version=$(sed -nE 's/^version:[[:space:]]*"?([^"[:space:]]+)"?.*/\1/p' CITATION.cff)
+          if [ "$tag_version" != "$cff_version" ]; then
+            echo "::error::tag $GITHUB_REF_NAME does not match CITATION.cff version $cff_version"
+            exit 1
+          fi
+          echo "version=$tag_version" >> "$GITHUB_OUTPUT"
+
+      - name: Every check passes
+        run: |
+          python data_validation/run_all_checks.py
+          python data_validation/qc_tools/consistency_check.py
+          python data_validation/croissant/build_croissant.py --check
+          python data_validation/croissant/check_croissant.py
+
+      - name: Build archives and checksums
+        env:
+          VERSION: ${{ steps.version.outputs.version }}
+        run: |
+          mkdir dist
+          for croissant in datasets/*/croissant.json; do
+            name=$(basename "$(dirname "$croissant")")
+            git archive --format=zip -o "dist/owasp-dsgai-${name}-${VERSION}.zip" HEAD \
+              "datasets/${name}" datasets/_shared CITATION.cff LICENSE
+          done
+          git archive --format=zip -o "dist/owasp-dsgai-datasets-${VERSION}.zip" HEAD \
+            datasets data_validation/reference_data data_validation/schemas CITATION.cff LICENSE
+          (cd dist && sha256sum ./*.zip > SHA256SUMS)
+          cat dist/SHA256SUMS
+
+      - name: Attest build provenance (Sigstore)
+        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
+        with:
+          subject-path: 'dist/*'
+
+      - name: Publish the GitHub release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          VERSION: ${{ steps.version.outputs.version }}
+        run: |
+          {
+            echo "Datasets release ${VERSION} of the OWASP GenAI Data Security Initiative."
+            echo
+            echo "Each dataset archive contains the dataset folder (with its MLCommons Croissant 1.1 metadata), the shared DSGAI taxonomy, CITATION.cff and the licence."
+            echo
+            echo "Verify a download:"
+            echo
+            echo '```'
+            echo "sha256sum -c SHA256SUMS --ignore-missing"
+            echo "gh attestation verify <file> --repo ${GITHUB_REPOSITORY}"
+            echo '```'
+          } > notes.md
+          gh release create "$GITHUB_REF_NAME" dist/* --title "Datasets ${VERSION}" --notes-file notes.md

.github/workflows/datasets-validate-all.yml

@@ -128,3 +128,25 @@ jobs:
             python data_validation/qc_tools/ner_pii_scan.py
             echo '```'
           } >> "$GITHUB_STEP_SUMMARY"
+
+  # MLCommons Croissant 1.1 metadata: up to date with the data (checksums,
+  # fields, dates), valid per mlcroissant, and loadable record by record.
+  croissant:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.11'
+
+      - name: Install requirements
+        run: python -m pip install --quiet -r data_validation/requirements-croissant.txt
+
+      - name: croissant.json files are up to date
+        run: python data_validation/croissant/build_croissant.py --check
+
+      - name: Validate and load every Croissant file
+        run: python data_validation/croissant/check_croissant.py

.zenodo.json

@@ -0,0 +1,11 @@
+{
+  "title": "OWASP GenAI Data Security Initiative: datasets",
+  "upload_type": "dataset",
+  "description": "Vulnerability, exploit-technique, risk-assessment, agent-trace, prompt-injection test-case and RAG fixture datasets for generative-AI data security, mapped to the OWASP GenAI Data Security (DSGAI) risk taxonomy, MITRE ATLAS, the OWASP Top 10 for LLM and Agentic Applications, NIST AI 100-2 and CWE. Each dataset ships MLCommons Croissant 1.1 metadata with Responsible AI fields.",
+  "creators": [{"name": "OWASP GenAI Data Security Initiative"}],
+  "license": "cc-by-sa-4.0",
+  "keywords": ["generative AI", "AI security", "data security", "OWASP", "DSGAI", "prompt injection", "LLM vulnerabilities"],
+  "related_identifiers": [
+    {"identifier": "https://github.com/GenAI-Security-Project/GenAI-Data-Security-Initiative", "relation": "isSupplementTo", "scheme": "url"}
+  ]
+}

CITATION.cff

@@ -0,0 +1,21 @@
+cff-version: 1.2.0
+title: "OWASP GenAI Data Security Initiative: datasets"
+message: "If you use these datasets, please cite them as below."
+type: dataset
+authors:
+  - name: "OWASP GenAI Data Security Initiative"
+repository-code: "https://github.com/GenAI-Security-Project/GenAI-Data-Security-Initiative"
+url: "https://genai.owasp.org/initiatives/ai-data-security-initiative/"
+license: CC-BY-SA-4.0
+# The datasets release version. A release is cut by tagging datasets-v<version>
+# (see .github/workflows/datasets-release.yml); the tag must match this value.
+version: "1.0.0"
+date-released: "2026-10-09"
+keywords:
+  - generative AI
+  - AI security
+  - data security
+  - OWASP
+  - DSGAI
+  - prompt injection
+  - LLM vulnerabilities

data_validation/README.md

@@ -76,6 +76,7 @@ After the automated checks pass, contributions are reviewed by cybersecurity and
 | `anomaly_detector.py` | A DSGAI entry most of a dataset maps to, records with far more mappings than usual, and one severity or category value dominating a dataset (datasets of 10+ records) |
 | `online_check.py` | Runs weekly (`datasets-online-checks` workflow), not on PRs, because it needs the network. **Fails** on a CVE the CVE Program has rejected or doesn't know, a withdrawn GHSA, a dead cited link (with the Wayback Machine snapshot to use instead), or a CVE in CISA KEV whose entry isn't marked `exploited_in_the_wild`. Reports every CVE's EPSS v4 score and KEV status. These change daily, so they're reported, not stored |
 | `ner_pii_scan.py` | Person names (and, in `incident_dataset`, organisation names) found by named-entity recognition (Microsoft Presidio + spaCy) in the datasets that require anonymization. Warnings only, in the CI job summary; reviewed false positives go in `reference_data/ner_allowlist.txt`. Optional dependencies: `requirements-ner.txt` |
+| `croissant/build_croissant.py`, `croissant/check_croissant.py` | Generate each dataset's MLCommons Croissant 1.1 metadata (with RAI fields) and check it in CI: up to date with the data, valid per `mlcroissant`, and loadable record by record |
 | `consistency_check.py` | **Fails** on internal references (`DSGAI-VULN-…`, `DSGAI-EXP-…`, `DSGAI-RA-…`) that point at no existing entry; notes CVEs cited by exploits that have no vulnerability entry |
 
 ---
@@ -117,6 +118,7 @@ data_validation/
 ├── run_all_checks.py        One command for everything (CI runs this)
 ├── validators/              Shared checks; each also runs on its own
 ├── qc_tools/                Reports for reviewers
+├── croissant/               Croissant 1.1 metadata generator and checker
 ├── schemas/                 Schemas for datasets without their own schema.json
 │                            (the rest are pointers to the dataset-local schemas)
 ├── reference_data/          DSGAI, MITRE ATLAS, CWE, OWASP Top 10, NIST AI 100-2, SPDX and