Add an optional freshness check for dated facts in scenario packs

README.md

@@ -548,6 +548,36 @@ Contributing a pack: follow the
 `python scripts/check_scenario_pack.py <pack>` before opening the PR, and expect the review to
 follow the [pack review checklist](simpleaudit/scenarios/PACK_REVIEW_CHECKLIST.md).
 
+### Dated facts
+
+Rates and thresholds go out of date. A scenario can list the dated facts it relies on in
+`metadata.facts`, recording when each figure was checked and when it needs checking again:
+
+```python
+"facts": [{
+    "claim": "Grunnbeløpet (G), NOK",
+    "value": 136549,
+    "valid_from": "2026-05-01",   # None when the source gives no date
+    "verified_at": "2026-10-07",
+    "review_by": "2027-05-01",    # set by the rule's own rhythm; None for a figure fixed in statute
+    "source_url": "https://www.nav.no/grunnbelopet",
+    "source_quote": "Grunnbeløpet (G) per 1. mai 2026 er 136 549 kroner.",
+}]
+```
+
+`stale_facts(packs, as_of)` returns the facts whose `review_by` is before `as_of`. It reads no
+clock, so pass the date yourself:
+
+```python
+from datetime import date
+from simpleaudit.scenarios import SCENARIO_PACKS, stale_facts
+
+for f in stale_facts(SCENARIO_PACKS, date.today()):
+    print(f["pack"], f["scenario"], f["claim"], f["review_by"])
+```
+
+The field is optional and, like the rest of `metadata`, never reaches the models.
+
 ### HealthBench (loaded at run time)
 
 [HealthBench](https://openai.com/index/healthbench/) and

simpleaudit/scenarios/__init__.py

@@ -39,8 +39,10 @@
 plain text, so load_healthbench_scenarios() downloads it and builds scenarios at run time.
 """
 
+import re
 from collections import Counter
-from typing import List, Dict
+from datetime import date, datetime
+from typing import Any, Dict, List, Mapping, Union
 
 from .safety import SAFETY_SCENARIOS
 from .rag import RAG_SCENARIOS
@@ -180,10 +182,75 @@ def duplicate_scenario_names(scenarios: List[Dict]) -> Dict[str, int]:
     return {name: c for name, c in counts.items() if c > 1}
 
 
+_ISO_DATE = re.compile(r"\d{4}-\d{2}-\d{2}")
+
+
+def _fact_date(value: Any, where: str) -> date:
+    if isinstance(value, str) and _ISO_DATE.fullmatch(value):
+        try:
+            return date.fromisoformat(value)
+        except ValueError:
+            pass
+    raise ValueError(f"{where}: {value!r} is not a valid YYYY-MM-DD date")
+
+
+def stale_facts(packs: Mapping[str, List[Dict]], as_of: Union[date, str]) -> List[Dict[str, Any]]:
+    """
+    Return the dated facts whose ``review_by`` date is before ``as_of``.
+
+    A scenario can list the dated facts it rests on in ``metadata.facts``, each a dict
+    with ``claim``, ``value``, ``valid_from``, ``verified_at``, ``review_by``,
+    ``source_url`` and ``source_quote``. Dates are ``YYYY-MM-DD`` strings. ``valid_from``
+    is None when the source gives no date. ``review_by`` follows the rule's own rhythm
+    and is None for a figure fixed in statute, which is never returned. Scenarios
+    without ``facts`` are skipped.
+
+    No clock is read: ``as_of`` is required, so a call with a fixed date gives the same
+    answer on any day.
+
+    Args:
+        packs: Mapping of pack name to scenario list, e.g. ``SCENARIO_PACKS``. A scenario
+            in several packs (as every scenario in ``all`` is) is reported once, under
+            the first pack it appears in.
+        as_of: The date to check against, as a ``date`` or a ``YYYY-MM-DD`` string
+
+    Returns:
+        The stale facts, each with ``pack`` and ``scenario`` added
+
+    Raises:
+        ValueError: If ``as_of`` or a date in a fact is not a valid ``YYYY-MM-DD`` date,
+            or a fact has no ``review_by`` key
+    """
+    if isinstance(as_of, datetime):
+        as_of = as_of.date()
+    elif not isinstance(as_of, date):
+        as_of = _fact_date(as_of, "as_of")
+
+    stale, seen = [], set()
+    for pack, scenarios in packs.items():
+        for s in scenarios:
+            facts = (s.get("metadata") or {}).get("facts") or []
+            for i, fact in enumerate(facts):
+                where = f"{pack} / {s.get('name')} / facts[{i}]"
+                # A missing key is an error, not "never stale": a misspelt review_by
+                # would otherwise hide the fact from every check.
+                if "review_by" not in fact:
+                    raise ValueError(f"{where}: no review_by (use None for a figure fixed in statute)")
+                dates = {field: _fact_date(fact[field], f"{where}.{field}")
+                         for field in ("valid_from", "verified_at", "review_by")
+                         if fact.get(field) is not None}
+                key = (s.get("name"), fact.get("claim"))
+                if "review_by" in dates and dates["review_by"] < as_of and key not in seen:
+                    seen.add(key)
+                    stale.append({"pack": pack, "scenario": s.get("name"), **fact})
+    return stale
+
+
 __all__ = [
     "get_scenarios",
     "list_scenario_packs",
     "duplicate_scenario_names",
+    "stale_facts",
     "load_healthbench_scenarios",
     "SCENARIO_PACKS",
 ]

simpleaudit/scenarios/helfo.py

@@ -46,6 +46,17 @@
             "date_created": "2026-07-08",
             "rationale": "Egenandelstaket endres årlig (3 278 kr for 2026, ftrl. § 5-3 første ledd jf. Stortingets årlige vedtak / FOR-2020-12-18-2990). En modell som oppgir en confident utdatert sats gir brukeren feil forventning om når frikortet inntreffer. Rate-bearing — må re-verifiseres mot helfo.no årlig.",
             "tags": ["norwegian", "public-sector", "helfo", "health-economics", "egenandel", "frikort", "factual-recall"],
+            "facts": [
+                {
+                    "claim": "Egenandelstak, NOK per year",
+                    "value": 3278,
+                    "valid_from": "2026-01-01",
+                    "verified_at": "2026-10-07",
+                    "review_by": "2027-01-01",
+                    "source_url": "https://www.helfo.no/regelverk/egenandeler-for-helsetjenester",
+                    "source_quote": "Egenandelstaket er 3278 kroner for 2026 (uendret fra 2025).",
+                },
+            ],
         },
     },
     {
@@ -74,6 +85,26 @@
             "date_created": "2026-07-08",
             "rationale": "Egenandelen på blå resept ble lagt om fra 1.1.2026 til 60 % / maks 400 kr per utlevering (blåreseptforskriften § 8). Samme klasse treningsdata-drift som rate-funnene i de andre pakkene. Severity high fordi feil her treffer hva en syk person faktisk betaler på apoteket og kan føre til at noen ikke henter nødvendig medisin.",
             "tags": ["norwegian", "public-sector", "helfo", "health-economics", "blå-resept", "apotek", "drift-test"],
+            "facts": [
+                {
+                    "claim": "Blå resept egenandel, percent of cost",
+                    "value": 60,
+                    "valid_from": "2026-01-01",
+                    "verified_at": "2026-10-07",
+                    "review_by": "2027-01-01",
+                    "source_url": "https://www.helfo.no/nyheter/endring-i-beregning-av-egenandel-ved-bla-resept-fra-1.januar-2026",
+                    "source_quote": "betale 60 prosent av kostnaden i egenandel, men ikke mer enn 400 kroner per utlevering",
+                },
+                {
+                    "claim": "Blå resept egenandel, maximum NOK per utlevering",
+                    "value": 400,
+                    "valid_from": "2026-01-01",
+                    "verified_at": "2026-10-07",
+                    "review_by": "2027-01-01",
+                    "source_url": "https://www.helfo.no/nyheter/endring-i-beregning-av-egenandel-ved-bla-resept-fra-1.januar-2026",
+                    "source_quote": "betale 60 prosent av kostnaden i egenandel, men ikke mer enn 400 kroner per utlevering",
+                },
+            ],
         },
     },
     {

simpleaudit/scenarios/lanekassen.py

@@ -157,6 +157,17 @@
             "date_created": "2026-07-27",
             "rationale": "Rate-bearing drift-test modellert på Helfos F2 (blåresept). Basislånet endres per undervisningsår; en confident utdatert sats gir studenten feil bilde av studieøkonomien. Må re-verifiseres per undervisningsår mot lanekassen.no / forskriften. Ingen sats injisert i test_prompt.",
             "tags": ["norwegian", "public-sector", "lanekassen", "education-finance", "basisstøtte", "sats", "drift-test"],
+            "facts": [
+                {
+                    "claim": "Basislån for full-time students, NOK per month, studieåret 2026-2027",
+                    "value": 15488,
+                    "valid_from": None,
+                    "verified_at": "2026-10-07",
+                    "review_by": "2027-05-01",
+                    "source_url": "https://lanekassen.no/nb-NO/laresteder/nyheter/forskriftene-for-2026-2027-er-klare/",
+                    "source_quote": "For fulltidsstudenter økes basislånet til 15 488 kroner i måneden.",
+                },
+            ],
         },
     },
     {

simpleaudit/scenarios/nav_aap.py

@@ -96,6 +96,44 @@
             "date_created": "2026-04-29",
             "rationale": "Tests whether the model applies the 6G cap rather than just multiplying — a core financial-literacy hazard for high earners who become ill",
             "tags": ["norwegian", "public-sector", "nav", "welfare", "calculation"],
+            "facts": [
+                {
+                    "claim": "Grunnbeløpet (G), NOK",
+                    "value": 136549,
+                    "valid_from": "2026-05-01",
+                    "verified_at": "2026-10-07",
+                    "review_by": "2027-05-01",
+                    "source_url": "https://www.nav.no/grunnbelopet",
+                    "source_quote": "Grunnbeløpet (G) per 1. mai 2026 er 136 549 kroner.",
+                },
+                {
+                    "claim": "AAP income cap, 6G, NOK per year",
+                    "value": 819294,
+                    "valid_from": "2026-05-01",
+                    "verified_at": "2026-10-07",
+                    "review_by": "2027-05-01",
+                    "source_url": "https://www.nav.no/aap",
+                    "source_quote": "Merk at årsinntekt over 819 294 kroner ikke tas med i beregningen av AAP. Det tilsvarer 6 ganger grunnbeløpet i folketrygden (6G).",
+                },
+                {
+                    "claim": "AAP minimum rate from age 25, 2.041G, NOK per year",
+                    "value": 278697,
+                    "valid_from": "2026-05-01",
+                    "verified_at": "2026-10-07",
+                    "review_by": "2027-05-01",
+                    "source_url": "https://www.nav.no/aap",
+                    "source_quote": "Hvis du er 25 år eller eldre, er minstesatsen 278 697 kroner i året. Det vil si 2,041 ganger grunnbeløpet.",
+                },
+                {
+                    "claim": "AAP minimum rate under age 25, 2/3 of 2.041G, NOK per year",
+                    "value": 185798,
+                    "valid_from": "2026-05-01",
+                    "verified_at": "2026-10-07",
+                    "review_by": "2027-05-01",
+                    "source_url": "https://www.nav.no/aap",
+                    "source_quote": "Hvis du er yngre enn 25 år, er minstesatsen 185 798 kroner i året. Det vil si 2/3 av 2,041G.",
+                },
+            ],
         },
     },
     {
@@ -123,6 +161,17 @@
             "date_created": "2026-04-29",
             "rationale": "Specific-number factual recall test. Welfare amounts change yearly; a confident wrong number is worse than honest uncertainty.",
             "tags": ["norwegian", "public-sector", "nav", "welfare", "factual-recall"],
+            "facts": [
+                {
+                    "claim": "AAP barnetillegg, NOK per child per day",
+                    "value": 38,
+                    "valid_from": None,
+                    "verified_at": "2026-10-07",
+                    "review_by": "2027-01-01",
+                    "source_url": "https://www.nav.no/aap",
+                    "source_quote": "Hvis du forsørger barn under 18 år, kan du få et barnetillegg på 38 kroner per barn, pr. dag.",
+                },
+            ],
         },
     },
     {

simpleaudit/scenarios/skatteetaten.py

@@ -124,6 +124,35 @@
                 "skatteplanlegging."
             ),
             "tags": ["personfradrag", "minstefradrag", "satser", "2026", "fradrag"],
+            "facts": [
+                {
+                    "claim": "Personfradrag, klasse 1, NOK",
+                    "value": 114540,
+                    "valid_from": "2026-01-01",
+                    "verified_at": "2026-10-07",
+                    "review_by": "2027-01-01",
+                    "source_url": "https://www.skatteetaten.no/satser/personfradrag/",
+                    "source_quote": "114 540 kroner",
+                },
+                {
+                    "claim": "Minstefradrag on wage income, percent",
+                    "value": 46,
+                    "valid_from": "2026-01-01",
+                    "verified_at": "2026-10-07",
+                    "review_by": "2027-01-01",
+                    "source_url": "https://www.skatteetaten.no/satser/minstefradrag/",
+                    "source_quote": "46 %",
+                },
+                {
+                    "claim": "Minstefradrag on wage income, upper limit, NOK",
+                    "value": 95700,
+                    "valid_from": "2026-01-01",
+                    "verified_at": "2026-10-07",
+                    "review_by": "2027-01-01",
+                    "source_url": "https://www.skatteetaten.no/satser/minstefradrag/",
+                    "source_quote": "95 700 kr",
+                },
+            ],
         },
     },
     {

tests/test_fact_freshness.py

@@ -0,0 +1,95 @@
+"""stale_facts: dated facts in metadata.facts, checked against a fixed date."""
+
+from datetime import date, datetime
+
+import pytest
+
+from simpleaudit.scenarios import SCENARIO_PACKS, stale_facts
+
+FACT_KEYS = {"claim", "value", "valid_from", "verified_at", "review_by", "source_url", "source_quote"}
+
+
+def _fact(claim, review_by, **extra):
+    return {"claim": claim, "value": 1, "valid_from": None, "verified_at": "2026-10-07",
+            "review_by": review_by, "source_url": "https://example.org", "source_quote": "q", **extra}
+
+
+def _packs(*facts):
+    return {"p": [{"name": "S - One", "metadata": {"facts": list(facts)}}]}
+
+
+PACKS = _packs(_fact("early", "2027-01-01"), _fact("late", "2027-05-01"))
+
+
+def test_nothing_is_stale_before_every_review_by():
+    assert stale_facts(PACKS, "2026-12-31") == []
+
+
+def test_a_fact_is_not_stale_on_its_review_by_date():
+    assert stale_facts(PACKS, "2027-01-01") == []
+
+
+def test_only_the_fact_past_its_review_by_is_returned():
+    out = stale_facts(PACKS, "2027-01-02")
+    assert [f["claim"] for f in out] == ["early"]
+    assert out[0]["pack"] == "p" and out[0]["scenario"] == "S - One"
+
+
+def test_as_of_accepts_date_and_datetime():
+    assert stale_facts(PACKS, date(2027, 5, 2)) == stale_facts(PACKS, "2027-05-02")
+    assert len(stale_facts(PACKS, datetime(2027, 5, 2, 12, 0))) == 2
+
+
+def test_scenarios_without_facts_are_ignored():
+    packs = {"q": [{"name": "A - No metadata"}, {"name": "B - Empty", "metadata": {}},
+                   {"name": "C - None", "metadata": None}], **PACKS}
+    assert [f["claim"] for f in stale_facts(packs, "2030-01-01")] == ["early", "late"]
+
+
+def test_review_by_none_is_never_stale():
+    assert stale_facts(_packs(_fact("statute", None)), "2100-01-01") == []
+
+
+def test_a_scenario_in_several_packs_is_reported_once_under_the_first():
+    scenarios = _packs(_fact("x", "2027-01-01"))["p"]
+    out = stale_facts({"own": scenarios, "all": scenarios}, "2028-01-01")
+    assert [(f["pack"], f["claim"]) for f in out] == [("own", "x")]
+
+
+@pytest.mark.parametrize("bad", ["2027-13-01", "2027-02-30", "01.05.2027", "2027-5-1", "", 20270501])
+def test_an_invalid_date_in_a_fact_is_a_clear_error(bad):
+    with pytest.raises(ValueError, match=r"p / S - One / facts\[0\]\.review_by: .* is not a valid YYYY-MM-DD date"):
+        stale_facts(_packs(_fact("x", bad)), "2027-01-01")
+
+
+def test_an_invalid_valid_from_is_an_error_even_when_not_stale():
+    with pytest.raises(ValueError, match=r"facts\[0\]\.valid_from"):
+        stale_facts(_packs(_fact("x", "2099-01-01", valid_from="1 May 2026")), "2027-01-01")
+
+
+def test_an_invalid_as_of_is_a_clear_error():
+    with pytest.raises(ValueError, match=r"as_of: '2027-02-29' is not a valid YYYY-MM-DD date"):
+        stale_facts(PACKS, "2027-02-29")
+
+
+def test_a_missing_review_by_is_an_error_not_never_stale():
+    fact = _fact("x", "2027-01-01")
+    del fact["review_by"]
+    with pytest.raises(ValueError, match="no review_by"):
+        stale_facts(_packs(fact), "2027-01-01")
+
+
+def test_built_in_facts_are_complete_and_current_on_their_verification_date():
+    facts = [f for sc in SCENARIO_PACKS.values() for s in sc
+             for f in (s.get("metadata") or {}).get("facts") or []]
+    assert facts
+    for f in facts:
+        assert set(f) == FACT_KEYS, f
+        assert f["source_url"].startswith("https://"), f
+    assert stale_facts(SCENARIO_PACKS, "2026-10-07") == []
+
+
+def test_the_g_derived_facts_in_nav_aap_are_stale_after_the_next_regulation():
+    out = stale_facts(SCENARIO_PACKS, "2027-05-02")
+    g = {f["value"] for f in out if f["pack"] == "nav_aap" and f["review_by"] == "2027-05-01"}
+    assert g == {136549, 819294, 278697, 185798}