Skip to content

Commit c35b2a9

Browse files
committed
New findings can be read by the model as soon as a scan finds them
Auto LLM review on the AI page validates every finding a scan turns up for the first time, the same way as pressing the button on each one. It is off by default. Questions are spaced to a requests per minute setting, on one clock shared by every scan, so two scans finishing together stay under it. Findings a learned rule covers send nothing and do not wait. A per scan limit reads the worst first, so a first scan of a large estate does not become a very large bill, and what is left over is in the audit log. Interrupted scans whose work is kept are reviewed too. The findings list badges every finding a model read or a learned rule graded, and a new AI review filter shows the reviewed ones or the rest. It works with select all matching, so the leftovers can be sent in one go.
1 parent 6a157bc commit c35b2a9

10 files changed

Lines changed: 271 additions & 14 deletions

File tree

‎app/core/ai.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,16 @@
9797
"ai.fetch_advisories": (True, False,
9898
"Read the advisory text from osv.dev for a dependency "
9999
"finding. Only the advisory id is sent."),
100+
# Off by default for the same reason the provider is. On, every finding
101+
# a scan turns up for the first time is read without anybody asking.
102+
"ai.auto_review": (False, False,
103+
"Have the model read every new finding a scan turns up."),
104+
"ai.auto_review_per_minute": (6, False,
105+
"Most questions the automatic review sends in a "
106+
"minute."),
107+
"ai.auto_review_max": (200, False,
108+
"Most new findings one scan hands to the automatic "
109+
"review, worst first. 0 for no limit."),
100110
# What the custom service charges, per million tokens, for the cost page.
101111
# Kept as text so 0.10 stays 0.10. Empty is not counted.
102112
"ai.price_input": ("", False, "Price per million input tokens."),
@@ -175,6 +185,11 @@ def read_form(form, text, number, url, checked) -> dict:
175185
"Lines of context", 5, 200),
176186
"ai.send_code": checked("ai.send_code"),
177187
"ai.fetch_advisories": checked("ai.fetch_advisories"),
188+
"ai.auto_review": checked("ai.auto_review"),
189+
"ai.auto_review_per_minute": number(form.get("ai.auto_review_per_minute") or 6,
190+
"Questions per minute", 1, 600),
191+
"ai.auto_review_max": number(form.get("ai.auto_review_max") or 200,
192+
"New findings per scan", 0, 100_000),
178193
}
179194

180195

‎app/core/analysis.py‎

Lines changed: 95 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -41,14 +41,16 @@
4141
import re
4242
import subprocess
4343
import threading
44+
import time
4445
from pathlib import Path
46+
from types import SimpleNamespace
4547

4648
import httpx
4749

4850
from app.core import ai, learned
4951
from app.core.db import session_scope
5052
from app.core.models import (Finding, FindingAnalysis, FindingKind,
51-
FindingState, Severity, Waiver, utcnow)
53+
FindingState, Severity, UserRole, Waiver, utcnow)
5254

5355
log = logging.getLogger("gcr.analysis")
5456

@@ -591,7 +593,7 @@ def _parse(text: str) -> tuple:
591593
# --------------------------------------------------------------------- run
592594

593595
def run(finding_id: int, user, force: bool = False, wait: bool = False,
594-
learn: bool = True, requeue: bool = True) -> dict:
596+
learn: bool = True, requeue: bool = True, paced: bool = False) -> dict:
595597
"""Ask about one finding and keep the answer. -> {ok, detail, id}
596598
597599
The caller has already checked the person may see the finding and may ask.
@@ -600,6 +602,7 @@ def run(finding_id: int, user, force: bool = False, wait: bool = False,
600602
refusing, which is what a bulk run wants. learn=False keeps the answer
601603
to this one finding. requeue=False leaves the findings of a rule this
602604
answer retires at their restored grade instead of asking about each.
605+
paced keeps any batch this starts under the automatic review's rate.
603606
"""
604607
with _busy_lock:
605608
if finding_id in _busy:
@@ -612,7 +615,7 @@ def run(finding_id: int, user, force: bool = False, wait: bool = False,
612615
return {"ok": False,
613616
"detail": "Two validations are already running. Try again in a moment."}
614617
try:
615-
return _run(finding_id, user, force, learn, requeue)
618+
return _run(finding_id, user, force, learn, requeue, paced)
616619
finally:
617620
with _busy_lock:
618621
_busy.discard(finding_id)
@@ -647,7 +650,7 @@ def _answer_from_rule(db, row, user) -> dict:
647650

648651

649652
def _run(finding_id: int, user, force: bool = False, learn: bool = True,
650-
requeue: bool = True) -> dict:
653+
requeue: bool = True, paced: bool = False) -> dict:
651654
from app.core import rbac
652655

653656
with session_scope() as db:
@@ -715,7 +718,8 @@ def _run(finding_id: int, user, force: bool = False, learn: bool = True,
715718

716719
# Started after the commit, so the batch reads the grades put back.
717720
if others and requeue:
718-
number = start_batch(others, user, force=True, learn=False, requeue=False)
721+
number = start_batch(others, user, force=True, learn=False, requeue=False,
722+
paced=paced)
719723
outcome += (f" The {len(others)} other finding{'' if len(others) == 1 else 's'} "
720724
f"it graded {'was' if len(others) == 1 else 'were'} put back and "
721725
f"queued for {'its' if len(others) == 1 else 'their'} own reading "
@@ -869,15 +873,16 @@ def rule_only(finding_id: int, user) -> dict:
869873

870874

871875
def start_batch(finding_ids: list, user, rules_only: bool = False, force: bool = False,
872-
learn: bool = True, requeue: bool = True) -> int:
876+
learn: bool = True, requeue: bool = True, paced: bool = False) -> int:
873877
"""Validate many findings in the background. -> the batch number.
874878
875879
Ordered so findings about the same thing sit together: the same
876880
repository, kind and rule or package. The first of a group goes to the
877881
model, and when what it learns covers the rest they are answered from the
878882
rule without a model call, which is where a bulk run saves its tokens.
879883
force asks the model about every one, past any rule, which is how the
880-
findings of a retired rule each get their own reading.
884+
findings of a retired rule each get their own reading. paced spaces the
885+
questions out to the automatic review's rate.
881886
"""
882887
with _batch_lock:
883888
_batch_seq[0] += 1
@@ -888,12 +893,12 @@ def start_batch(finding_ids: list, user, rules_only: bool = False, force: bool =
888893
"done": 0, "asked": 0, "from_rules": 0, "lowered": 0,
889894
"skipped_protected": 0, "skipped_done": 0, "errors": 0,
890895
"tokens": 0, "rules_learned": 0, "skipped_no_rule": 0,
891-
"rules_only": rules_only, "results": []}
896+
"rules_only": rules_only, "paced": paced, "results": []}
892897
# Keep the twenty newest; a batch is a progress display, not a record.
893898
for old in sorted(_batches)[:-20]:
894899
_batches.pop(old, None)
895900
worker = threading.Thread(target=_batch, args=(number, list(finding_ids), user, rules_only,
896-
force, learn, requeue),
901+
force, learn, requeue, paced),
897902
name=f"ai-batch-{number}", daemon=True)
898903
worker.start()
899904
return number
@@ -921,7 +926,8 @@ def _note(number: int, **change) -> None:
921926

922927

923928
def _batch(number: int, finding_ids: list, user, rules_only: bool = False,
924-
force: bool = False, learn: bool = True, requeue: bool = True) -> None:
929+
force: bool = False, learn: bool = True, requeue: bool = True,
930+
paced: bool = False) -> None:
925931
from app.core import rbac
926932
from app.core.models import AIGrade, LearnedRule
927933

@@ -934,6 +940,11 @@ def _batch(number: int, finding_ids: list, user, rules_only: bool = False,
934940
(f.package_name if f.kind == FindingKind.DEPENDENCY else f.rule) or "",
935941
-learned.RANK.get(f.severity.value, 0), f.id))
936942
plan = []
943+
# A finding a rule already covers is answered without a request,
944+
# so it does not wait its turn. A rule learned partway through
945+
# can only make the pacing slower than it needs to be.
946+
rules = learned.load_rules(db) if paced and not force else []
947+
per_minute = int(ai.config(db).get("ai.auto_review_per_minute") or 0)
937948
for f in order:
938949
why = learned.protected(f)
939950
if why:
@@ -944,7 +955,8 @@ def _batch(number: int, finding_ids: list, user, rules_only: bool = False,
944955
if done is not None:
945956
plan.append((f.id, f.vuln_id, "done"))
946957
continue
947-
plan.append((f.id, f.vuln_id, "ask"))
958+
covered = bool(rules) and learned.match(rules, f) is not None
959+
plan.append((f.id, f.vuln_id, "rule" if covered else "ask"))
948960
rules_before = db.query(LearnedRule).count()
949961
for finding_id, vuln_id, step in plan:
950962
label = vuln_id or str(finding_id)
@@ -957,10 +969,12 @@ def _batch(number: int, finding_ids: list, user, rules_only: bool = False,
957969
_note(number, done=1, skipped_done=1,
958970
result=f"{label}: already validated for this code")
959971
continue
972+
if paced and step == "ask" and not rules_only:
973+
_wait_turn(per_minute)
960974
try:
961975
result = (rule_only(finding_id, user) if rules_only
962976
else run(finding_id, user, force=force, wait=True,
963-
learn=learn, requeue=requeue))
977+
learn=learn, requeue=requeue, paced=paced))
964978
except Exception: # one bad finding must not stop the rest
965979
log.exception("bulk validation of %s failed", finding_id)
966980
result = {"ok": False, "detail": "failed; see the log"}
@@ -985,3 +999,72 @@ def _batch(number: int, finding_ids: list, user, rules_only: bool = False,
985999
with _batch_lock:
9861000
if number in _batches:
9871001
_batches[number]["finished"] = utcnow()
1002+
1003+
1004+
# ------------------------------------------------------------ automatic
1005+
1006+
# Who the automatic review says it is. It reads what a scan found for
1007+
# everybody, so it sees what an administrator sees, and its name is on every
1008+
# answer it leaves so nobody mistakes it for a person.
1009+
AUTO_REVIEWER = SimpleNamespace(username="auto-review", role=UserRole.ADMIN,
1010+
disabled=False)
1011+
1012+
# One clock for every paced batch. Two scans finishing close together start
1013+
# two batches, and each keeping to the rate on its own would be twice it.
1014+
_pace_lock = threading.Lock()
1015+
_next_turn = [0.0]
1016+
1017+
1018+
def _wait_turn(per_minute: int) -> None:
1019+
"""Hold until the next question may go. 0 or less never waits."""
1020+
if per_minute <= 0:
1021+
return
1022+
gap = 60.0 / per_minute
1023+
with _pace_lock:
1024+
now = time.monotonic()
1025+
turn = max(now, _next_turn[0])
1026+
_next_turn[0] = turn + gap
1027+
if turn > now:
1028+
time.sleep(turn - now)
1029+
1030+
1031+
def auto_review(finding_ids) -> int:
1032+
"""Hand a scan's new findings to the model, when that is switched on.
1033+
-> the batch number, or 0 when nothing was started.
1034+
1035+
Worst first, up to the per scan limit, so a first scan of a large estate
1036+
reads the criticals and stops instead of running up a bill for every low
1037+
there is. The rest are still there to pick from the findings list.
1038+
Never raises: a scan is not worth failing over this.
1039+
"""
1040+
from app.core.audit import record
1041+
1042+
ids = sorted({int(i) for i in finding_ids or []})
1043+
if not ids:
1044+
return 0
1045+
try:
1046+
with session_scope() as db:
1047+
cfg = ai.config(db)
1048+
if not cfg.get("ai.auto_review") or not ai.enabled(cfg) or ai.missing(cfg):
1049+
return 0
1050+
limit = int(cfg.get("ai.auto_review_max") or 0)
1051+
rows = (db.query(Finding.id, Finding.severity)
1052+
.filter(Finding.id.in_(ids)).all())
1053+
rows.sort(key=lambda r: (-learned.RANK.get(learned._value(r.severity), 0), r.id))
1054+
chosen = [r.id for r in rows]
1055+
left = 0
1056+
if limit and len(chosen) > limit:
1057+
left = len(chosen) - limit
1058+
chosen = chosen[:limit]
1059+
if not chosen:
1060+
return 0
1061+
number = start_batch(chosen, AUTO_REVIEWER, paced=True)
1062+
record(db, "ai.auto_review", actor=AUTO_REVIEWER.username,
1063+
detail=(f"batch {number}: {len(chosen)} new finding(s) queued at "
1064+
f"{cfg.get('ai.auto_review_per_minute')} a minute"
1065+
+ (f", {left} over the per scan limit left for a person"
1066+
if left else "")))
1067+
return number
1068+
except Exception: # noqa: BLE001
1069+
log.exception("could not start the automatic review")
1070+
return 0

‎app/core/salvage.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,8 +218,10 @@ def salvage(session, results_dir) -> dict:
218218
log.warning("could not keep the software inventory from the "
219219
"interrupted scan", exc_info=True)
220220

221-
counts.pop("new_findings", None)
221+
new_ids = [f.id for f in counts.pop("new_findings", None) or []]
222222
counts["repositories"] = len(entries)
223223
log.warning("salvaged %d repository(ies) from an interrupted scan: %s",
224224
len(entries), counts)
225+
# For the automatic review, which the caller starts once these commit.
226+
counts["new_finding_ids"] = new_ids
225227
return counts

‎app/core/scanner.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,9 @@ def _keep_what_finished(run_id, results_dir, why: str) -> dict:
360360
log.exception("could not keep what the scan had already found")
361361
return {"repositories": 0}
362362

363+
from app.core import analysis
364+
analysis.auto_review(kept.pop("new_finding_ids", None))
365+
363366
if not kept.get("repositories"):
364367
return kept
365368

@@ -449,6 +452,8 @@ def salvage_interrupted() -> int:
449452
f"from a run that was interrupted"),
450453
outcome="warn")
451454
_drop_results(directory)
455+
from app.core import analysis
456+
analysis.auto_review(kept.pop("new_finding_ids", None))
452457
except Exception: # noqa: BLE001 - never stop the app booting
453458
log.exception("could not salvage %s", directory.name)
454459

@@ -1000,6 +1005,9 @@ def run_scan(triggered_by: str = "", trigger: str = "manual",
10001005
tagged = connection_store.by_id(db, connection.id)
10011006
counts = ingest_report(db, report, scanned_repos=scanned,
10021007
connection=tagged)
1008+
# Ids now, while the rows are attached. The review starts once
1009+
# this session has committed them.
1010+
new_ids = [f.id for f in counts.get("new_findings") or []]
10031011

10041012
# The software inventory, from the SBOM bundle beside the
10051013
# report. Its own step because it can fail on a scan that
@@ -1057,6 +1065,9 @@ def run_scan(triggered_by: str = "", trigger: str = "manual",
10571065
counts.pop("new_findings", None)
10581066
log.info("scan done: %s", counts)
10591067

1068+
from app.core import analysis
1069+
analysis.auto_review(new_ids)
1070+
10601071
# The findings are in the database now, so the copies on disk have
10611072
# done their job. Removed here rather than in a finally, because
10621073
# every other way out of this function wants them kept.

0 commit comments

Comments
 (0)