Skip to content

Commit 1476d04

Browse files
committed
Security report: deeper findings, Word download, and one request limit
Each issue in the security report now carries what a reviewer needs to act on it: - The masked code involved, read while the copy is held, with a credential's own line withheld. - A CVSS 3.1 vector from the review, scored in code by the published formula. Ratings still come from likelihood times impact, so reports already written keep their ratings. - The weakness (CWE), its OWASP Top 10 and API Security Top 10 category, and references, all worked out from the CWE with no model involved. Old reports get them too, at render time. - Numbered steps telling how an attacker would abuse it, an example of the fix, whether the code proves it or a running system must be checked, and one safe step that confirms the fix. The report gains a verdict on the cover, a findings at a glance count, OWASP coverage tables, a retest plan and a standards appendix. "Download as Word" builds the same report as a .docx (python-docx), on download, from the stored report. New setting ai.max_request_tokens (86000): no single request may pass it, what is sent plus the answer included. Batches and the repository read are sized to fit, and a question that would not fit is split before it is sent rather than after it fails. All of it still runs as the report's background job. The report's checkout is now memory only: with no room in /dev/shm the report is written from the scanner's records rather than putting code on disk. Single finding validation and diagrams now prefer memory too.
1 parent 20599b1 commit 1476d04

12 files changed

Lines changed: 1017 additions & 22 deletions

‎app/core/ai.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,12 @@
8686
"ai.max_output_tokens": (26000, False,
8787
"The most the service writes in one answer, "
8888
"reasoning included."),
89+
# The service's limit on one request, what is sent and what comes back
90+
# together. Long jobs size every question to fit under it and split the
91+
# work into more questions rather than send one that would not fit.
92+
"ai.max_request_tokens": (86000, False,
93+
"The most one request may use, sent and answered "
94+
"together."),
8995
"ai.tokens_per_minute": (40000, False,
9096
"Tokens a long job may use in a minute, in and out "
9197
"together. 0 for no limit."),
@@ -193,6 +199,8 @@ def read_form(form, text, number, url, checked) -> dict:
193199
"ai.timeout": number(form.get("ai.timeout") or 300, "Time limit", 30, 1800),
194200
"ai.max_output_tokens": number(form.get("ai.max_output_tokens") or 26000,
195201
"Service answer limit", 1000, 200000),
202+
"ai.max_request_tokens": number(form.get("ai.max_request_tokens") or 86000,
203+
"Request limit", 8000, 2_000_000),
196204
"ai.tokens_per_minute": number(form.get("ai.tokens_per_minute") or 0,
197205
"Tokens per minute", 0, 10_000_000),
198206
"ai.context_lines": number(form.get("ai.context_lines") or 40,

‎app/core/analysis.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -491,7 +491,7 @@ def _code(f, cfg: dict, branch: str, connection: str) -> tuple:
491491
return "", [], ["No code was sent: this finding is not tied to a file."]
492492

493493
context = max(5, min(200, int(cfg.get("ai.context_lines") or 40)))
494-
with recheck.branch_tip(connection, f["repository"], branch) as (root, why):
494+
with recheck.branch_tip(connection, f["repository"], branch, in_memory=True) as (root, why):
495495
if root is None:
496496
return "", [], [f"No code was sent: the branch could not be "
497497
f"fetched ({why})."]

‎app/core/assessment.py‎

Lines changed: 168 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@
5252
from app.core.models import (Finding, FindingKind, FindingState, RepoAssessment,
5353
Repository, utcnow)
5454

55+
from app.core import report_standards
56+
5557
log = logging.getLogger("gcr.assessment")
5658

5759
# One report at a time. Each holds a copy of a branch and a run of questions
@@ -66,7 +68,7 @@
6668
# A planning figure for one reviewed group's answer, in tokens. The prompt
6769
# holds each field to a word limit, and this is comfortably over what that
6870
# comes to.
69-
ITEM_TOKENS = 450
71+
ITEM_TOKENS = 900
7072
# The share of the service's answer limit an answer is planned to use. The
7173
# rest is left for the model's thinking, which counts against the same limit.
7274
ANSWER_SHARE = 0.4
@@ -76,8 +78,16 @@
7678
MAX_SNIPPET_CHARS = 3_000
7779
MAX_SNIPPETS = 3
7880
OVERVIEW_CHARS = 40_000
81+
# Characters per token when planning what is sent. Code runs denser than
82+
# prose, so this errs on the side of fewer characters per question.
83+
CHARS_PER_TOKEN = 3
84+
# Room kept free in every request for the wrapper the service adds.
85+
REQUEST_SAFETY = 2_000
86+
# The masked lines shown in the report beside each issue.
87+
DISPLAY_CONTEXT = 4
88+
MAX_DISPLAY_CHARS = 1_600
7989
# The summary and overview answers are short, and planned as such.
80-
OVERVIEW_TOKENS = 2_500
90+
OVERVIEW_TOKENS = 6_000
8191
SUMMARY_TOKENS = 2_500
8292

8393
SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3, "info": 4,
@@ -497,10 +507,20 @@ def _snippet(root: Path, group: dict, context: int) -> str:
497507
"title": "what is wrong, in under twelve words, plain language",
498508
"likelihood": 1 to 5,
499509
"impact": 1 to 5,
500-
"what": "what was found, at most 60 words",
510+
"cvss": "the CVSS 3.1 base vector for this issue as it stands in this code, such as \
511+
AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N, or empty for a false positive",
512+
"what": "what is wrong and how the code gets there, at most 90 words",
501513
"evidence": "what in the code or data shows it, naming files and lines, at most 60 words",
502514
"why": "what could happen to the business, at most 60 words",
503-
"fixes": ["up to four concrete steps, each at most 25 words"]}"""
515+
"attack": ["three to five short steps telling how an attacker would abuse it, \
516+
in plain words a non expert follows, each at most 30 words; empty for a false positive"],
517+
"fixes": ["up to four concrete steps, each at most 25 words"],
518+
"fix_example": "a short example of the corrected code or setting, at most 12 lines, \
519+
or empty when a fix is not code; never a secret value",
520+
"status": "confirmed_in_source when the code you were shown proves it, or \
521+
needs_runtime when it depends on a setting or system you were not shown",
522+
"runtime_check": "for needs_runtime, what to check in the running system, at most 30 words",
523+
"retest": "one safe step that proves the fix worked, at most 30 words"}"""
504524

505525
STYLE = """\
506526
Write for business owners who do not work in security: US English, about a \
@@ -630,6 +650,11 @@ def __init__(self, cfg: dict):
630650
if cfg.get("ai.provider") != "custom":
631651
cap = min(cap, _int(cfg.get("ai.max_tokens"), 1000, 10_000_000, 16000))
632652
self.cap = cap
653+
# One request, sent and answered together, may not pass this. The
654+
# custom service writes up to its own answer limit whatever is asked,
655+
# so the room left for what is sent is the limit less that answer.
656+
self.limit = _int(cfg.get("ai.max_request_tokens"), 8000, 10_000_000, 86000)
657+
self.custom = cfg.get("ai.provider") == "custom"
633658
self.pacer = _Pacer(cfg.get("ai.tokens_per_minute") or 0)
634659
self.calls = self.input_tokens = self.output_tokens = 0
635660
self.provider = self.model = ""
@@ -638,10 +663,28 @@ def __init__(self, cfg: dict):
638663
def batch_size(self) -> int:
639664
return max(1, min(MAX_BATCH, int(self.cap * ANSWER_SHARE) // ITEM_TOKENS))
640665

666+
def input_room(self, system: str = "") -> int:
667+
"""Characters that may be sent with this system prompt in one request."""
668+
answer = self.cap if self.custom else min(self.cap, self.limit // 3)
669+
tokens = self.limit - answer - REQUEST_SAFETY - len(system) // CHARS_PER_TOKEN
670+
return max(2_000, tokens * CHARS_PER_TOKEN)
671+
672+
def fits(self, system: str, prompt: str) -> bool:
673+
return len(prompt) <= self.input_room(system)
674+
641675
def ask(self, system: str, prompt: str, planned_out: int) -> tuple:
642-
"""-> (parsed object or None, problem, cut_off)."""
643-
self.pacer.wait(len(system + prompt) // 4 + planned_out)
644-
answer = ai.complete(self.cfg, system, prompt, max_tokens=self.cap,
676+
"""-> (parsed object or None, problem, cut_off).
677+
678+
A question that would not fit in one request is not sent: it comes
679+
back as cut off, which is what makes the caller split it.
680+
"""
681+
if not self.fits(system, prompt):
682+
return None, "too large for one request", True
683+
sent = (len(system) + len(prompt)) // CHARS_PER_TOKEN
684+
out_cap = self.cap if self.custom else max(
685+
1000, min(self.cap, self.limit - sent - REQUEST_SAFETY))
686+
self.pacer.wait(sent + planned_out)
687+
answer = ai.complete(self.cfg, system, prompt, max_tokens=out_cap,
645688
purpose=PURPOSE)
646689
self.calls += 1
647690
self.input_tokens += answer.get("input_tokens") or 0
@@ -679,9 +722,71 @@ def _clean_item(raw: dict, fallback: dict | None = None) -> dict:
679722
"why": tidy(raw.get("why"), 600),
680723
"fixes": [tidy(x, 300) for x in fixes[:4] if str(x or "").strip()],
681724
}
725+
attack = raw.get("attack") if isinstance(raw.get("attack"), list) else []
726+
item["attack"] = ([] if verdict == "false_positive" else
727+
[tidy(x, 300) for x in attack[:5] if str(x or "").strip()])
728+
item["fix_example"] = _code_text(raw.get("fix_example"))
729+
status = str(raw.get("status") or "").strip().lower()
730+
item["status"] = status if status in ("confirmed_in_source", "needs_runtime") else ""
731+
item["runtime_check"] = (tidy(raw.get("runtime_check"), 300)
732+
if item["status"] == "needs_runtime" else "")
733+
item["retest"] = tidy(raw.get("retest"), 300)
734+
vector, score = report_standards.cvss31(raw.get("cvss"))
735+
item["cvss"], item["cvss_score"] = ("", None) if verdict == "false_positive" else (vector, score)
682736
return item
683737

684738

739+
def _code_text(value) -> str:
740+
"""A model's example code, masked and kept short. Lines, not prose."""
741+
from app.core.diagram import mask
742+
743+
text = str(value or "").replace("\r\n", "\n").strip().strip("`").strip()
744+
if not text:
745+
return ""
746+
lines = [mask(line)[:160] for line in text.split("\n")[:12]]
747+
return "\n".join(lines)[:1400]
748+
749+
750+
def _display_code(lines: list, line_numbers: list, credential: bool = False,
751+
mule: bool = False) -> str:
752+
"""The masked lines shown beside an issue in the report."""
753+
from app.core.diagram import mask
754+
755+
out, used = [], 0
756+
for line_no in (line_numbers or [])[:2]:
757+
lo = max(1, line_no - DISPLAY_CONTEXT)
758+
hi = min(len(lines), line_no + DISPLAY_CONTEXT)
759+
part = []
760+
for n in range(lo, hi + 1):
761+
text = lines[n - 1][:160]
762+
if credential and n in line_numbers:
763+
text = ("<the flagged value is withheld: Mule ciphertext>" if mule
764+
else "<the flagged value is withheld>")
765+
part.append(f"{n:5} {mask(text)}")
766+
block = "\n".join(part)
767+
if used + len(block) > MAX_DISPLAY_CHARS:
768+
break
769+
out.append(block)
770+
used += len(block)
771+
return "\n ...\n".join(out)
772+
773+
774+
def _file_lines(root: Path, rel: str) -> list:
775+
if root is None or not rel:
776+
return []
777+
path = (root / rel).resolve()
778+
try:
779+
path.relative_to(root.resolve())
780+
except ValueError:
781+
return []
782+
if not path.is_file() or path.is_symlink() or path.stat().st_size > 2_000_000:
783+
return []
784+
try:
785+
return path.read_text("utf-8", "replace").splitlines()
786+
except OSError:
787+
return []
788+
789+
685790
def _group_evidence(ref: str, group: dict, root, send_code: bool, context: int) -> str:
686791
facts = [f"ref: {ref}", f"kind: {group['kind']}", f"rule: {group['rule']}",
687792
f"scanner title: {group['title']}",
@@ -891,7 +996,9 @@ def _write(repository: str, branch: str, connection: str, actor: str) -> None:
891996
notes = []
892997

893998
_update(repository, branch, stage="Fetching the branch")
894-
with recheck.branch_tip(connection, repository, branch, in_memory=True) as (root, why):
999+
# The code is held in memory only, never on disk, so nothing of it is
1000+
# left behind on the server whatever happens to this job.
1001+
with recheck.branch_tip(connection, repository, branch, memory_only=True) as (root, why):
8951002
in_memory = bool(root is not None and str(root).startswith(recheck.MEMORY_DIR))
8961003
if root is None:
8971004
notes.append(f"The branch could not be fetched ({why}), so the review "
@@ -916,6 +1023,21 @@ def _write(repository: str, branch: str, connection: str, actor: str) -> None:
9161023
notes += gather_notes
9171024
# Everything that reads the code happens inside this block, so the
9181025
# copy is gone before the first question about findings is asked.
1026+
# What the report shows beside each issue, read now while the copy
1027+
# is here. Masked like everything else, and only when code may be
1028+
# shared at all.
1029+
shown = {}
1030+
if root is not None and send_code:
1031+
for g in groups:
1032+
if g["file"] and g["lines"]:
1033+
g["display_code"] = _display_code(
1034+
_file_lines(Path(root), g["file"]), g["lines"],
1035+
g["kind"] == "credential", bool(g.get("mule_encrypted")))
1036+
# The overview may only name files it was shown, so those are the
1037+
# files its concerns can point into.
1038+
for label in re.findall(r'<evidence label="File ([^"]+)">', overview_evidence):
1039+
rel = html.unescape(label)
1040+
shown[rel] = _file_lines(Path(root), rel)
9191041
refs = [f"G{i + 1}" for i in range(len(groups))]
9201042
# A group already decided, by a person or by its own validation, is
9211043
# not asked about again: the report carries that decision.
@@ -931,9 +1053,15 @@ def _write(repository: str, branch: str, connection: str, actor: str) -> None:
9311053
overview = {"what_it_is": "", "positives": [], "concerns": []}
9321054
if overview_evidence:
9331055
_update(repository, branch, stage="Reading the repository")
934-
prompt = (f"Repository: {repository}\nBranch: {branch}\n"
935-
f"Visibility: {facts['visibility'] or 'unknown'}\n"
936-
+ _site_lines(site) + "\n" + overview_evidence)
1056+
head = (f"Repository: {repository}\nBranch: {branch}\n"
1057+
f"Visibility: {facts['visibility'] or 'unknown'}\n"
1058+
+ _site_lines(site) + "\n")
1059+
room = asker.input_room(OVERVIEW_SYSTEM) - len(head) - 200
1060+
if len(overview_evidence) > room:
1061+
cut_at = overview_evidence.rfind("</evidence>", 0, room)
1062+
overview_evidence = overview_evidence[:cut_at + 11] if cut_at > 0 else overview_evidence[:room]
1063+
notes.append("The repository read was cut to fit in one request.")
1064+
prompt = head + overview_evidence
9371065
obj, problem, cut = asker.ask(OVERVIEW_SYSTEM, prompt, OVERVIEW_TOKENS)
9381066
if obj is None and not cut and "JSON" not in problem:
9391067
raise RuntimeError(problem)
@@ -953,9 +1081,14 @@ def _write(repository: str, branch: str, connection: str, actor: str) -> None:
9531081

9541082
# ---- the findings, a batch at a time
9551083
batches, current_batch, size = [], [], 0
1084+
# Each batch is one request, so what it carries is held under both the
1085+
# usual size and what one request may hold.
1086+
room = min(MAX_BATCH_CHARS, asker.input_room(REVIEW_SYSTEM) - 500)
9561087
for ref, text in zip(asked, evidence):
1088+
if len(text) > room:
1089+
text = text[:room - 200] + "\n</evidence>"
9571090
if current_batch and (len(current_batch) >= asker.batch_size
958-
or size + len(text) > MAX_BATCH_CHARS):
1091+
or size + len(text) > room):
9591092
batches.append(current_batch)
9601093
current_batch, size = [], 0
9611094
current_batch.append((ref, text))
@@ -1000,14 +1133,17 @@ def progress():
10001133
finding_ids=group["finding_ids"], group_ids=group["all_ids"],
10011134
known_exploited=group["known_exploited"],
10021135
kev_floor=group["kev_floor"],
1003-
waived=group["waived"], waiver=group["waiver"])
1136+
waived=group["waived"], waiver=group["waiver"],
1137+
code=group.get("display_code", ""), cwe=group["cwe"] or "")
10041138
items.append(item)
10051139
for concern in overview["concerns"]:
10061140
concern.update(source="review", rule="", kind="review", count=0,
10071141
severity="", finding_ids=[], group_ids=[],
10081142
location=(f"{concern['file']}, line {concern['line']}"
10091143
if concern["file"] and concern["line"]
1010-
else concern["file"]))
1144+
else concern["file"]),
1145+
code=(_display_code(shown.get(concern["file"]) or [], [concern["line"]])
1146+
if concern["file"] and concern["line"] else ""))
10111147
items.append(concern)
10121148

10131149
for item in items:
@@ -1528,6 +1664,23 @@ def document_context(view_: dict) -> dict:
15281664
for r in RATINGS}
15291665
waived = {r: sum(1 for i in items if i["rating"] == r and i.get("waived"))
15301666
for r in RATINGS}
1667+
# The standards an issue falls under follow from its CWE, so a report
1668+
# written before they were shown gets them too. Worked out here, never
1669+
# stored, so a corrected table corrects every report.
1670+
for item in items:
1671+
std = report_standards.standards_for(item.get("cwe"))
1672+
item.setdefault("owasp", std["owasp"])
1673+
item.setdefault("api", std["api"])
1674+
item["owasp_name"], item["api_name"] = std["owasp_name"], std["api_name"]
1675+
item["cwe_label"] = std["cwe"]
1676+
item["references"] = std["references"]
1677+
item.setdefault("cvss", "")
1678+
item.setdefault("cvss_score", None)
1679+
item["cvss_band"] = report_standards.cvss_band(item.get("cvss_score"))
1680+
retests = [i for i in items if i.get("retest") and i["rating"] != "Informational"]
15311681
return {"r": result, "items": items, "tally": tally, "waived_tally": waived,
15321682
"matrix": matrix(items), "impact_words": IMPACT_WORDS,
1533-
"likelihood_words": LIKELIHOOD_WORDS, "row": view_["row"]}
1683+
"likelihood_words": LIKELIHOOD_WORDS, "row": view_["row"],
1684+
"verdict": report_standards.verdict(items),
1685+
"coverage": report_standards.coverage(items),
1686+
"retests": retests, "standards": report_standards.STANDARDS}

0 commit comments

Comments
 (0)