Skip to content

Commit 6e1357f

Browse files
committed
Security report: one row for repeated false alarms, and fewer of them
A report on this repository listed one Django CSRF rule nineteen times, once per template, and still carried false positives the code disproved. Closed issues with the same rule and the same verdict are now one row that names every file and covers every finding. A real issue is never folded in. A rule named for a framework, such as django-no-csrf-token, is now set aside in code when the repository does not use that framework, instead of being asked about once per file. A framework counts as used when a build file names it or the code imports it, so a missing requirements file never hides a real finding. Test fixtures and tool folders are not read for this. The policy reader no longer takes a sentence about a Content-Security-Policy for a policy: every source in a directive has to look like one. The review is told a protection can sit in the code that calls the flagged code, and to ask for it. A search that lands on a definition now brings the whole definition, so the check its body makes is in view. The first admin docstring said the password goes to the log. It goes only to a file at mode 600, which the log names, and the docstring now says so. The credential import warning logs the error type, not its text, since a database error can echo the values it was handed.
1 parent 4a758df commit 6e1357f

4 files changed

Lines changed: 176 additions & 4 deletions

File tree

‎app/auth/service.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -403,8 +403,10 @@ def cleanup_expired(session) -> int:
403403
def ensure_bootstrap_admin(session) -> dict:
404404
"""Make an admin on first boot, once.
405405
406-
The password is printed to the log and never stored anywhere else. They
407-
have to change it and set up TOTP before the app lets them do anything.
406+
The password goes only to first-admin.txt in the data directory, mode 600,
407+
which is deleted the moment the account changes it. The log names the
408+
file, never the password. They have to change it and set up TOTP before
409+
the app lets them do anything.
408410
"""
409411
if not live.auth_config().local_enabled:
410412
return {}

‎app/core/assessment.py‎

Lines changed: 89 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -957,7 +957,12 @@ def _snippet(root: Path, group: dict, context: int) -> str:
957957
it, the column type, the config that turns a feature on) is missing, the \
958958
verdict is needs_review and need names what to fetch. The code you ask for \
959959
is searched for and sent back, and a search that finds nothing is itself \
960-
evidence that the thing is absent. Use the product profile, and run these checks:
960+
evidence that the thing is absent. A protection can sit in the code that \
961+
calls the flagged code: a script another module starts, a function whose \
962+
caller checks, proxies or pins first. Before confirming that a check is \
963+
missing, look at the caller; when it was not shown, use needs_review and \
964+
ask in need for where the flagged script or function is called, by its file \
965+
or function name. Use the product profile, and run these checks:
961966
{REVIEW_CHECKS}
962967
963968
{STYLE}
@@ -1547,6 +1552,53 @@ def _review(asker: _Asker, batch: list, progress, head: str = "") -> dict:
15471552
return out
15481553

15491554

1555+
def _merge_closed(items: list) -> list:
1556+
"""Closed scanner issues with the same rule and the same verdict, as one.
1557+
1558+
A real issue is never merged: each one keeps its own row, rating and
1559+
fix. Only what carries no risk, closed for the same reason by the same
1560+
rule, is folded together, with every file it was in and every finding
1561+
it covers, so a false positive correction still reaches all of them.
1562+
"""
1563+
out, kept = [], {}
1564+
for item in items:
1565+
key = None
1566+
if (item.get("source") == "scanner" and item.get("verdict") in CLOSED
1567+
and not item.get("decided") and item.get("rule")
1568+
and item.get("rating") == "Informational"):
1569+
key = (item["rule"], item["verdict"])
1570+
if key is None or key not in kept:
1571+
if key is not None:
1572+
item["files_covered"] = [item["file"]] if item.get("file") else []
1573+
kept[key] = item
1574+
out.append(item)
1575+
continue
1576+
first = kept[key]
1577+
first["count"] = (first.get("count") or 0) + (item.get("count") or 0)
1578+
first["group_ids"] = (first.get("group_ids") or []) + (item.get("group_ids") or [])
1579+
first["finding_ids"] = ((first.get("finding_ids") or [])
1580+
+ (item.get("finding_ids") or []))[:12]
1581+
if item.get("file") and item["file"] not in first["files_covered"]:
1582+
first["files_covered"].append(item["file"])
1583+
for item in kept.values():
1584+
files = item.pop("files_covered")
1585+
if len(files) > 1:
1586+
shown = ", ".join(files[:6]) + (f" and {len(files) - 6} more"
1587+
if len(files) > 6 else "")
1588+
item["location"] = f"{len(files)} files: {shown}"
1589+
base = (f"{item['rule']} does not fit this code" if item.get("settled") == "check 8"
1590+
else item["title"])
1591+
item["title"] = short_title(f"{base} ({len(files)} files)", base)
1592+
# The code beside one file does not stand for the rest.
1593+
item["code"] = ""
1594+
item.pop("fix_before", None)
1595+
item.pop("fix_after", None)
1596+
item["evidence"] = tidy((item.get("evidence") or "") + f" The same rule was "
1597+
f"closed the same way in {len(files)} files, "
1598+
f"{item['count']} alerts in all.", 900)
1599+
return out
1600+
1601+
15501602
def _pack(texts: list, room: int, size: int) -> list:
15511603
"""[(ref, text)] into batches, each one request: at most size groups and
15521604
room characters, a group too big on its own cut to fit."""
@@ -1588,6 +1640,33 @@ def _pack(texts: list, room: int, size: int) -> list:
15881640
"""
15891641

15901642

1643+
LOOKUP_DEF_LINES = 90 # lines of a definition a hit on it brings
1644+
_DEFINITION = re.compile(r"^(\s*)(?:async\s+def|def|class|(?:export\s+)?(?:async\s+)?function)\s+\w")
1645+
1646+
1647+
def _block_end(lines: list, at: int) -> int:
1648+
"""The last line of the definition that starts at line at (1 based), or 0."""
1649+
head = lines[at - 1] if 0 < at <= len(lines) else ""
1650+
m = _DEFINITION.match(head)
1651+
if not m:
1652+
return 0
1653+
indent, end = len(m.group(1)), at
1654+
braces = "{" in head and not head.rstrip().endswith(":")
1655+
depth = head.count("{") - head.count("}")
1656+
for n in range(at + 1, len(lines) + 1):
1657+
line = lines[n - 1]
1658+
if braces:
1659+
depth += line.count("{") - line.count("}")
1660+
end = n
1661+
if depth <= 0:
1662+
break
1663+
continue
1664+
if line.strip() and len(line) - len(line.lstrip()) <= indent:
1665+
break
1666+
end = n
1667+
return end
1668+
1669+
15911670
def _grep_files(root: Path, find: str) -> list:
15921671
"""Files in the branch holding find, as plain text. Vendored copies last."""
15931672
try:
@@ -1624,6 +1703,11 @@ def _lookup(root: Path, need: list) -> str:
16241703
spans = []
16251704
for hit in hits:
16261705
lo, hi = max(1, hit - LOOKUP_CONTEXT), min(len(lines), hit + LOOKUP_CONTEXT)
1706+
# A hit on a definition brings the whole of it, up to a limit,
1707+
# so the check its body makes is in view.
1708+
end = _block_end(lines, hit)
1709+
if end:
1710+
hi = max(hi, min(end, hit + LOOKUP_DEF_LINES))
16271711
if spans and lo <= spans[-1][1] + 1:
16281712
spans[-1][1] = max(spans[-1][1], hi)
16291713
else:
@@ -2156,6 +2240,10 @@ def progress():
21562240
item["rating"] = rating_for(item["score"])
21572241
_floor(item)
21582242

2243+
# Closed issues that are the same reason over and over (one rule that
2244+
# does not fit, flagged in forty templates) are one row, not forty.
2245+
items = _merge_closed(items)
2246+
21592247
# A waiver should be challenged when the problem is real, it carries a
21602248
# rating, and its reason names no control, states something the code
21612249
# shows is not so, or is about a different problem.

‎app/core/report_profile.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,10 @@ def build(root, paths=None) -> dict:
148148
for f in marks["files"])
149149
or any(w in build_text for w in marks["words"])):
150150
frameworks.append(name)
151+
# A framework the code imports is used, whether or not a build file names
152+
# it, so a rule for it is never set aside on a missing requirements file.
153+
seen_imports = _imported_frameworks(root, paths)
154+
frameworks += [f for f in FRAMEWORKS if f in seen_imports and f not in frameworks]
151155

152156
dockerfiles = []
153157
for rel in sorted(files):
@@ -232,6 +236,27 @@ def build(root, paths=None) -> dict:
232236
"has_github_workflows": any(p.startswith(".github/workflows/") for p in files)}
233237

234238

239+
# One source in a CSP directive: a quoted keyword ('self', 'nonce-...'), a
240+
# scheme (https:, data:), a host with an optional scheme, port and path, a
241+
# wildcard, or a template placeholder standing in for one.
242+
_SOURCE = re.compile(
243+
r"^(?:'[^']+'|[a-z][a-z0-9+.-]*:(?://\S*)?|\*(?:\.[\w.-]+)?(?::\S+)?|"
244+
r"(?:[a-z][a-z0-9+.-]*://)?[\w*-]+(?:\.[\w-]+)+(?::\d+|:\*)?(?:/\S*)?|"
245+
r"[{$][^\s]*|%\S+)$", re.IGNORECASE)
246+
247+
248+
def _looks_like_sources(sources: str) -> bool:
249+
"""Whether a directive's text is a list of sources rather than a sentence.
250+
251+
A docstring that says "script-src at all. 'unsafe-inline' under style-src"
252+
is about a policy, and must not be read as one.
253+
"""
254+
words = [w.strip(",\\") for w in sources.replace("\\n", " ").split()]
255+
words = [w for w in words if w and w not in ("+", ".", "\"", "'")]
256+
return bool(words) and all(_SOURCE.match(w.rstrip("\"'.")) or _SOURCE.match(w)
257+
for w in words)
258+
259+
235260
def _inline_script(text: str) -> tuple:
236261
"""Does a policy in this file let inline script run. -> (yes, line, nonce)
237262
@@ -250,6 +275,8 @@ def _inline_script(text: str) -> tuple:
250275
continue
251276
for name, sources in _DIRECTIVE.findall(line):
252277
name, low = name.lower(), sources.lower()
278+
if not _looks_like_sources(low):
279+
continue # prose about a policy, not a policy
253280
if name == "default-src" and has_script_src:
254281
continue
255282
keyed = any(k in low for k in ("'nonce-", "'sha256-", "'sha384-",
@@ -304,6 +331,54 @@ def as_text(profile: dict, fork_of: str = "", code: bool = True) -> str:
304331
return "\n".join(lines)
305332

306333

334+
# How code shows it uses a framework, read from the first lines of source
335+
# files. Only imports: a word in a comment or a string proves nothing.
336+
_IMPORTS = {
337+
"django": re.compile(r"^\s*(?:from|import)\s+django\b", re.M),
338+
"flask": re.compile(r"^\s*(?:from|import)\s+flask\b", re.M),
339+
"fastapi": re.compile(r"^\s*(?:from|import)\s+fastapi\b", re.M),
340+
"rails": re.compile(r"^\s*require\s+['\"]rails", re.M),
341+
"laravel": re.compile(r"^\s*use\s+Illuminate\\", re.M),
342+
"symfony": re.compile(r"^\s*use\s+Symfony\\", re.M),
343+
"spring": re.compile(r"^\s*import\s+org\.springframework\.", re.M),
344+
"express": re.compile(r"""(?:require\(\s*|from\s+)['"]express['"]"""),
345+
"react": re.compile(r"""(?:require\(\s*|from\s+)['"]react(?:-dom)?['"]"""),
346+
"vue": re.compile(r"""(?:require\(\s*|from\s+)['"]vue['"]"""),
347+
"angular": re.compile(r"""from\s+['"]@angular/core['"]"""),
348+
}
349+
_IMPORT_EXTS = {".py", ".rb", ".php", ".java", ".kt", ".js", ".mjs", ".cjs", ".jsx",
350+
".ts", ".tsx", ".vue"}
351+
MAX_IMPORT_FILES = 4000
352+
353+
354+
def _imported_frameworks(root: Path, paths: list) -> set:
355+
found, read = set(), 0
356+
for rel in paths:
357+
if read >= MAX_IMPORT_FILES or len(found) == len(_IMPORTS):
358+
break
359+
if _skip(rel) or Path(rel).suffix.lower() not in _IMPORT_EXTS:
360+
continue
361+
# A fixture that imports a framework to test a rule against it, or a
362+
# tool folder, says nothing about what the product is built on.
363+
bits = rel.split("/")
364+
if any(b.startswith(".") or b in ("fixtures", "testdata", "__fixtures__")
365+
for b in bits[:-1]):
366+
continue
367+
read += 1
368+
head = _read(root / rel)[:20_000]
369+
for name, pattern in _IMPORTS.items():
370+
if name not in found and pattern.search(head):
371+
found.add(name)
372+
return found
373+
374+
375+
# Rule ids written as framework-rule-name, with no pack prefix, such as
376+
# django-no-csrf-token. Only framework names that are not also ordinary
377+
# words a rule could start with.
378+
_PREFIX_FRAMEWORKS = ("django", "flask", "rails", "laravel", "symfony", "spring",
379+
"angular", "express", "react", "vue")
380+
381+
307382
def rule_mismatch(rule: str, profile: dict) -> str:
308383
"""Why a rule cannot apply to this repository, or "".
309384
@@ -316,6 +391,11 @@ def rule_mismatch(rule: str, profile: dict) -> str:
316391
return ""
317392
parts = rule.lower().split(".")
318393
if len(parts) < 3:
394+
head = re.split(r"[-_]", parts[-1], maxsplit=1)[0]
395+
if (head in _PREFIX_FRAMEWORKS and head in FRAMEWORKS
396+
and head not in profile.get("frameworks", [])):
397+
return (f"The rule is written for the {head} framework, which this "
398+
"repository does not use.")
319399
return ""
320400
language, framework = parts[0], parts[1]
321401
needs = RULE_LANGUAGES.get(language)

‎app/main.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,9 @@ def _startup() -> None:
147147
if connection_store.adopt_settings(db):
148148
log.info("made the first connection from the existing settings")
149149
except Exception as exc:
150-
log.warning("could not adopt the existing credential: %s", exc)
150+
# The type only. A database error can echo the values it was
151+
# given, and one of them here is the credential.
152+
log.warning("could not adopt the existing credential: %s", type(exc).__name__)
151153

152154
# Dynamic scans finished before not found answers, repeats and the
153155
# model's likely false positives were dealt with. Each scan once, in

0 commit comments

Comments
 (0)