Skip to content

Commit 0a3b2a1

Browse files
committed
Forgotten guards in Express, Python and Spring, an API's callers, and records returned whole
Routes beside a guarded one. Rules for Express, FastAPI and Flask, and Spring mark every route handler with no guard semgrep can see. Triage reads the rest: a guard on the route (middleware, Depends on a sign in check, @login_required, @PreAuthorize, an inline middleware that calls one), one over the whole router or app (router.use, router dependencies, include_router, an auth middleware, a before_request that refuses, anyRequest().authenticated()), or a path public by design. What is left is kept for a person only when another route in the same file is guarded and it is not; a file of routes none of which ask for a sign in is advisory. Comments are blanked before a route's arguments are read, so an apostrophe in a handler cannot hide its guard, and a security headers or session middleware is never taken for a sign in check. Checked on 24 repositories here: 763 handlers cleared, 167 advisory, and the rest kept were routers that mix guarded and unguarded routes. On this product's own code, every route but one is cleared; the one is a route that checks the session itself on purpose. An API's callers. The browser scan of a site now reads, from its scripts, the calls its pages make to other sites on the list: the API base in the app's configuration and the names joined to it, with parameter names kept and values dropped. A scan of one of those sites starts from them, and counts them as routes its code declares. Hosts not on the list are never kept. Records returned whole. The reasoning pass asks one small question about each handler that looks a record up from the request and returns it as it is, with the class it returns beside it: whether it hands over other people's contact details or keys, staff fields or an IP address. Found in code for nothing, C# and Java included, at most 15 a repository, each cached against the code and filed only when the second look agrees. A new data-exposure category, and a switch on the AI page, on by default.
1 parent f7529ab commit 0a3b2a1

9 files changed

Lines changed: 981 additions & 3 deletions

File tree

‎app/core/ai.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,10 @@
127127
"Most handlers not read before that are asked "
128128
"about in one repository per scan, the likeliest "
129129
"first. The rest wait for the next scan."),
130+
"ai.code_reasoning_exposure": (True, False,
131+
"Also ask about handlers that look a record up from "
132+
"the request and return it whole, C# and Java "
133+
"included. At most 15 a repository, each once."),
130134
# What the custom service charges, per million tokens, for the cost page.
131135
# Kept as text so 0.10 stays 0.10. Empty is not counted.
132136
"ai.price_input": ("", False, "Price per million input tokens."),
@@ -218,6 +222,7 @@ def read_form(form, text, number, url, checked) -> dict:
218222
"Monthly token limit", 0, 2_000_000_000),
219223
"ai.code_reasoning_per_repo": number(form.get("ai.code_reasoning_per_repo") or 25,
220224
"Handlers per repository", 1, 500),
225+
"ai.code_reasoning_exposure": checked("ai.code_reasoning_exposure"),
221226
}
222227

223228

‎app/core/code_reasoning.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@
7373
"confirming it"),
7474
"workflow": ("CWE-841", "A step the process requires is skipped, or a state captured earlier "
7575
"is acted on after it changed"),
76+
"data-exposure": ("CWE-213", "A response hands over more of a record than its caller should "
77+
"have"),
7678
}
7779
SEVERITIES = {"high": Severity.HIGH, "medium": Severity.MEDIUM, "low": Severity.LOW}
7880

@@ -1085,6 +1087,11 @@ def reason(session, cfg: dict, repository: str, root: Path, limit: int | None =
10851087
gap_asked, gap_cached = _sibling_gaps(session, cfg, repository, root, run, found)
10861088
asked += gap_asked
10871089
cached += gap_cached
1090+
# Handlers that look a record up from the request and return it whole.
1091+
# Found in code for nothing; only those are asked about, each once.
1092+
exp_asked, exp_cached = _record_exposure(session, cfg, repository, root, run, found)
1093+
asked += exp_asked
1094+
cached += exp_cached
10881095
session.flush()
10891096
return {"units": len(units), "cached": cached, "asked": asked, "calls": run.calls,
10901097
"tokens": run.tokens, "found": found, "stopped": run.stopped,
@@ -1155,6 +1162,65 @@ def _sibling_gaps(session, cfg: dict, repository: str, root: Path, run, found: l
11551162
return asked, cached
11561163

11571164

1165+
def _record_exposure(session, cfg: dict, repository: str, root: Path, run, found: list) -> tuple:
1166+
"""Ask about handlers that return a looked up record as it is. -> (asked, cached).
1167+
1168+
The same rules as the sibling question: cached against a digest of the
1169+
handler and the class it returns, filed only when the second look calls
1170+
it real. C# and Java handlers are read here even though the main pass
1171+
does not read those languages.
1172+
"""
1173+
from app.core import record_exposure as rx
1174+
1175+
if cfg.get("ai.code_reasoning_exposure") is False:
1176+
return 0, 0
1177+
try:
1178+
cap = int(cfg.get("ai.code_reasoning_exposure_max") or rx.MAX_CANDIDATES)
1179+
cands = rx.candidates(root, limit=cap)
1180+
except Exception: # noqa: BLE001 - never fail the pass
1181+
log.warning("could not look for returned records in %s", repository, exc_info=True)
1182+
return 0, 0
1183+
if not cands:
1184+
return 0, 0
1185+
digests = {id(c): rx.digest_of(c) for c in cands}
1186+
known = {}
1187+
with session.no_autoflush:
1188+
rows = (session.query(CodeReasoningCache)
1189+
.filter(CodeReasoningCache.digest.in_(list(digests.values())))
1190+
.order_by(CodeReasoningCache.id).all())
1191+
for row in rows:
1192+
if row.repository == repository or row.digest not in known:
1193+
known[row.digest] = row
1194+
asked = cached = 0
1195+
for cand in cands:
1196+
unit, digest = cand["unit"], digests[id(cand)]
1197+
row = known.get(digest)
1198+
items = rx.cached_items(row, unit)
1199+
if items is not None:
1200+
cached += 1
1201+
if row.repository != repository:
1202+
_remember_digest(session, repository, digest, rx.remember_key(cand), row.result)
1203+
found += [(unit, item) for item in items]
1204+
continue
1205+
if run.stopped:
1206+
break
1207+
item = rx.ask(run, cand, mask, _block, CATEGORIES, SEVERITIES)
1208+
if item is None:
1209+
continue
1210+
asked += 1
1211+
kept = []
1212+
if not item.get("none") and run.confirm(unit, item, root)["v"] == "real":
1213+
written = None if run.stopped else run.writeup(unit, item)
1214+
if written:
1215+
item["d"] = written
1216+
kept = [item]
1217+
if run.stopped and not kept and not item.get("none"):
1218+
continue
1219+
_remember_digest(session, repository, digest, rx.remember_key(cand), rx.result_of(kept, unit))
1220+
found += [(unit, i) for i in kept]
1221+
return asked, cached
1222+
1223+
11581224
def _remember_digest(session, repository: str, digest: str, label: str, result: str) -> None:
11591225
with session.no_autoflush:
11601226
row = (session.query(CodeReasoningCache)

‎app/core/dom_scan.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,3 +85,80 @@ def scan(collected: dict, base_url: str) -> list:
8585
fix="Do not deploy source maps to production, or restrict access to them.",
8686
cwe="CWE-540"))
8787
return findings
88+
89+
90+
# ------------------------------------------- the API calls a page's code makes
91+
92+
# A single page app keeps where its API lives in a configuration object, and
93+
# builds each call by adding a name to it. Minifying renames the objects but
94+
# keeps the property names, so these survive a production build.
95+
_BASE_KEYS = r"(?:base_?url|api_?url|api_?base(?:_?url)?|api_?endpoint|api_?root|service_?url|backend_?url|api_?host)"
96+
_BASE_VALUE = re.compile(r"\b" + _BASE_KEYS + r"""\s*[:=]\s*["'`](https?://[^"'`\s]{4,200})["'`]""", re.I)
97+
_JOINED = re.compile(
98+
r"\b" + _BASE_KEYS + r"""\s*\}?\s*\+\s*["'`]([^"'`\s]{2,160})["'`]|"""
99+
r"\$\{[\w.$]*?" + _BASE_KEYS + r"""\}([^`$\s]{2,160})""", re.I)
100+
_ABSOLUTE = re.compile(r"""["'`](https?://([A-Za-z0-9.\-]+)(/[A-Za-z0-9_\-./{}%]*)?(?:\?[^"'`\s]*)?)["'`]""")
101+
MAX_CALLS = 120
102+
103+
104+
def _clean_call(path: str) -> str:
105+
"""A path to ask for: values dropped, parameter names kept with an empty value."""
106+
from urllib.parse import parse_qsl, urlsplit
107+
108+
parts = urlsplit(path)
109+
base = re.sub(r"\{[^}/]*\}|\$\{[^}]*\}", "1", parts.path or "/")
110+
base = re.sub(r"/{2,}", "/", base if base.startswith("/") else "/" + base)
111+
if not re.fullmatch(r"/[A-Za-z0-9._~\-/%]*", base):
112+
return ""
113+
names = [k for k, _v in parse_qsl(parts.query, keep_blank_values=True)
114+
if re.fullmatch(r"[A-Za-z_][\w\-]{0,40}", k)]
115+
query = "&".join(f"{k}=" for k in dict.fromkeys(names))
116+
return base + (f"?{query}" if query else "")
117+
118+
119+
from app.core.dast_paths import STATIC # noqa: E402 - pictures and styles are not calls
120+
121+
122+
def api_calls(collected: dict, site_host: str, hosts) -> dict:
123+
"""The calls the page's code makes to other sites on the list.
124+
125+
hosts are the host names of the sites being scanned; a call to anywhere
126+
else is never kept, so this can only point a scan at a site already in
127+
scope. -> {host: [path, ...]}"""
128+
from urllib.parse import urljoin, urlsplit
129+
130+
hosts = {h.lower() for h in hosts or ()} - {(site_host or "").lower()}
131+
if not hosts:
132+
return {}
133+
texts = [s.get("text") or "" for s in collected.get("scripts") or []]
134+
texts += [t or "" for t in collected.get("inline") or []]
135+
out: dict = {}
136+
137+
def keep(url: str):
138+
parts = urlsplit(url)
139+
host = (parts.hostname or "").lower()
140+
if host not in hosts:
141+
return
142+
path = _clean_call(parts.path + (f"?{parts.query}" if parts.query else ""))
143+
if path and STATIC.search(path.split("?")[0]):
144+
return
145+
if path and path != "/":
146+
bucket = out.setdefault(host, [])
147+
if path not in bucket and sum(len(v) for v in out.values()) < MAX_CALLS:
148+
bucket.append(path)
149+
150+
for text in texts:
151+
bases = [m.group(1) for m in _BASE_VALUE.finditer(text)]
152+
bases = [b for b in bases if (urlsplit(b).hostname or "").lower() in hosts]
153+
for m in _ABSOLUTE.finditer(text):
154+
if (m.group(2) or "").lower() in hosts and m.group(3) and m.group(3) != "/":
155+
keep(m.group(1))
156+
if not bases:
157+
continue
158+
for m in _JOINED.finditer(text):
159+
fragment = m.group(1) or m.group(2) or ""
160+
if fragment.startswith(("http:", "https:", "//")) or not re.match(r"[A-Za-z0-9_/]", fragment):
161+
continue
162+
for base in bases:
163+
keep(urljoin(base if base.endswith("/") else base + "/", fragment.lstrip("/")))
164+
return out

‎app/core/external.py‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2615,7 +2615,37 @@ def _start_scan_proxy(session, host, upstream, workdir):
26152615
workdir=workdir, auth_hosts=[host])
26162616

26172617

2618-
def _browser_dom(scan_id, url, opts, proxy) -> tuple:
2618+
def _keep_client_calls(asset_id: int, url: str, collected: dict) -> None:
2619+
"""Keep the calls this site's pages make to other sites on the list."""
2620+
from urllib.parse import urlsplit
2621+
2622+
from app.core import dom_scan
2623+
2624+
with session_scope() as db:
2625+
hosts = [h for (h,) in db.query(ExternalAsset.host).all()]
2626+
calls = dom_scan.api_calls(collected, urlsplit(url).hostname or "", hosts)
2627+
asset = db.get(ExternalAsset, asset_id)
2628+
if asset is not None:
2629+
asset.client_calls = json.dumps({"at": utcnow().isoformat(), "calls": calls}) if calls else ""
2630+
2631+
2632+
def client_calls_for(db, host: str) -> dict:
2633+
"""The paths other sites' pages call on this host. -> {source host: [path]}"""
2634+
out = {}
2635+
host = (host or "").lower()
2636+
for src, raw in (db.query(ExternalAsset.host, ExternalAsset.client_calls)
2637+
.filter(ExternalAsset.client_calls.like(f"%{host}%")).all()):
2638+
try:
2639+
calls = (json.loads(raw or "{}").get("calls") or {}).get(host) or []
2640+
except (ValueError, AttributeError):
2641+
continue
2642+
paths = [p for p in calls if isinstance(p, str) and p.startswith("/")]
2643+
if paths:
2644+
out[src] = paths[:120]
2645+
return out
2646+
2647+
2648+
def _browser_dom(scan_id, url, opts, proxy, asset_id=None) -> tuple:
26192649
"""Load the page in a real browser, run its scripts, and read the JavaScript
26202650
it loaded. -> (findings, an engines summary row). Findings are a secret
26212651
hardcoded in client code or source exposed through a source map; both are
@@ -2629,6 +2659,11 @@ def _browser_dom(scan_id, url, opts, proxy) -> tuple:
26292659
if not collected.get("ok"):
26302660
return [], {"tool": "dom", "name": name, "status": "skipped", "seconds": 0,
26312661
"detail": collected.get("error") or "the browser gave no answer"}
2662+
if asset_id is not None:
2663+
try:
2664+
_keep_client_calls(asset_id, url, collected)
2665+
except Exception: # noqa: BLE001 - the scan stands without it
2666+
log.exception("scan %s: the page's API calls were not kept", scan_id)
26322667
try:
26332668
findings = dom_scan.scan(collected, url)
26342669
except Exception: # noqa: BLE001 - the rest of the scan stands without it
@@ -2929,6 +2964,26 @@ def _builtin_run(scan_id, host, asset_id, profile, opts, auth, proxy) -> None:
29292964
engines[-1]["lost"] = lost["repos"]
29302965
except Exception: # noqa: BLE001 - the scan stands without the comparison
29312966
log.exception("scan %s: the code reading was not compared", scan_id)
2967+
# The calls other sites' pages make to this one, read from their scripts
2968+
# by their own scans: an API behind a portal is reached through them.
2969+
called = []
2970+
if url and plan:
2971+
with session_scope() as db:
2972+
from_pages = client_calls_for(db, host)
2973+
if from_pages:
2974+
for paths in from_pages.values():
2975+
for p in paths:
2976+
full = url.rstrip("/") + p
2977+
if full not in called:
2978+
called.append(full)
2979+
bare = p.split("?")[0]
2980+
if bare not in wb["routes"]:
2981+
wb["routes"].append(bare)
2982+
n = len(called)
2983+
engines.append({"tool": "client-calls", "name": "Calls from other sites' pages",
2984+
"status": "ok", "seconds": 0,
2985+
"detail": f"{n} path{'s' if n != 1 else ''} this site is called on "
2986+
"by the code of " + ", ".join(sorted(from_pages))[:300]})
29322987
# Signing in, for the engines. Baseline and testssl stay signed out:
29332988
# one reads the public posture, the other only talks TLS.
29342989
session, secrets = {}, []
@@ -2967,9 +3022,11 @@ def _builtin_run(scan_id, host, asset_id, profile, opts, auth, proxy) -> None:
29673022
_update(scan_id, engines=json.dumps(engines))
29683023
engine_proxy = scan_proxy.url if scan_proxy else proxy
29693024
try:
2970-
# Links on the front page, then the routes from the code.
3025+
# Links on the front page, then the routes from the code, then
3026+
# what other sites' pages call here.
29713027
extra = dast_engines.links_in(url, base["telemetry"].get("body_sample") or "")
29723028
extra += [r for r in routes if r not in extra]
3029+
extra += [c for c in called if c not in extra]
29733030
timeouts = engine_timeouts_for(asset_id)
29743031
for tool in plan:
29753032
if scan_id in _stopping:
@@ -3031,7 +3088,7 @@ def _builtin_run(scan_id, host, asset_id, profile, opts, auth, proxy) -> None:
30313088
if url and profile in ("thorough", "premium") and external_probe.playwright_python() \
30323089
and scan_id not in _stopping:
30333090
_update(scan_id, step="reading the page in a browser")
3034-
dom_findings, dom_row = _browser_dom(scan_id, url, opts, proxy)
3091+
dom_findings, dom_row = _browser_dom(scan_id, url, opts, proxy, asset_id)
30353092
findings += dom_findings
30363093
engines.append(dom_row)
30373094
_update(scan_id, engines=json.dumps(engines))

‎app/core/models.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2411,6 +2411,10 @@ class ExternalAsset(Base):
24112411
# routes from that the newest scan did not. JSON written at the end of
24122412
# each scan, "" when nothing was lost, NULL before it was ever worked out.
24132413
code_lost = Column(Text)
2414+
# The calls this site's own pages make to other sites on the list, read
2415+
# from its scripts by the browser scan: {"at", "calls": {host: [path]}}.
2416+
# A scan of one of those sites starts from them.
2417+
client_calls = Column(Text, default="")
24142418

24152419
# Who answers for the site, as addresses, comma separated.
24162420
executive_owner = Column(String(500), default="")

0 commit comments

Comments
 (0)