Skip to content

Commit dbf3a0d

Browse files
committed
Run the scan engines through a local sign in proxy, on by default
mitmproxy already ships in the dynamic scanning environment. Each scan now starts a local instance the stock engines run through, with an addon that does two things to every request they make: it stamps the site's session on it, so every engine sees what a signed in user sees rather than only the engines that sign in well on their own, and it refuses any host the scan is not scoped to, so an engine that follows a link or a redirect off the target cannot reach it. It is on by default and needs no setup: when mitmproxy is present the scan uses it, and when it is not the engines go direct exactly as before, so nothing breaks. testssl always reads the site's own TLS directly, not through the proxy. A setting on the DAST admin page turns it off. Verified on a live site: an in scope request is forwarded with the session added, an out of scope host is refused, and the engines fall back cleanly when the proxy is not there.
1 parent 8973309 commit dbf3a0d

6 files changed

Lines changed: 204 additions & 1 deletion

File tree

‎app/core/dast_checks/mitm_addon.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
"""A mitmproxy addon for one scan.
2+
3+
The stock engines (ffuf, nikto, nuclei, wapiti, sqlmap, dalfox) are each pointed
4+
at a local mitmproxy for the run. This addon then does two things to every
5+
request they make:
6+
7+
stamps the signed in session on it, so every engine sees what a signed in
8+
user sees, not just the engines that sign in well on their own
9+
10+
refuses any host the scan is not scoped to, so an engine that follows a
11+
link or a redirect off the target cannot touch anything but the sites the
12+
scan is allowed to reach
13+
14+
It runs inside the DAST virtual environment's python, which does not have the
15+
app on its path, so it imports only mitmproxy and the standard library, and
16+
takes its settings from the environment the parent set.
17+
"""
18+
19+
import json
20+
import os
21+
22+
from mitmproxy import http
23+
24+
_AUTH = json.loads(os.environ.get("GCR_MITM_AUTH", "{}") or "{}")
25+
_HOSTS = [h.lower().strip().rstrip(".") for h in
26+
json.loads(os.environ.get("GCR_MITM_HOSTS", "[]") or "[]") if h]
27+
_ENFORCE = os.environ.get("GCR_MITM_SCOPE", "1") == "1"
28+
29+
30+
def _in_scope(host: str) -> bool:
31+
host = (host or "").lower().rstrip(".")
32+
if not host:
33+
return False
34+
for allowed in _HOSTS:
35+
if host == allowed or host.endswith("." + allowed):
36+
return True
37+
return False
38+
39+
40+
def request(flow: http.HTTPFlow) -> None:
41+
host = flow.request.pretty_host
42+
if _ENFORCE and _HOSTS and not _in_scope(host):
43+
flow.response = http.Response.make(
44+
403, b"blocked: out of the scan's scope\n",
45+
{"Content-Type": "text/plain"})
46+
return
47+
for name, value in _AUTH.items():
48+
flow.request.headers[str(name)] = str(value)

‎app/core/dast_proxy.py‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
"""A per scan local proxy, so every engine signs in and stays in scope.
2+
3+
mitmproxy ships in the DAST virtual environment already (wapiti brings it). For
4+
a scan it is started on a local port with the addon in dast_checks/mitm_addon.py,
5+
the stock engines are pointed at it, and it stamps the session on their requests
6+
and refuses any host out of scope. If mitmproxy is not there the scan runs
7+
exactly as before: start() returns None and the engines go direct.
8+
"""
9+
10+
from __future__ import annotations
11+
12+
import json
13+
import logging
14+
import os
15+
import shutil
16+
import socket
17+
import subprocess
18+
import time
19+
from pathlib import Path
20+
21+
log = logging.getLogger("gcr.dast_proxy")
22+
23+
_ADDON = Path(__file__).resolve().parent / "dast_checks" / "mitm_addon.py"
24+
25+
26+
def mitmdump() -> str:
27+
"""The mitmdump to run, or empty when it is not installed."""
28+
candidates = []
29+
dast_py = os.environ.get("GCR_DAST_PYTHON", "").strip()
30+
if dast_py:
31+
candidates.append(str(Path(dast_py).with_name("mitmdump")))
32+
candidates += ["/opt/gcr/dast-venv/bin/mitmdump", shutil.which("mitmdump") or ""]
33+
for cand in candidates:
34+
if cand and os.access(cand, os.X_OK):
35+
return cand
36+
return ""
37+
38+
39+
def _free_port() -> int:
40+
s = socket.socket()
41+
try:
42+
s.bind(("127.0.0.1", 0))
43+
return s.getsockname()[1]
44+
finally:
45+
s.close()
46+
47+
48+
class Handle:
49+
"""A running scan proxy. url is what the engines are pointed at."""
50+
51+
def __init__(self, url: str, proc: subprocess.Popen):
52+
self.url = url
53+
self._proc = proc
54+
55+
def stop(self) -> None:
56+
try:
57+
self._proc.terminate()
58+
self._proc.wait(timeout=5)
59+
except Exception: # noqa: BLE001 - make sure it is gone
60+
try:
61+
self._proc.kill()
62+
except Exception: # noqa: BLE001
63+
pass
64+
65+
66+
def start(auth_headers: dict, hosts: list, upstream: str = "",
67+
workdir: Path = None) -> "Handle | None":
68+
"""Start the scan proxy. auth_headers is stamped on every request; hosts is
69+
the scope (company domains plus the target); upstream is a site proxy to
70+
chain through. -> a Handle, or None when mitmproxy is not available or did
71+
not come up, so the caller just runs the engines direct."""
72+
exe = mitmdump()
73+
if not exe or not _ADDON.is_file():
74+
return None
75+
port = _free_port()
76+
confdir = Path(workdir or "/tmp") / "mitm"
77+
confdir.mkdir(parents=True, exist_ok=True)
78+
argv = [exe, "--listen-host", "127.0.0.1", "--listen-port", str(port),
79+
"-s", str(_ADDON), "--ssl-insecure", "-q",
80+
"--set", "flow_detail=0", "--set", f"confdir={confdir}"]
81+
if upstream:
82+
argv += ["--mode", f"upstream:{upstream}"]
83+
env = dict(os.environ,
84+
GCR_MITM_AUTH=json.dumps(auth_headers or {}),
85+
GCR_MITM_HOSTS=json.dumps([h for h in (hosts or []) if h]),
86+
GCR_MITM_SCOPE="1")
87+
try:
88+
proc = subprocess.Popen(argv, env=env,
89+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
90+
except Exception as exc: # noqa: BLE001 - never let the proxy break a scan
91+
log.warning("scan proxy did not start: %s", exc)
92+
return None
93+
url = f"http://127.0.0.1:{port}"
94+
for _ in range(50): # up to ~10s for it to generate its CA and bind
95+
if proc.poll() is not None:
96+
log.warning("scan proxy exited before it was ready")
97+
return None
98+
try:
99+
with socket.create_connection(("127.0.0.1", port), timeout=0.3):
100+
return Handle(url, proc)
101+
except OSError:
102+
time.sleep(0.2)
103+
try:
104+
proc.terminate()
105+
except Exception: # noqa: BLE001
106+
pass
107+
log.warning("scan proxy did not come up in time")
108+
return None

‎app/core/external.py‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2079,6 +2079,25 @@ def _session_headers(sess: dict) -> dict:
20792079
return out
20802080

20812081

2082+
def _start_scan_proxy(session, host, upstream, workdir):
2083+
"""A local mitmproxy for the engine run, or None. It signs every engine's
2084+
requests in and refuses any host out of scope. Off in settings, not
2085+
installed, or failed to start -> None, and the engines run direct."""
2086+
from app.core import dast_proxy
2087+
from app.core import settings_store as s
2088+
from app.core.dast_checks import scope
2089+
2090+
with session_scope() as db:
2091+
on = s.get(db, "dast.scan_proxy")
2092+
on = True if on is None else bool(on)
2093+
domains = scope.company_domains(s.get(db, "web.company_domains") or [])
2094+
if not on:
2095+
return None
2096+
hosts = list(dict.fromkeys([host] + [d for d in domains if d]))
2097+
return dast_proxy.start(_session_headers(session), hosts, upstream=upstream or "",
2098+
workdir=workdir)
2099+
2100+
20822101
def _run_curated_rules(scan_id, asset_id, url, wb, session, opts, rulesets) -> tuple:
20832102
"""Run the enabled curated rules in the chosen rulesets against the site.
20842103
@@ -2233,6 +2252,13 @@ def _builtin_run(scan_id, host, asset_id, profile, opts, auth, proxy) -> None:
22332252
if url and plan:
22342253
workdir = app_settings.work_dir / "dast" / str(scan_id)
22352254
workdir.mkdir(parents=True, exist_ok=True)
2255+
scan_proxy = _start_scan_proxy(session, host, proxy, workdir)
2256+
if scan_proxy:
2257+
engines.append({"tool": "scan-proxy", "name": "Scan proxy", "status": "ok",
2258+
"seconds": 0, "detail": "the engines run through a local proxy "
2259+
"that signs each request in and refuses any host out of scope"})
2260+
_update(scan_id, engines=json.dumps(engines))
2261+
engine_proxy = scan_proxy.url if scan_proxy else proxy
22362262
try:
22372263
# Links on the front page, then the routes from the code.
22382264
extra = dast_engines.links_in(url, base["telemetry"].get("body_sample") or "")
@@ -2250,9 +2276,12 @@ def _builtin_run(scan_id, host, asset_id, profile, opts, auth, proxy) -> None:
22502276
_update(scan_id, step=f"running {dast_engines.ENGINE_NAMES[tool]}")
22512277
args = external_auth.engine_args(tool, session) \
22522278
if tool in external_auth.SIGNS_IN else []
2279+
# testssl reads the site's own TLS, so it must reach it directly,
2280+
# not through the proxy that terminates TLS with its own cert.
2281+
tool_proxy = proxy if tool == "testssl" else engine_proxy
22532282
result = dast_engines.run_engine(tool, url, host, profile, workdir,
22542283
scan_id, paths=extra, auth_args=args,
2255-
proxy=proxy)
2284+
proxy=tool_proxy)
22562285
if session:
22572286
result["signed_in"] = bool(args)
22582287
if tool == "ffuf":
@@ -2265,6 +2294,8 @@ def _builtin_run(scan_id, host, asset_id, profile, opts, auth, proxy) -> None:
22652294
engines.append(result)
22662295
_update(scan_id, engines=json.dumps(engines))
22672296
finally:
2297+
if scan_proxy:
2298+
scan_proxy.stop()
22682299
shutil.rmtree(workdir, ignore_errors=True)
22692300
# The premium curated rules: the enhanced, high fidelity checks. They run
22702301
# last, only on premium, and only against the company's own domains. With

‎app/core/settings_store.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,11 @@
388388
"can be made to fetch a URL will hit it here, which proves "
389389
"server side request forgery. Leave empty to switch the "
390390
"out of band checks off; they are then skipped."),
391+
"dast.scan_proxy": (True, False,
392+
"Run the scanning engines through a local proxy for each "
393+
"scan, so every engine signs in with the site's session "
394+
"and no engine can reach a host the scan is not scoped to. "
395+
"On by default; off makes the engines go direct."),
391396
"dast.exposure_check": (False, False,
392397
"On a telemetry refresh, also load each site through the "
393398
"non allow listed proxy below and take a second "

‎app/web/routes_external.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1575,6 +1575,7 @@ def _rule_context(db, user):
15751575
scan_profiles=saved, profile_rulesets=profile_rulesets,
15761576
available_rulesets=_available_rulesets(db),
15771577
oob_url=settings_store.get(db, "dast.oob_url") or "",
1578+
scan_proxy_on=bool(settings_store.get(db, "dast.scan_proxy")),
15781579
whitelisted_ip=settings_store.get(db, "dast.whitelisted_ip") or "",
15791580
whitelisted_vantage=settings_store.get(db, "dast.whitelisted_vantage") or "direct",
15801581
ip_echo_url=settings_store.get(db, "dast.ip_echo_url") or "https://api.ipify.org",
@@ -1687,6 +1688,7 @@ async def save_dast_settings(request: Request, user: User = Depends(current_user
16871688
vantage = "proxy" if str(form.get("whitelisted_vantage") or "") == "proxy" else "direct"
16881689
echo = str(form.get("ip_echo_url") or "").strip()[:500]
16891690
exposure = _checked(form, "exposure_check")
1691+
scan_proxy_on = _checked(form, "scan_proxy")
16901692
proxy_typed = str(form.get("unwhitelisted_proxy") or "").strip()[:1000]
16911693
proxy_clear = _checked(form, "proxy_clear")
16921694
for label, value in (("callback address", oob), ("IP echo service", echo),
@@ -1697,6 +1699,7 @@ async def save_dast_settings(request: Request, user: User = Depends(current_user
16971699
with session_scope() as db:
16981700
_guard(db, user)
16991701
settings_store.set_value(db, "dast.oob_url", oob, user.username)
1702+
settings_store.set_value(db, "dast.scan_proxy", scan_proxy_on, user.username)
17001703
settings_store.set_value(db, "dast.whitelisted_ip", whitelisted, user.username)
17011704
settings_store.set_value(db, "dast.whitelisted_vantage", vantage, user.username)
17021705
settings_store.set_value(db, "dast.ip_echo_url",

‎app/web/templates/dast_admin.html‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,14 @@
306306
must be reachable from the sites you scan. Leave empty to switch the
307307
out of band checks off.</span></label>
308308

309+
<label style="margin-top:.8rem">
310+
<input type="checkbox" name="scan_proxy" value="1" {{ 'checked' if scan_proxy_on }}>
311+
Run the engines through a local sign in proxy
312+
<span class="faint small">Each scan starts a local proxy the engines run through,
313+
so every engine signs in with the site's session and none can reach a host the
314+
scan is not scoped to. On by default. testssl always reads the site's TLS
315+
directly.</span></label>
316+
309317
<div class="card-head" style="margin-top:1rem">Exposure check</div>
310318
<label style="margin-top:.4rem">Which vantage is allow listed
311319
<select name="whitelisted_vantage">

0 commit comments

Comments
 (0)