Skip to content

Commit 18f2acf

Browse files
committed
Tie sites to the code that serves them, and unpin or remove repositories in bulk
Every scan now reads each repository's default branch for the host names its own files say it answers on: Kubernetes ingress and route hosts, Helm values under ingress, nginx, Apache, Traefik and Caddy rules, a CNAME file, CDN aliases and certificates in Terraform, deploy environment urls in CI, OpenAPI servers, allowed hosts and app url settings, and a package homepage. Other http or https addresses in the code are kept as named, since the code may only call them. Nothing is fetched and nothing in the checkout runs. A site whose host a repository serves now links to it, and so does another environment of that host on the same domain. A wildcard, a host the code only names, or a repository serving more than 20 hosts is a suggestion. Parsed lists are cached until the next scan so pages stay fast. A repository's page lists the sites its code serves, and an administrator can add the ones that are not External sites yet in one press. On a site's page, Code behind it gains a tick box per repository, a select all, and Unpin selected and Remove selected. The route checks each name against what the person can see, allows only unpin and remove for several, caps a submit at 200, and audits each one.
1 parent 4fdfd46 commit 18f2acf

14 files changed

Lines changed: 720 additions & 24 deletions

‎app/core/code_hosts.py‎

Lines changed: 343 additions & 0 deletions
Large diffs are not rendered by default.

‎app/core/external.py‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,10 @@ def default_environment(db) -> str:
110110
# it counts without anybody saying so. Below this it is a suggestion.
111111
LINK_THRESHOLD = 60
112112
MAX_WHITEBOX_REPOS = 3
113+
# A repository whose files serve more hosts than this is likely shared
114+
# infrastructure (a DNS, CDN or ingress repository for many sites), so a host
115+
# it serves is a suggestion rather than a link.
116+
SHARED_SERVES = 20
113117
MAX_WHITEBOX_SINKS = 300
114118
MAX_FINDINGS = 3000
115119

@@ -988,6 +992,37 @@ def is_blocked(host: str, blocked_list: list, own_flag: bool) -> bool:
988992
r"sandbox|sbx|demo|dr)\d*$")
989993

990994

995+
# Each repository's parsed host list, kept until a scan replaces it. Ingest
996+
# checks the list when it stores it, so this only parses, and only once per
997+
# scan rather than on every page that works out links.
998+
_HOSTS_CACHE: dict = {}
999+
_HOSTS_CACHE_MAX = 20000
1000+
1001+
1002+
def _code_hosts_of(repo) -> list:
1003+
raw = repo.default_hosts or ""
1004+
if not raw:
1005+
return []
1006+
stamp = (repo.default_hosts_at, len(raw))
1007+
hit = _HOSTS_CACHE.get(repo.id)
1008+
if hit is not None and hit[0] == stamp:
1009+
return hit[1]
1010+
try:
1011+
value = json.loads(raw)
1012+
except ValueError:
1013+
value = []
1014+
entries = [e for e in value if isinstance(e, dict) and isinstance(e.get("host"), str)
1015+
and e.get("kind") in ("serves", "names")] if isinstance(value, list) else []
1016+
for e in entries:
1017+
e.setdefault("how", "")
1018+
if not isinstance(e.get("files"), list):
1019+
e["files"] = []
1020+
if len(_HOSTS_CACHE) >= _HOSTS_CACHE_MAX:
1021+
_HOSTS_CACHE.clear()
1022+
_HOSTS_CACHE[repo.id] = (stamp, entries)
1023+
return entries
1024+
1025+
9911026
def _stem(text: str) -> str:
9921027
"""A name with its environment words and punctuation taken off."""
9931028
value = re.sub(r"[^a-z0-9]", "", (text or "").lower())
@@ -1035,6 +1070,26 @@ def __init__(self, db, user=None):
10351070
stem = _stem(repo.name)
10361071
if len(stem) >= 4:
10371072
self.by_stem.setdefault(stem, []).append(repo.id)
1073+
# Host names each repository's own files serve, or name, read off
1074+
# its default branch by every scan. See app.core.code_hosts.
1075+
self.code_serves, self.code_names, self.code_wild = {}, {}, []
1076+
self.code_serve_stem, self.serve_count = {}, {}
1077+
for repo in repos:
1078+
entries = _code_hosts_of(repo)
1079+
serves = [e for e in entries if e["kind"] == "serves"]
1080+
self.serve_count[repo.id] = len(serves)
1081+
for e in serves:
1082+
if e["host"].startswith("*."):
1083+
self.code_wild.append((e["host"][1:], repo.id, e))
1084+
continue
1085+
self.code_serves.setdefault(e["host"], {}).setdefault(repo.id, e)
1086+
label, _dot, rest = e["host"].partition(".")
1087+
stem = _stem(label)
1088+
if len(stem) >= 4 and rest:
1089+
self.code_serve_stem.setdefault((stem, rest), {}).setdefault(repo.id, e)
1090+
for e in entries:
1091+
if e["kind"] == "names":
1092+
self.code_names.setdefault(e["host"], {}).setdefault(repo.id, e)
10381093
self.app_names = {r.id: ((applies.get(r.id, {}).get("application_name") or {})
10391094
.get("value") or "").strip() for r in repos}
10401095
# Hosts a diagram found written into each repository's code.
@@ -1100,6 +1155,30 @@ def add(repo_id, score, reason):
11001155

11011156
for repo_id, how in (self.by_host.get(asset.host) or {}).items():
11021157
add(repo_id, 95 if how == "its web addresses" else 80, f"host named in {how}")
1158+
# What the repository's own files say it serves.
1159+
host = asset.host
1160+
for repo_id, e in (self.code_serves.get(host) or {}).items():
1161+
where = e["files"][0] if e["files"] else "its files"
1162+
if self.serve_count.get(repo_id, 0) > SHARED_SERVES:
1163+
add(repo_id, 55, f"its {e['how']} serves this host ({where}), but it serves "
1164+
f"{self.serve_count[repo_id]} hosts, so it may be shared "
1165+
"infrastructure")
1166+
else:
1167+
add(repo_id, 88, f"its {e['how']} serves this host ({where})")
1168+
for suffix, repo_id, e in self.code_wild:
1169+
if host.endswith(suffix) and host != suffix.lstrip("."):
1170+
add(repo_id, 55, f"its {e['how']} serves *{suffix}, which covers this host")
1171+
label0, _dot, rest0 = host.partition(".")
1172+
stem0 = _stem(label0)
1173+
if len(stem0) >= 4 and rest0:
1174+
for repo_id, e in (self.code_serve_stem.get((stem0, rest0)) or {}).items():
1175+
if e["host"] != host and self.serve_count.get(repo_id, 0) <= SHARED_SERVES:
1176+
add(repo_id, 70, f"its {e['how']} serves {e['host']}, another "
1177+
"environment of this host")
1178+
for repo_id, e in (self.code_names.get(host) or {}).items():
1179+
where = e["files"][0] if e["files"] else "its code"
1180+
add(repo_id, 45, f"its code names this host ({where}), which may only mean "
1181+
"it calls it")
11031182
for app in self.apps_by_host.get(asset.host, []):
11041183
for repo_id in app["repo_ids"]:
11051184
add(repo_id, 90, f"part of web application {app['name']}, which lists this host")

‎app/core/ingest.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -968,6 +968,14 @@ def ingest_report(session, report_path: Path, scanned_repos: set = None,
968968
[r for r in routes if isinstance(r, str) and r.startswith("/")
969969
and len(r) <= 200][:300])
970970
repo.default_routes_at = utcnow()
971+
# The host names it serves or names, the same way: only a run that
972+
# read them replaces the list.
973+
hosts = entry.get("default_hosts")
974+
if isinstance(hosts, list):
975+
from app.core import code_hosts
976+
977+
repo.default_hosts = json.dumps(code_hosts.load(hosts))
978+
repo.default_hosts_at = utcnow()
971979
dast = entry.get("dast")
972980
if isinstance(dast, dict) and dast.get("verdict"):
973981
repo.dast_verdict = str(dast.get("verdict"))[:16]

‎app/core/models.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,10 @@ class Repository(Base):
409409
# stays off the row.
410410
default_routes = Column(Text, default="")
411411
default_routes_at = Column(DateTime(timezone=True))
412+
# Host names the default branch serves (ingress, server, deploy, CDN,
413+
# allowed hosts) or names in its code. JSON from app.core.code_hosts.
414+
default_hosts = Column(Text, default="")
415+
default_hosts_at = Column(DateTime(timezone=True))
412416

413417
# Every branch the last scan actually walked here, as a list of names.
414418
#

‎app/web/routes_external.py‎

Lines changed: 47 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,10 @@
5050
NOT_FOUND = "Not Found"
5151

5252

53+
# The most repositories one submit on a site's page may change.
54+
MAX_BULK_REPOS = 200
55+
56+
5357
def _back(where: str, message: str, kind: str = "ok") -> str:
5458
from urllib.parse import quote
5559

@@ -978,7 +982,10 @@ async def clear_tls_pins(asset_id: int, request: Request, user: User = Depends(c
978982

979983
@router.post("/external/{asset_id}/repos")
980984
async def link_repo(asset_id: int, request: Request, user: User = Depends(current_user)):
981-
"""Link a repository by name, or confirm, hide or forget a found one."""
985+
"""Link a repository by name, or confirm, hide or forget a found one.
986+
987+
Several ticked on the site's page come as several "repository" fields.
988+
Only unpin (forget) and remove (hide) take more than one."""
982989
await check_csrf(request)
983990
from app.core.models import Repository
984991

@@ -989,33 +996,52 @@ async def link_repo(asset_id: int, request: Request, user: User = Depends(curren
989996
if re.fullmatch(rf"/external/{int(asset_id)}/scans/\d{{1,9}}", back):
990997
here = back + "#code"
991998
action = str(form.get("action") or "link")
999+
if action not in ("link", "hide", "forget"):
1000+
return redirect(_back(here, "Pick what to do with them.", "error"))
1001+
raw = [str(v) for v in form.getlist("repository") if str(v).strip()]
1002+
if not raw:
1003+
return redirect(_back(here, "Tick at least one repository.", "error"))
1004+
if len(raw) > MAX_BULK_REPOS:
1005+
return redirect(_back(here, f"At most {MAX_BULK_REPOS} at a time.", "error"))
1006+
if len(raw) > 1 and action == "link":
1007+
return redirect(_back(here, "Link them one at a time.", "error"))
1008+
names = list(dict.fromkeys(clean_text(v, "Repository", limit=320) for v in raw))
9921009
with session_scope() as db:
9931010
_guard(db, user)
9941011
asset = _asset(db, asset_id)
995-
name = clean_text(form.get("repository"), "Repository", limit=320)
9961012
# Through rbac, so a name the person cannot see is the same as a
9971013
# name that does not exist.
998-
repo = (rbac.visible_repositories(db, user)
999-
.filter(Repository.full_name == name).one_or_none())
1000-
if repo is None:
1001-
return redirect(_back(here, f"No repository called {name}.", "error"))
1002-
row = (db.query(ExternalAssetRepo)
1003-
.filter(ExternalAssetRepo.asset_id == asset.id,
1004-
ExternalAssetRepo.repository_id == repo.id).one_or_none())
1014+
repos = {r.full_name: r for r in rbac.visible_repositories(db, user)
1015+
.filter(Repository.full_name.in_(names)).all()}
1016+
missing = [n for n in names if n not in repos]
1017+
if missing:
1018+
return redirect(_back(here, f"No repository called {missing[0]}.", "error"))
1019+
rows = {r.repository_id: r for r in db.query(ExternalAssetRepo)
1020+
.filter(ExternalAssetRepo.asset_id == asset.id,
1021+
ExternalAssetRepo.repository_id.in_([r.id for r in repos.values()]))
1022+
.all()}
1023+
for name in names:
1024+
repo = repos[name]
1025+
row = rows.get(repo.id)
1026+
if action == "forget":
1027+
if row is not None:
1028+
db.delete(row)
1029+
else:
1030+
if row is None:
1031+
row = ExternalAssetRepo(asset_id=asset.id, repository_id=repo.id)
1032+
db.add(row)
1033+
row.source = "hidden" if action == "hide" else "manual"
1034+
row.added_by = user.username[:120]
1035+
record(db, f"external.repo_{action}", actor=user.username,
1036+
target=asset.host, detail=repo.full_name, source_ip=client_ip(request))
1037+
one = len(names) == 1
1038+
what = names[0] if one else f"{len(names)} repositories"
10051039
if action == "forget":
1006-
if row is not None:
1007-
db.delete(row)
1008-
message = f"{repo.full_name} is back to being worked out."
1040+
message = f"{what} {'is' if one else 'are'} back to being worked out."
1041+
elif action == "hide":
1042+
message = f"{what} will not be linked to {asset.host}."
10091043
else:
1010-
if row is None:
1011-
row = ExternalAssetRepo(asset_id=asset.id, repository_id=repo.id)
1012-
db.add(row)
1013-
row.source = "hidden" if action == "hide" else "manual"
1014-
row.added_by = user.username[:120]
1015-
message = (f"{repo.full_name} will not be linked to {asset.host}."
1016-
if action == "hide" else f"Linked {repo.full_name}.")
1017-
record(db, f"external.repo_{action}", actor=user.username,
1018-
target=asset.host, detail=repo.full_name, source_ip=client_ip(request))
1044+
message = f"Linked {what}."
10191045
return redirect(_back(here, message))
10201046

10211047

‎app/web/routes_pages.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2413,6 +2413,38 @@ async def web_apps_import_check(request: Request,
24132413
return page(request, "web_apps_import_confirm.html", **context)
24142414

24152415

2416+
def _served_hosts(db, user, repo) -> dict:
2417+
"""The host names this repository's files say it serves, for its page.
2418+
2419+
Anybody who can see the repository sees what its own files say. Whether
2420+
each host is an External site, and the button to add the ones that are
2421+
not, is only for somebody who manages External sites. Hosts the code
2422+
only names are shown when they are already a site, since a list of every
2423+
address the code calls says nothing about what it serves.
2424+
"""
2425+
from app.core import code_hosts, external
2426+
from app.core.models import ExternalAsset
2427+
2428+
entries = code_hosts.load(repo.default_hosts or "[]") if repo.default_hosts else []
2429+
can_sites = rbac.can(user, "external.manage") and external.enabled(db)
2430+
sites = {}
2431+
if can_sites and entries:
2432+
wanted = [e["host"] for e in entries if not e["host"].startswith("*.")]
2433+
for i in range(0, len(wanted), 500):
2434+
for asset_id, host in (db.query(ExternalAsset.id, ExternalAsset.host)
2435+
.filter(ExternalAsset.host.in_(wanted[i:i + 500]))):
2436+
sites[host] = asset_id
2437+
served = []
2438+
for e in entries:
2439+
if e["kind"] == "names" and e["host"] not in sites:
2440+
continue
2441+
served.append(dict(e, site_id=sites.get(e["host"])))
2442+
missing = [e["host"] for e in served if e["kind"] == "serves" and not e["site_id"]
2443+
and not e["host"].startswith("*.")]
2444+
return {"served": served, "served_at": repo.default_hosts_at,
2445+
"can_sites": can_sites, "served_missing": missing}
2446+
2447+
24162448
@router.get("/repositories/{repo_id:int}")
24172449
def repository_about(repo_id: int, request: Request, msg: str = "",
24182450
kind: str = "", user: User = Depends(current_user)):
@@ -2479,6 +2511,7 @@ def repository_about(repo_id: int, request: Request, msg: str = "",
24792511
"set_by": r.set_by or "", "set_at": r.set_at}
24802512
for r in scan_scope.settings_of(db, repo.full_name)["branches"]
24812513
if r.mode == "none"]
2514+
context.update(_served_hosts(db, user, repo))
24822515
from app.core import ai as _ai, code_reasoning as _cr
24832516
context["can_reason"] = (rbac.can(user, "finding.validate")
24842517
and _cr.is_on(_ai.config(db)))

‎app/web/static/external_repos.js‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/* Code behind it, on a site's page: tick several repositories, or all of
2+
them, and unpin or remove them together. In a static file because the
3+
content security policy does not allow inline scripts. */
4+
(function () {
5+
"use strict";
6+
7+
var form = document.getElementById("repos-bulk");
8+
if (!form) { return; }
9+
var all = document.querySelector("[data-repos-check-all]");
10+
var count = form.querySelector("[data-repos-count]");
11+
var buttons = form.querySelectorAll("[data-repos-needs-tick]");
12+
13+
function ticks() {
14+
return Array.prototype.slice.call(document.querySelectorAll("[data-repos-tick]"));
15+
}
16+
17+
function refresh() {
18+
var boxes = ticks();
19+
var n = boxes.filter(function (b) { return b.checked; }).length;
20+
if (count) { count.textContent = n ? n + " ticked" : "None ticked"; }
21+
Array.prototype.forEach.call(buttons, function (b) { b.disabled = !n; });
22+
if (all) {
23+
all.checked = n > 0 && n === boxes.length;
24+
all.indeterminate = n > 0 && n < boxes.length;
25+
}
26+
}
27+
28+
if (all) {
29+
all.addEventListener("change", function () {
30+
ticks().forEach(function (b) { b.checked = all.checked; });
31+
refresh();
32+
});
33+
}
34+
document.addEventListener("change", function (e) {
35+
if (e.target.matches && e.target.matches("[data-repos-tick]")) { refresh(); }
36+
});
37+
form.addEventListener("submit", function (e) {
38+
var n = ticks().filter(function (b) { return b.checked; }).length;
39+
var hide = e.submitter && e.submitter.value === "hide";
40+
if (hide && !window.confirm("Remove " + n + " repositor" + (n === 1 ? "y" : "ies") +
41+
" from this site? They will not be linked again " +
42+
"unless someone links them by hand.")) {
43+
e.preventDefault();
44+
}
45+
});
46+
refresh();
47+
})();

‎app/web/templates/external_site.html‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -375,11 +375,24 @@
375375
{%- if links | length > linked | length %} <span class="faint small">, {{ links | length - linked | length }} suggested</span>{% endif %}</span></summary>
376376
<div class="faint small" style="padding:.4rem 1rem 0">Picked in Edit, or worked out from the Web Applications tab, web addresses, the code and names. Linked at 60 and over.</div>
377377
{% if links %}
378+
{# Rows tick into this one form, so several can be unpinned or removed
379+
at once. Each row keeps its own buttons in its own form. #}
380+
<form method="post" action="{{ base }}/external/{{ asset.id }}/repos" id="repos-bulk"
381+
class="card-body repos-bulk" style="display:flex;gap:.4rem;align-items:center;flex-wrap:wrap">
382+
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
383+
<span class="faint small" data-repos-count>None ticked</span>
384+
<button class="btn sm" name="action" value="forget" type="submit" data-repos-needs-tick
385+
title="Hand picked or code found links go back to being worked out">Unpin selected</button>
386+
<button class="btn sm danger" name="action" value="hide" type="submit" data-repos-needs-tick
387+
title="Never link these repositories to this site">Remove selected</button>
388+
</form>
378389
<div class="table-scroll"><table>
379-
<thead><tr><th>Repository</th><th>Why</th><th>Open findings</th><th></th></tr></thead>
390+
<thead><tr><th><input type="checkbox" data-repos-check-all aria-label="Select every repository"></th><th>Repository</th><th>Why</th><th>Open findings</th><th></th></tr></thead>
380391
<tbody>
381392
{% for l in links %}
382393
<tr>
394+
<td><input type="checkbox" name="repository" value="{{ l.full_name }}" form="repos-bulk"
395+
data-repos-tick aria-label="Select {{ l.full_name }}"></td>
383396
<td class="small"><a href="{{ base }}/repositories/{{ l.id }}">{{ l.full_name }}</a>
384397
<div>{% if l.source == 'manual' %}<span class="badge ok">picked</span>{% elif l.linked %}<span class="badge ok">linked</span>{% else %}<span class="badge pending">suggested</span>{% endif %}
385398
<span class="faint">{{ l.score }}</span></div>
@@ -695,4 +708,5 @@
695708
</form></div>
696709
</div>
697710
<script src="{{ base }}/static/external.js?v={{ asset_v('external.js') }}"></script>
711+
<script src="{{ base }}/static/external_repos.js?v={{ asset_v('external_repos.js') }}"></script>
698712
{% endblock %}

0 commit comments

Comments
 (0)