Skip to content

Commit 1473017

Browse files
committed
Record every URI path found by a scan in a searchable table
Each built-in DAST scan now upserts its path inventory into external_site_paths, one row per (site, path), deduplicated across scans. first_seen_at is set on insert and never changed; last_seen_at, status, reachable, engines, and methods (ever-seen union) are refreshed on each scan. After every scan, paths absent from the site's last three completed scans are pruned. The table is indexed on path, so queries like path LIKE '/api/%' or path LIKE '%.php' run fast across the whole estate. The AI chat feature has access to it via the ask schema, with a join hint and example filters.
1 parent a6497cb commit 1473017

3 files changed

Lines changed: 130 additions & 1 deletion

File tree

‎app/core/ask.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,19 @@
193193
"critical": "count", "high": "count", "medium": "count", "low": "count",
194194
"info": "count", "profile": "", "authenticated": "boolean",
195195
}),
196+
"external_site_paths": ("URI paths found on external sites, one row per site "
197+
"and path, deduplicated across scans.", {
198+
"id": "key",
199+
"asset_id": "-> external_assets.id",
200+
"path": "the URI path, e.g. /api/v1/users or /app/upload.php",
201+
"first_seen_at": "when first found",
202+
"last_seen_at": "when most recently found by any scan",
203+
"status": "last HTTP status code (200, 301, 401, 404 etc.)",
204+
"reachable": "boolean; 1 means a real page confirmed by the scan",
205+
"methods": "JSON list of HTTP methods ever seen, e.g. [\"GET\",\"POST\"]",
206+
"engines": "JSON list of scan engines that found it last",
207+
"from_code": "boolean; linked repository code also declares this route",
208+
}),
196209
"users": ("People who can sign in to this tool.", {
197210
"id": "key", "username": "", "display_name": "", "email": "",
198211
"github_login": "", "role": "ADMIN, DEVELOPER, READER (upper case)",
@@ -215,6 +228,10 @@
215228
- What a repository uses (a framework, Power Apps, a package) is in components.name, \
216229
findings.file_path, repositories.application_name, repositories.note and \
217230
repositories.full_name. There is no list of languages.
231+
- external_site_paths.asset_id = external_assets.id. Filter reachable = 1 \
232+
for confirmed real pages. Use path LIKE '/api/%' to find API endpoints, \
233+
path LIKE '%.php' for PHP files, and so on. JOIN with external_assets to \
234+
get the host name for each path.
218235
- Booleans are 0 and 1. Dates are stored as UTC timestamps."""
219236

220237
PLAN_SYSTEM = """You help an administrator look things up in Git Code Review, \

‎app/core/external.py‎

Lines changed: 81 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@
4141
from app.core.db import session_scope
4242
from app.core.models import (
4343
DastServer, ExternalAsset, ExternalAssetRepo, ExternalAssetTag,
44-
ExternalDastScan, Finding, FindingState, Repository, RepoDiagram, utcnow,
44+
ExternalDastScan, ExternalSitePath, Finding, FindingState,
45+
Repository, RepoDiagram, utcnow,
4546
)
4647

4748
log = logging.getLogger("gcr.external")
@@ -3535,6 +3536,83 @@ def merge(findings: list) -> list:
35353536
return out[:MAX_FINDINGS]
35363537

35373538

3539+
_PRUNE_KEEP_SCANS = 3
3540+
3541+
3542+
def _upsert_site_paths(scan_id: int, asset_id: int, records: list) -> None:
3543+
"""Persist paths from one scan into the deduplicated per-site table."""
3544+
if not records:
3545+
return
3546+
now = datetime.now(timezone.utc)
3547+
with session_scope() as db:
3548+
existing = {row.path: row for row in
3549+
db.query(ExternalSitePath).filter_by(asset_id=asset_id).all()}
3550+
for rec in records:
3551+
path = rec.get("p") or ""
3552+
if not path:
3553+
continue
3554+
reachable = rec.get("r")
3555+
status = rec.get("s")
3556+
engines_json = json.dumps(rec.get("e") or [])
3557+
new_methods = {m.upper() for m in (rec.get("m") or ["GET"]) if m}
3558+
from_code = bool(rec.get("c"))
3559+
if path in existing:
3560+
row = existing[path]
3561+
try:
3562+
old_methods = set(json.loads(row.methods or "[]"))
3563+
except (ValueError, TypeError):
3564+
old_methods = set()
3565+
row.methods = json.dumps(sorted(old_methods | new_methods))
3566+
row.engines = engines_json
3567+
row.last_seen_at = now
3568+
row.last_scan_id = scan_id
3569+
if status is not None:
3570+
row.status = status
3571+
if reachable is not None:
3572+
row.reachable = reachable
3573+
if from_code:
3574+
row.from_code = True
3575+
else:
3576+
db.add(ExternalSitePath(
3577+
asset_id=asset_id,
3578+
path=path,
3579+
first_seen_at=now,
3580+
last_seen_at=now,
3581+
last_scan_id=scan_id,
3582+
status=status,
3583+
reachable=reachable,
3584+
methods=json.dumps(sorted(new_methods)),
3585+
engines=engines_json,
3586+
from_code=from_code,
3587+
))
3588+
3589+
3590+
def _prune_site_paths(asset_id: int) -> int:
3591+
"""Delete paths absent from the most recent _PRUNE_KEEP_SCANS completed scans.
3592+
3593+
Does nothing until the site has at least that many scans with path data,
3594+
so a path must be genuinely missed N times in a row before it is dropped.
3595+
Returns the number of rows removed.
3596+
"""
3597+
with session_scope() as db:
3598+
recent = (db.query(ExternalDastScan.id)
3599+
.filter(ExternalDastScan.asset_id == asset_id,
3600+
ExternalDastScan.status == "done",
3601+
ExternalDastScan.paths_count.isnot(None))
3602+
.order_by(ExternalDastScan.scanned_at.desc())
3603+
.limit(_PRUNE_KEEP_SCANS).all())
3604+
if len(recent) < _PRUNE_KEEP_SCANS:
3605+
return 0
3606+
from sqlalchemy import or_
3607+
recent_ids = [row.id for row in recent]
3608+
deleted = (db.query(ExternalSitePath)
3609+
.filter(ExternalSitePath.asset_id == asset_id,
3610+
or_(ExternalSitePath.last_scan_id.is_(None),
3611+
ExternalSitePath.last_scan_id.notin_(recent_ids)))
3612+
.delete(synchronize_session=False))
3613+
return deleted
3614+
3615+
35383616
def _paths_and_code(scan_id, asset_id, url, seen, findings, wb, opts, stopped,
35393617
signed_in: bool = False) -> list:
35403618
"""What the scan reached, and the repositories whose routes it answered.
@@ -3569,6 +3647,8 @@ def fetch(path):
35693647
paths_real=paths["counts"]["reachable"],
35703648
code_sinks=json.dumps(sinks), code_matches=json.dumps(
35713649
_code_matches(asset_id, records, index, names)))
3650+
_upsert_site_paths(scan_id, asset_id, records)
3651+
_prune_site_paths(asset_id)
35723652
if signed_in or stopped or not controls:
35733653
# Signed in, a 200 is what a guarded route gives too. The comparison
35743654
# only means something from outside.

‎app/core/models.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2811,6 +2811,38 @@ class ExternalDastScan(Base):
28112811
recorded_at = Column(DateTime(timezone=True), default=utcnow)
28122812

28132813

2814+
class ExternalSitePath(Base):
2815+
"""One URI path found on an external site, deduplicated across scans.
2816+
2817+
One row per (asset_id, path). first_seen_at is set on insert and never
2818+
updated. last_seen_at, status and reachable are refreshed on every scan
2819+
that sees the path again. methods accumulates every HTTP method ever seen
2820+
(ever-seen union across scans). Rows absent from the last N completed
2821+
scans are pruned after each scan.
2822+
"""
2823+
2824+
__tablename__ = "external_site_paths"
2825+
2826+
id = Column(Integer, primary_key=True)
2827+
asset_id = Column(Integer, ForeignKey("external_assets.id", ondelete="CASCADE"),
2828+
nullable=False, index=True)
2829+
path = Column(String(300), nullable=False)
2830+
first_seen_at = Column(DateTime(timezone=True), nullable=False, default=utcnow)
2831+
last_seen_at = Column(DateTime(timezone=True), nullable=False, default=utcnow)
2832+
last_scan_id = Column(Integer, ForeignKey("external_dast_scans.id", ondelete="SET NULL"),
2833+
nullable=True)
2834+
status = Column(Integer)
2835+
reachable = Column(Boolean)
2836+
methods = Column(String(100), default="")
2837+
engines = Column(String(200), default="")
2838+
from_code = Column(Boolean, default=False, nullable=False)
2839+
2840+
__table_args__ = (
2841+
UniqueConstraint("asset_id", "path", name="uq_site_path"),
2842+
Index("ix_site_path_path", "path"),
2843+
)
2844+
2845+
28142846
class ExternalScanSchedule(Base):
28152847
"""A dynamic scan of one site that starts on its own.
28162848

0 commit comments

Comments
 (0)