Skip to content

Commit d6cbb3c

Browse files
committed
Backups: nightly and before any bulk delete, and delete all says it is the whole database
"Delete all findings" sat beside the export buttons that act on the current view, and was read as "delete what this view shows". It deleted every finding in the database, with no backup to go back to. The button now reads "Delete ALL N findings in the database", and the dialog says it is not the current view, that the filters are ignored, and how to delete only what is on screen instead. A backup is now taken before a delete of all findings, and before a delete of everything a filter matches. If the backup cannot be written, nothing is deleted. The same backup is taken nightly at 05:30. Both are the encrypted file the Backup and restore page already makes and restores, kept in a host folder (GCR_BACKUP_PATH, ./backups by default) at mode 600, with the newest five of each kind kept. The password comes from GCR_BACKUP_PASSWORD in .env, which install.sh now generates, and never from the database. Without it the nightly backup is skipped with a reason, and bulk deletes are refused. The Backup and restore page lists the automatic backups and offers each for download. Only names in the current listing are served.
1 parent ebe79ff commit d6cbb3c

8 files changed

Lines changed: 352 additions & 10 deletions

File tree

‎app/core/autobackup.py‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
"""Backups nobody has to remember to take.
2+
3+
Two kinds, both the same encrypted file the Backup and restore page makes, so
4+
either one goes back through that page:
5+
6+
nightly taken once a night by the scheduler
7+
before-delete taken just before a bulk delete of findings, which does not
8+
undo any other way
9+
10+
The five newest of each kind are kept. The two kinds are counted apart, so a
11+
run of deletes cannot push the nightly copies out, and five quiet nights cannot
12+
push out the copy taken before a delete.
13+
14+
The password comes from GCR_BACKUP_PASSWORD, which install.sh writes into .env.
15+
It is never kept in the database, because a backup whose password went down
16+
with the database could never be opened. With no password set, nothing is
17+
written and the reason is given; a backup under a made up password nobody
18+
knows would be worse than none, because it looks like protection.
19+
"""
20+
21+
from __future__ import annotations
22+
23+
import logging
24+
import os
25+
import re
26+
import threading
27+
from datetime import datetime, timezone
28+
from pathlib import Path
29+
30+
log = logging.getLogger("gcr.autobackup")
31+
32+
KEEP = 5
33+
KINDS = ("nightly", "before-delete")
34+
35+
# The only names this module writes, and so the only names it will hand back
36+
# out. Anything else in the folder is somebody else's and is left alone.
37+
NAME = re.compile(r"^gcr-(nightly|before-delete)-(\d{8}-\d{6})\.zip$")
38+
39+
_lock = threading.Lock()
40+
41+
42+
class NotConfigured(Exception):
43+
"""No password, so no backup can be written."""
44+
45+
46+
def folder() -> Path:
47+
"""Where the files go. A host folder in production, so they outlive the
48+
container and the database volume both."""
49+
raw = os.environ.get("GCR_BACKUP_DIR", "").strip()
50+
if raw:
51+
return Path(raw)
52+
from app.config import settings
53+
return settings.data_dir / "backups"
54+
55+
56+
def _password() -> str:
57+
from app.core import backup
58+
value = os.environ.get("GCR_BACKUP_PASSWORD", "")
59+
if not value:
60+
raise NotConfigured(
61+
"No backup password is set. Put GCR_BACKUP_PASSWORD in .env "
62+
"(install.sh writes one) and recreate the container.")
63+
if len(value) < backup.MIN_PASSWORD:
64+
raise NotConfigured(
65+
f"GCR_BACKUP_PASSWORD is shorter than {backup.MIN_PASSWORD} "
66+
"characters, so it is not used.")
67+
return value
68+
69+
70+
def configured() -> bool:
71+
try:
72+
_password()
73+
except NotConfigured:
74+
return False
75+
return True
76+
77+
78+
def listing() -> list[dict]:
79+
"""The backups on disk, newest first."""
80+
where = folder()
81+
if not where.is_dir():
82+
return []
83+
found = []
84+
for entry in where.iterdir():
85+
match = NAME.match(entry.name)
86+
if not match or not entry.is_file():
87+
continue
88+
stat = entry.stat()
89+
found.append({
90+
"name": entry.name,
91+
"kind": match.group(1),
92+
"taken_at": datetime.strptime(match.group(2), "%Y%m%d-%H%M%S")
93+
.replace(tzinfo=timezone.utc),
94+
"bytes": stat.st_size,
95+
})
96+
found.sort(key=lambda item: item["taken_at"], reverse=True)
97+
return found
98+
99+
100+
def path_of(name: str) -> Path | None:
101+
"""The file behind a name from the listing, or None.
102+
103+
The name has to be one this module writes and has to be in the folder
104+
right now. Nothing from a request is ever joined onto a path unchecked.
105+
"""
106+
if not isinstance(name, str) or not NAME.match(name):
107+
return None
108+
for item in listing():
109+
if item["name"] == name:
110+
return folder() / item["name"]
111+
return None
112+
113+
114+
def _prune(kind: str) -> int:
115+
mine = [item for item in listing() if item["kind"] == kind]
116+
removed = 0
117+
for item in mine[KEEP:]:
118+
try:
119+
(folder() / item["name"]).unlink()
120+
removed += 1
121+
except OSError:
122+
log.warning("could not remove old backup %s", item["name"], exc_info=True)
123+
return removed
124+
125+
126+
def take(kind: str, actor: str = "system") -> dict:
127+
"""Write one backup and trim that kind back to the newest five.
128+
129+
Raises NotConfigured when there is no password, and lets anything else
130+
through, because a caller about to delete findings must know it failed.
131+
"""
132+
if kind not in KINDS:
133+
raise ValueError(f"unknown backup kind {kind!r}")
134+
password = _password()
135+
136+
from app.core import backup
137+
from app.core.audit import record
138+
from app.core.db import session_scope
139+
140+
with _lock:
141+
where = folder()
142+
where.mkdir(parents=True, exist_ok=True)
143+
stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
144+
name = f"gcr-{kind}-{stamp}.zip"
145+
final = where / name
146+
partial = where / f".{name}.partial"
147+
148+
with session_scope() as db:
149+
blob = backup.create(db, password, actor=actor)
150+
151+
# Written beside the real name and moved into place, so a half written
152+
# file never carries a name the listing would offer for a restore.
153+
fd = os.open(partial, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
154+
try:
155+
with os.fdopen(fd, "wb") as handle:
156+
handle.write(blob)
157+
handle.flush()
158+
os.fsync(handle.fileno())
159+
os.replace(partial, final)
160+
except BaseException:
161+
try:
162+
partial.unlink()
163+
except OSError:
164+
pass
165+
raise
166+
167+
removed = _prune(kind)
168+
with session_scope() as db:
169+
record(db, "backup.created", actor=actor or "system", target=name,
170+
detail=f"{kind}, {len(blob)} bytes"
171+
+ (f", {removed} older removed" if removed else ""))
172+
173+
log.info("%s backup written: %s (%d bytes)", kind, name, len(blob))
174+
return {"name": name, "bytes": len(blob), "removed": removed}
175+
176+
177+
def nightly() -> None:
178+
"""The scheduler's job. Never raises: a failed night is logged and the
179+
next night tries again."""
180+
try:
181+
take("nightly", actor="system")
182+
except NotConfigured as exc:
183+
log.warning("nightly backup skipped: %s", exc)
184+
except Exception: # noqa: BLE001 - logged, and tomorrow is another go
185+
log.exception("nightly backup failed")

‎app/core/scheduler.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,17 @@ def start() -> None:
252252
replace_existing=True, misfire_grace_time=120,
253253
coalesce=True, max_instances=1)
254254

255+
# The nightly backup. Early morning, after the evening sweep and the
256+
# overnight bulk window have usually finished, though it only reads and is
257+
# safe beside a scan. Without a backup password it logs why and skips.
258+
from app.core import autobackup
259+
260+
_scheduler.add_job(autobackup.nightly,
261+
CronTrigger(hour=5, minute=30, timezone=_zone()),
262+
id="nightly-backup", replace_existing=True,
263+
misfire_grace_time=6 * 3600, coalesce=True,
264+
max_instances=1)
265+
255266
# Organizations nobody has a connection for. Weekly is plenty, since a new
256267
# organization is rare and the check asks GitHub about every token. It
257268
# does nothing unless it is switched on, so arming it always is harmless.

‎app/web/routes_api.py‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,29 @@ async def delete_many(request: Request, user: User = Depends(current_user)):
511511
elif not wanted and not matching:
512512
raise HTTPException(status_code=400, detail="Nothing was selected.")
513513

514+
# A delete of the whole list, or of everything a filter matches, takes a
515+
# backup first. Nothing else can bring those rows back: on 2026-11-13 a
516+
# delete of every finding was meant for one view, and the rows came back
517+
# only by carving them out of the database files. No backup, no delete.
518+
saved = None
519+
if everything or matching:
520+
from app.core import autobackup
521+
522+
try:
523+
saved = await run_in_threadpool(
524+
autobackup.take, "before-delete", user.username)
525+
except autobackup.NotConfigured as exc:
526+
raise HTTPException(
527+
status_code=503,
528+
detail=f"Nothing was deleted. A backup is taken before a "
529+
f"bulk delete, and this one could not be. {exc}")
530+
except Exception: # noqa: BLE001 - details go to the log, not the page
531+
log.exception("backup before bulk delete failed")
532+
raise HTTPException(
533+
status_code=503,
534+
detail="Nothing was deleted. The backup taken before a bulk "
535+
"delete failed; the server log says why.")
536+
514537
with session_scope() as db:
515538
# Through the scope filter every time, so a list of ids from somewhere
516539
# else deletes nothing rather than somebody else's findings.
@@ -540,7 +563,8 @@ async def delete_many(request: Request, user: User = Depends(current_user)):
540563
db.flush()
541564

542565
record(db, "finding.deleted", actor=user.username,
543-
target=f"{removed} findings", detail=scope,
566+
target=f"{removed} findings",
567+
detail=scope + (f", backup {saved['name']}" if saved else ""),
544568
source_ip=client_ip(request))
545569

546570
# The report files are the thing other people read. Leaving them
@@ -550,6 +574,9 @@ async def delete_many(request: Request, user: User = Depends(current_user)):
550574

551575
note = (f"{removed} finding(s) deleted. The next scan files them again if "
552576
"the code still matches.")
577+
if saved:
578+
note += (f" A backup was taken first ({saved['name']}); it is on the "
579+
"Backup and restore page.")
553580
return redirect(f"/findings?msg={quote(note)}&kind=ok")
554581

555582

@@ -3589,6 +3616,31 @@ async def create_backup(request: Request, user: User = Depends(current_user)):
35893616
"Cache-Control": "no-store"})
35903617

35913618

3619+
@router.get("/backup/files/{name}")
3620+
def download_saved_backup(name: str, request: Request,
3621+
user: User = Depends(current_user)):
3622+
"""One of the automatic backups, to keep somewhere else or restore.
3623+
3624+
Only a name the listing holds right now is served, so nothing typed into
3625+
the address is ever joined onto a path.
3626+
"""
3627+
rbac.require(user, "settings.write")
3628+
3629+
from fastapi.responses import FileResponse
3630+
3631+
from app.core import autobackup
3632+
3633+
path = autobackup.path_of(name)
3634+
if path is None:
3635+
raise HTTPException(status_code=404, detail="No such backup.")
3636+
with session_scope() as db:
3637+
record(db, "backup.downloaded", actor=user.username, target=path.name,
3638+
source_ip=client_ip(request))
3639+
return FileResponse(path, media_type="application/zip", filename=path.name,
3640+
headers={"Cache-Control": "no-store",
3641+
"X-Content-Type-Options": "nosniff"})
3642+
3643+
35923644
@router.post("/backup/restore")
35933645
async def restore_backup(request: Request, user: User = Depends(current_user)):
35943646
"""Replace this installation with the contents of a backup.

‎app/web/routes_pages.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,12 @@ def _sort_order(sort: str, direction: str):
301301
return [ordered, Finding.id.asc()]
302302

303303

304+
def _backup_ready() -> bool:
305+
"""Whether a bulk delete could take its backup first."""
306+
from app.core import autobackup
307+
return autobackup.configured()
308+
309+
304310
def _expiry_window(days: int) -> dict:
305311
"""What an expiry picker may offer: the default, and both ends of it.
306312
@@ -1666,6 +1672,8 @@ def _choices(name, **fixed):
16661672
# screen.
16671673
all_total=(_visible_count()
16681674
if rbac.can(user, "findings.delete") else 0),
1675+
backup_ready=(_backup_ready()
1676+
if rbac.can(user, "findings.delete") else False),
16691677
**_expiry_window(90),
16701678
)
16711679
return page(request, "findings.html", **context)
@@ -4716,6 +4724,7 @@ def backup_page(request: Request, msg: str = "", kind: str = "",
47164724
user: User = Depends(current_user)):
47174725
"""Take a copy of this installation, or put one back."""
47184726
rbac.require(user, "settings.write")
4727+
from app.core import autobackup
47194728
from app.core import backup as backup_store
47204729
from app.core.models import Base
47214730

@@ -4737,6 +4746,9 @@ def backup_page(request: Request, msg: str = "", kind: str = "",
47374746
counts=counts,
47384747
total_rows=sum(counts.values()),
47394748
min_password=backup_store.MIN_PASSWORD,
4749+
auto_backups=autobackup.listing(),
4750+
auto_configured=autobackup.configured(),
4751+
auto_keep=autobackup.KEEP,
47404752
message=clean_notice(msg, limit=300),
47414753
message_kind="error" if kind == "error" else "ok")
47424754
return page(request, "backup.html", **context)

‎app/web/templates/backup.html‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,44 @@
5252
</div>
5353
</div>
5454

55+
<div class="card" id="automatic">
56+
<div class="card-head"><span class="head-text">Automatic backups</span></div>
57+
{% if not auto_configured %}
58+
<div class="notice bad">
59+
<svg viewBox="0 0 16 16" fill="currentColor"><path d="M8 1.5 15 14H1L8 1.5zm-.75 4.7h1.5v3.6h-1.5V6.2zm0 4.6h1.5v1.5h-1.5v-1.5z"/></svg>
60+
<div><strong>Switched off: no backup password is set.</strong> Put
61+
GCR_BACKUP_PASSWORD (at least {{ min_password }} characters) in .env and
62+
recreate the container. Until then there is no nightly backup, and
63+
deleting all findings is refused because it cannot take one first.</div>
64+
</div>
65+
{% endif %}
66+
<p class="small faint">One every night at 05:30, and one just before any
67+
delete of all findings or of everything a filter matches. The newest
68+
{{ auto_keep }} of each kind are kept. They are the same encrypted file as
69+
the one above, under the password in GCR_BACKUP_PASSWORD, so any of them
70+
goes back through <a href="#put-one-back">Put one back</a>. Keep a copy of
71+
that password somewhere other than this server.</p>
72+
{% if auto_backups %}
73+
<div class="table-scroll">
74+
<table>
75+
<thead><tr><th>Taken (UTC)</th><th>Kind</th><th>Size</th><th></th></tr></thead>
76+
<tbody>
77+
{% for b in auto_backups %}
78+
<tr>
79+
<td>{{ b.taken_at.strftime('%Y-%m-%d %H:%M:%S') }}</td>
80+
<td>{{ 'Nightly' if b.kind == 'nightly' else 'Before a delete' }}</td>
81+
<td>{{ '%.1f'|format(b.bytes / 1048576) }} MB</td>
82+
<td><a class="btn sm" href="{{ base }}/backup/files/{{ b.name|urlencode }}">Download</a></td>
83+
</tr>
84+
{% endfor %}
85+
</tbody>
86+
</table>
87+
</div>
88+
{% elif auto_configured %}
89+
<p class="small faint">None yet. The first one is taken tonight.</p>
90+
{% endif %}
91+
</div>
92+
5593
<div class="card" id="whats-in-it">
5694
<div class="card-head"><span class="head-text">What goes in</span></div>
5795
<div class="table-scroll">

0 commit comments

Comments
 (0)