From 6ce21550727aeb57213a4b0c4818ff54bf420c62 Mon Sep 17 00:00:00 2001
From: unit-adm <314923187+mumtaz6@users.noreply.github.com>
Date: Sun, 4 Oct 2026 11:29:33 +0530
Subject: [PATCH 1/3] Put a node that answers back in the ring

A leader that shuts down takes itself out of the ring on its way out. The
next leader inherits that ring, and never saw the node fail, so when the
node came back and answered its pings, nothing changed from its point of
view: it never set the ring again, and the node stayed out for good. A
rolling restart that restarted the leader could leave it there.

The leader now puts back a node that answers its ping, isn't leaving and
isn't in the ring. Only one that answers: a leader that left and is gone
hasn't failed yet in the new leader's count, and putting it back would
send requests to it until it did.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 server/e2e/cluster_test.go        | 26 ++++++++++++++++++++++++++
 server/e2e/health_test.go         | 21 +++++++++++++++++++++
 server/internal/cluster_leader.go | 21 +++++++++++++++++++++
 3 files changed, 68 insertions(+)

diff --git a/server/e2e/cluster_test.go b/server/e2e/cluster_test.go
index 132c5ae..ffcf642 100644
--- a/server/e2e/cluster_test.go
+++ b/server/e2e/cluster_test.go
@@ -2590,3 +2590,29 @@ func TestClusterRestartedOwnerKeepsSubscriptions(t *testing.T) {
 	}
 	c.assertAlive(t, c.nodes, "after the restart")
 }
+
+// TestClusterGracefulRestartRejoins restarts the leader gracefully, then a
+// follower: each is back in the ring, and ready, soon after. A leader that
+// shuts down takes itself out of the ring; the next leader inherited that
+// ring, and never saw the node fail, so it never set the ring again when
+// the node came back.
+func TestClusterGracefulRestartRejoins(t *testing.T) {
+	c := startCluster(t, names...)
+	leader, err := c.waitLeader(c.nodes, 10*time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, name := range []string{leader, followerOf(leader)} {
+		n := c.node(name)
+		n.shutdown()
+		if err := n.start(); err != nil {
+			t.Fatal(err)
+		}
+		for _, m := range c.nodes {
+			waitReadyz(t, m, 30*time.Second)
+		}
+		if _, err := c.waitLeader(c.nodes, 10*time.Second); err != nil {
+			t.Fatalf("after %s's restart: %v", name, err)
+		}
+	}
+}
diff --git a/server/e2e/health_test.go b/server/e2e/health_test.go
index 3366bf4..2bfaed4 100644
--- a/server/e2e/health_test.go
+++ b/server/e2e/health_test.go
@@ -131,3 +131,24 @@ func TestHealthCluster(t *testing.T) {
 		}
 	}
 }
+
+// waitReadyz waits until n's /_readyz says it's ready, and returns what it
+// said before, each answer once.
+func waitReadyz(t *testing.T, n *clusterNode, timeout time.Duration) []string {
+	t.Helper()
+	var notReady []string
+	deadline := time.Now().Add(timeout)
+	for {
+		code, body := monitorGet(t, n.server, "/_readyz")
+		if code == http.StatusOK {
+			return notReady
+		}
+		if len(notReady) == 0 || notReady[len(notReady)-1] != body {
+			notReady = append(notReady, body)
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("%s not ready after %s: %v\nlogs:\n%s", n.name, timeout, notReady, n.logs.String())
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+}
diff --git a/server/internal/cluster_leader.go b/server/internal/cluster_leader.go
index d87bc12..b636253 100644
--- a/server/internal/cluster_leader.go
+++ b/server/internal/cluster_leader.go
@@ -206,6 +206,8 @@ func (c *Cluster) sendPings() {
 	// The ring version the followers saw the cluster route by: the lowest,
 	// as a switch only ever goes up.
 	routed := 0
+	// The nodes that answered this ping, and aren't leaving.
+	answered := make(map[string]bool)
 	for _, node := range c.nodes {
 		var pong ClusterPong
 		// A node that stalls without its connection failing does not answer:
@@ -220,6 +222,7 @@ func (c *Cluster) sendPings() {
 			RingVersion: c.getRingVersion()}, &pong, c.fo.heartBeat)
 		if err == nil {
 			node.setCapabilities(pong.NodeCapabilities)
+			answered[node.name] = !pong.Leaving
 			if v := pong.RingVersion; v != 0 && (routed == 0 || v < routed) {
 				routed = v
 			}
@@ -276,6 +279,24 @@ func (c *Cluster) sendPings() {
 		c.adoptRingVersion(v)
 	}
 
+	// The ring may lack a node that answers, which this leader never saw
+	// fail: a leader that left (shutting down) took itself out of the ring,
+	// and the next leader, inheriting that ring, sees the node answer when
+	// it is back, which is no change of its own. So a node that answers and
+	// isn't in the ring is put back. Only one that answers: one that left
+	// and is gone hasn't failed yet here, and is taken out as it fails.
+	if !rehash {
+		inRing := map[string]bool{}
+		for _, name := range c.getRingNodes() {
+			inRing[name] = true
+		}
+		for _, node := range live {
+			if answered[node.name] && !inRing[node.name] {
+				rehash = true
+			}
+		}
+	}
+
 	if rehash {
 		var activeNodes []string
 		for _, node := range live {

From 58b9ea5d68a186d75be4b3f69dfd8e558d623c7b Mon Sep 17 00:00:00 2001
From: unit-adm <314923187+mumtaz6@users.noreply.github.com>
Date: Sun, 4 Oct 2026 11:29:51 +0530
Subject: [PATCH 2/3] Backups off the cluster, reconciliation after a restore,
 and the runbooks

docs/backup-restore.md:

- A checkpoint describes itself (checkpoint.json: node, run, ring and
  engine versions, key ids, the copy's counts), and a node won't start at
  one into a running cluster unless its peers were restored from the same
  run, or it is started with -restored.
- The backup run (server/cmd/backup, deploy/kubernetes/backups.yaml)
  checkpoints every node under one run id, keeps the manifest in each
  checkpoint, and has each node upload its checkpoint, archived,
  compressed and encrypted with age, to S3 with Object Lock, locked and
  tagged by tier (deploy/aws: bucket, lifecycle, put-only writer, reader
  and escrow policies).
- -restored reconciles each topic with its other holders by message
  digests and counts, idempotently, before the node takes clients.
- The security journal: each security change a node makes is fsynced on
  the node, uploaded every 10 s, and replayed by -journal on a restore,
  so nothing revoked after the run comes back.
- The keyring is escrowed in Secrets Manager (backup escrow-keyring).
- The weekly restore test (backup verify, restore-test.yaml) opens each
  node's checkpoint of the newest run on a scratch server, checks it and
  a canary written before the run, and pushes its success;
  backup-alerts.yaml alerts on it.
- Runbooks A to D, a restore Job per node, backup runs and fetch -node.

The new cluster calls check their sender over TLS, as the others do. A
checkpoint counts memdb's records as it copies them: memdb's Size counts a
key's versions. storedOn, an e2e helper, reads until the relay is quiet:
it stopped at n messages, and missed some when a topic held more.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 deploy/aws/README.md                         | 105 +++++
 deploy/aws/escrow-policy.json                |  15 +
 deploy/aws/lifecycle.json                    |  15 +
 deploy/aws/reader-policy.json                |  27 ++
 deploy/aws/writer-policy.json                |  16 +
 deploy/kubernetes/backup-alerts.yaml         |  68 +++
 deploy/kubernetes/backups.yaml               |  57 +++
 deploy/kubernetes/restore-node-job.yaml      |  60 +++
 deploy/kubernetes/restore-test.yaml          |  80 ++++
 docs/README.md                               |   3 +
 docs/backup-restore.md                       | 198 +++++++++
 go.mod                                       |  23 +
 go.sum                                       |  50 +++
 server/cmd/backup/main.go                    | 209 +++++++++
 server/cmd/backup/tools.go                   | 315 ++++++++++++++
 server/cmd/backup/verify.go                  | 359 ++++++++++++++++
 server/e2e/backup_run_test.go                | 160 +++++++
 server/e2e/offsite_test.go                   | 268 ++++++++++++
 server/e2e/reconcile_test.go                 | 263 ++++++++++++
 server/e2e/restore_cluster_test.go           | 322 ++++++++++++++
 server/e2e/restore_test_run_test.go          | 107 +++++
 server/e2e/runbook_test.go                   | 126 ++++++
 server/internal/backup/archive.go            | 145 +++++++
 server/internal/backup/backup.go             | 119 ++++++
 server/internal/backup/backup_test.go        | 120 ++++++
 server/internal/backup/escrow.go             | 150 +++++++
 server/internal/backup/s3.go                 |  98 +++++
 server/internal/checkpoint.go                | 152 ++++++-
 server/internal/cluster.go                   |  33 +-
 server/internal/cluster_caps.go              |   2 +-
 server/internal/cluster_health.go            |   3 +
 server/internal/cluster_reconcile.go         | 425 +++++++++++++++++++
 server/internal/cluster_tls.go               |  28 ++
 server/internal/cluster_tls_test.go          |  26 +-
 server/internal/db/adapter.go                |  14 +-
 server/internal/db/checkpoint_info.go        | 105 +++++
 server/internal/db/unitdb/checkpoint.go      |  46 +-
 server/internal/db/unitdb/checkpoint_test.go |  36 +-
 server/internal/health.go                    |   3 +
 server/internal/metrics.go                   |   1 +
 server/internal/offsite.go                   | 411 ++++++++++++++++++
 server/internal/offsite_test.go              |  98 +++++
 server/internal/restore.go                   | 137 ++++++
 server/internal/revocation.go                |  19 +-
 server/internal/store/canary.go              |  56 +++
 server/internal/store/canary_test.go         |  20 +
 server/internal/store/checkpoint_test.go     |   7 +-
 server/internal/store/security.go            |  31 +-
 server/main.go                               |  24 ++
 49 files changed, 5113 insertions(+), 42 deletions(-)
 create mode 100644 deploy/aws/README.md
 create mode 100644 deploy/aws/escrow-policy.json
 create mode 100644 deploy/aws/lifecycle.json
 create mode 100644 deploy/aws/reader-policy.json
 create mode 100644 deploy/aws/writer-policy.json
 create mode 100644 deploy/kubernetes/backup-alerts.yaml
 create mode 100644 deploy/kubernetes/backups.yaml
 create mode 100644 deploy/kubernetes/restore-node-job.yaml
 create mode 100644 deploy/kubernetes/restore-test.yaml
 create mode 100644 docs/backup-restore.md
 create mode 100644 server/cmd/backup/main.go
 create mode 100644 server/cmd/backup/tools.go
 create mode 100644 server/cmd/backup/verify.go
 create mode 100644 server/e2e/backup_run_test.go
 create mode 100644 server/e2e/offsite_test.go
 create mode 100644 server/e2e/reconcile_test.go
 create mode 100644 server/e2e/restore_cluster_test.go
 create mode 100644 server/e2e/restore_test_run_test.go
 create mode 100644 server/e2e/runbook_test.go
 create mode 100644 server/internal/backup/archive.go
 create mode 100644 server/internal/backup/backup.go
 create mode 100644 server/internal/backup/backup_test.go
 create mode 100644 server/internal/backup/escrow.go
 create mode 100644 server/internal/backup/s3.go
 create mode 100644 server/internal/cluster_reconcile.go
 create mode 100644 server/internal/db/checkpoint_info.go
 create mode 100644 server/internal/offsite.go
 create mode 100644 server/internal/offsite_test.go
 create mode 100644 server/internal/restore.go
 create mode 100644 server/internal/store/canary.go
 create mode 100644 server/internal/store/canary_test.go

diff --git a/deploy/aws/README.md b/deploy/aws/README.md
new file mode 100644
index 0000000..e23826b
--- /dev/null
+++ b/deploy/aws/README.md
@@ -0,0 +1,105 @@
+# unitdb's backups on AWS
+
+What a cluster's off-site backups need on AWS (docs/backup-restore.md),
+made once per cluster by an AWS admin: a bucket with Object Lock, a
+put-only identity for the cluster (an access key in a Kubernetes Secret
+here; any of the AWS SDK's credential sources will do), and the backup key
+and the escrowed keyring in Secrets Manager, readable only by the weekly
+restore test's identity and named admins. A cluster per AWS account keeps
+staging and production apart.
+
+Below, `CLUSTER` is the cluster's name (`staging`, `production`): its
+prefix in the bucket and in the secrets' names. `BUCKET` is the bucket,
+`REGION` its region, `ACCOUNT` the account id. Replace them in the JSON
+files here before using them.
+
+## 1. The bucket: Object Lock, and a lifecycle by tier
+
+Object Lock can only be turned on when the bucket is made. There is no
+default retention: each object is locked in compliance mode by the node
+that uploads it, for as long as its tier is kept, so that daily runs can
+expire after a week while monthly ones are kept 13 months (server/internal/backup:
+daily 8 days, weekly 29, monthly and the journal 396). Compliance mode: not
+even the account's root user deletes a locked version before its date.
+
+```sh
+aws s3api create-bucket --bucket BUCKET --region REGION \
+  --create-bucket-configuration LocationConstraint=REGION \
+  --object-lock-enabled-for-bucket
+aws s3api put-public-access-block --bucket BUCKET \
+  --public-access-block-configuration BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true
+aws s3api put-bucket-encryption --bucket BUCKET \
+  --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"}}]}'
+aws s3api put-bucket-lifecycle-configuration --bucket BUCKET --lifecycle-configuration file://lifecycle.json
+```
+
+The lifecycle rules (`lifecycle.json`) expire each object by its `tier`
+tag once its lock has passed; the old versions go a day later.
+
+## 2. The cluster's identity: put only
+
+An IAM user, `unitdb-backup-writer-CLUSTER`, with `writer-policy.json`
+only: it puts objects, with their lock and tag, under `CLUSTER/`, and can
+neither read, list nor delete them. A compromised cluster can't read or
+erase its backups.
+
+```sh
+aws iam create-user --user-name unitdb-backup-writer-CLUSTER
+aws iam put-user-policy --user-name unitdb-backup-writer-CLUSTER \
+  --policy-name unitdb-backup-put-only --policy-document file://writer-policy.json
+aws iam create-access-key --user-name unitdb-backup-writer-CLUSTER
+kubectl -n unitdb create secret generic unitdb-backup-writer \
+  --from-literal=AWS_ACCESS_KEY_ID=<AccessKeyId> \
+  --from-literal=AWS_SECRET_ACCESS_KEY=<SecretAccessKey>
+```
+
+Check the policy before the first run, with the IAM policy simulator or a
+test upload: the nodes' uploads use exactly `s3:PutObject` (multipart
+included), `s3:PutObjectRetention`, `s3:PutObjectTagging` and, when an
+upload fails, `s3:AbortMultipartUpload`. The e2e tests run against MinIO as
+its root user, so they don't check the policy.
+
+Rotate the key by making a second one, updating the Secret, restarting the
+nodes, and deleting the first.
+
+## 3. The backup key
+
+Archives are encrypted with [age](https://age-encryption.org) to the backup
+key's public half; the private half is only in Secrets Manager.
+
+```sh
+age-keygen -o backup-key.txt     # prints the public key: age1...
+aws secretsmanager create-secret --region REGION \
+  --name unitdb/CLUSTER/backup-key --secret-string file://backup-key.txt
+shred -u backup-key.txt
+```
+
+The public half goes into the nodes' environment as `BACKUP_AGE_RECIPIENT`
+(docs/backup-restore.md lists the nodes' settings).
+
+## 4. The keyring's escrow
+
+A backup restores only with the keyring of its time. Put the keyring in
+escrow before every change of the cluster's `UNITDB_KEYRING`, with an
+admin's identity (`escrow-policy.json`):
+
+```sh
+BACKUP_CLUSTER=CLUSTER backup escrow-keyring -keyring keyring.json
+```
+
+It keeps every key it ever held: a key gone from the keyring stays as a
+`read` key, and a key id with another key is refused. Only then update
+the cluster's Secret.
+
+## 5. Readers
+
+`reader-policy.json` reads the bucket under `CLUSTER/` and the two secrets:
+for the weekly restore test's identity and the named admins' roles,
+nothing else. A restore (docs/backup-restore.md):
+
+```sh
+export BACKUP_S3_BUCKET=BUCKET BACKUP_CLUSTER=CLUSTER
+backup fetch -run <run id> -out restore/run          # each node's checkpoint
+backup fetch-journal -since <run id> -out restore/journal
+# then each node: -db_path=restore/run/<node> -restored -journal=restore/journal
+```
diff --git a/deploy/aws/escrow-policy.json b/deploy/aws/escrow-policy.json
new file mode 100644
index 0000000..bb15f13
--- /dev/null
+++ b/deploy/aws/escrow-policy.json
@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "UnitdbKeyringEscrow",
+      "Effect": "Allow",
+      "Action": [
+        "secretsmanager:GetSecretValue",
+        "secretsmanager:PutSecretValue",
+        "secretsmanager:CreateSecret"
+      ],
+      "Resource": "arn:aws:secretsmanager:REGION:ACCOUNT:secret:unitdb/CLUSTER/keyring-*"
+    }
+  ]
+}
diff --git a/deploy/aws/lifecycle.json b/deploy/aws/lifecycle.json
new file mode 100644
index 0000000..d48a2cf
--- /dev/null
+++ b/deploy/aws/lifecycle.json
@@ -0,0 +1,15 @@
+{
+  "Rules": [
+    {"ID": "daily", "Status": "Enabled", "Filter": {"Tag": {"Key": "tier", "Value": "daily"}},
+     "Expiration": {"Days": 8}, "NoncurrentVersionExpiration": {"NoncurrentDays": 1}},
+    {"ID": "weekly", "Status": "Enabled", "Filter": {"Tag": {"Key": "tier", "Value": "weekly"}},
+     "Expiration": {"Days": 29}, "NoncurrentVersionExpiration": {"NoncurrentDays": 1}},
+    {"ID": "monthly", "Status": "Enabled", "Filter": {"Tag": {"Key": "tier", "Value": "monthly"}},
+     "Expiration": {"Days": 396}, "NoncurrentVersionExpiration": {"NoncurrentDays": 1}},
+    {"ID": "journal", "Status": "Enabled", "Filter": {"Tag": {"Key": "tier", "Value": "journal"}},
+     "Expiration": {"Days": 396}, "NoncurrentVersionExpiration": {"NoncurrentDays": 1}},
+    {"ID": "delete-markers", "Status": "Enabled", "Filter": {},
+     "Expiration": {"ExpiredObjectDeleteMarker": true},
+     "AbortIncompleteMultipartUpload": {"DaysAfterInitiation": 2}}
+  ]
+}
diff --git a/deploy/aws/reader-policy.json b/deploy/aws/reader-policy.json
new file mode 100644
index 0000000..892cade
--- /dev/null
+++ b/deploy/aws/reader-policy.json
@@ -0,0 +1,27 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "UnitdbBackupRead",
+      "Effect": "Allow",
+      "Action": ["s3:GetObject", "s3:GetObjectVersion"],
+      "Resource": "arn:aws:s3:::BUCKET/CLUSTER/*"
+    },
+    {
+      "Sid": "UnitdbBackupList",
+      "Effect": "Allow",
+      "Action": "s3:ListBucket",
+      "Resource": "arn:aws:s3:::BUCKET",
+      "Condition": {"StringLike": {"s3:prefix": ["CLUSTER/*"]}}
+    },
+    {
+      "Sid": "UnitdbBackupKeys",
+      "Effect": "Allow",
+      "Action": "secretsmanager:GetSecretValue",
+      "Resource": [
+        "arn:aws:secretsmanager:REGION:ACCOUNT:secret:unitdb/CLUSTER/backup-key-*",
+        "arn:aws:secretsmanager:REGION:ACCOUNT:secret:unitdb/CLUSTER/keyring-*"
+      ]
+    }
+  ]
+}
diff --git a/deploy/aws/writer-policy.json b/deploy/aws/writer-policy.json
new file mode 100644
index 0000000..d0bc8c6
--- /dev/null
+++ b/deploy/aws/writer-policy.json
@@ -0,0 +1,16 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "UnitdbBackupPutOnly",
+      "Effect": "Allow",
+      "Action": [
+        "s3:PutObject",
+        "s3:PutObjectRetention",
+        "s3:PutObjectTagging",
+        "s3:AbortMultipartUpload"
+      ],
+      "Resource": "arn:aws:s3:::BUCKET/CLUSTER/*"
+    }
+  ]
+}
diff --git a/deploy/kubernetes/backup-alerts.yaml b/deploy/kubernetes/backup-alerts.yaml
new file mode 100644
index 0000000..db04733
--- /dev/null
+++ b/deploy/kubernetes/backup-alerts.yaml
@@ -0,0 +1,68 @@
+# Alerts on unitdb's backups (docs/backup-restore.md), for the Prometheus
+# Operator. The metrics: each node's /_metrics on its monitor
+# port (checkpoints, uploads, the security journal), the Pushgateway (the
+# weekly restore test), and kube-state-metrics (the backup job).
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: unitdb-backups
+  namespace: unitdb
+spec:
+  groups:
+    - name: unitdb-backups
+      rules:
+        # A node's last checkpoint is more than a day old: the daily run
+        # (backups.yaml) didn't reach it.
+        - alert: UnitdbCheckpointStale
+          expr: time() - min by (pod) (unitdb_checkpoint_last_success_timestamp_seconds) > 26 * 3600
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "{{ $labels.pod }}: no checkpoint for over 26 h"
+        - alert: UnitdbCheckpointFailing
+          expr: increase(unitdb_checkpoint_failures_total[2h]) > 0
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.pod }}: a checkpoint failed (see its logs, context checkpoint)"
+        # A node's last upload to the bucket is more than a day old.
+        - alert: UnitdbBackupUploadStale
+          expr: time() - min by (pod) (unitdb_backup_upload_last_success_timestamp_seconds) > 26 * 3600
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "{{ $labels.pod }}: no backup uploaded for over 26 h"
+        - alert: UnitdbBackupJobFailed
+          expr: kube_job_status_failed{namespace="unitdb", job_name=~"unitdb-backup-.*"} > 0
+          labels:
+            severity: critical
+          annotations:
+            summary: "the backup run {{ $labels.job_name }} failed: a node's checkpoint or upload (see the Job's output, its manifest)"
+        # Security changes not in the bucket: a whole-cluster restore now
+        # would bring back what they revoked.
+        - alert: UnitdbSecurityJournalLagging
+          expr: max by (pod) (unitdb_security_journal_lag_seconds) > 300
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "{{ $labels.pod }}: security changes over 5 min old aren't in the bucket"
+        - alert: UnitdbSecurityJournalNotRecording
+          expr: increase(unitdb_security_journal_record_errors_total[10m]) > 0
+          labels:
+            severity: critical
+          annotations:
+            summary: "{{ $labels.pod }}: a security change couldn't be written to the journal"
+        # The weekly restore test hasn't passed for over 8 days, or never
+        # has: this fires from deployment until its first pass.
+        - alert: UnitdbRestoreTestStale
+          expr: |
+            (time() - max by (cluster) (unitdb_backup_restore_verified_timestamp_seconds) > 8 * 86400)
+              or absent(unitdb_backup_restore_verified_timestamp_seconds)
+          for: 1h
+          labels:
+            severity: critical
+          annotations:
+            summary: "no backup has been proven to restore for over 8 days (restore-test.yaml)"
diff --git a/deploy/kubernetes/backups.yaml b/deploy/kubernetes/backups.yaml
new file mode 100644
index 0000000..3be809f
--- /dev/null
+++ b/deploy/kubernetes/backups.yaml
@@ -0,0 +1,57 @@
+# unitdb's daily backup run (docs/backup-restore.md): at 21:30 UTC, the
+# backup command (server/cmd/backup, in the unitdb image) has every
+# node take a checkpoint of one run, one node after another, each with up to
+# three tries, into checkpoints/ckpt-<run>-<node> on its volume, and keeps
+# the run's manifest.json (each node's checkpoint.json, or why it failed) in
+# every node's checkpoint. Then each node uploads its checkpoint, encrypted,
+# and the manifest, to the bucket (-upload; deploy/aws/README.md). A node
+# that failed either fails the job: a restore needs every node of one run.
+# A volume snapshot of the nodes' claims, if you take one, goes after it:
+# only the checkpoints in it restore, not the live store beside them.
+#
+# The job's exit status is its metric: alert on a failed Job, or on
+# kube_cronjob_status_last_successful_time older than 26 h.
+#
+# Restoring: docs/backup-restore.md. One node lost: an empty claim, not a
+# checkpoint. The whole cluster: every node at its checkpoint of one run.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: unitdb-backup
+  namespace: unitdb
+spec:
+  schedule: "30 21 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      # The command retries each node itself; a second Job would start a new
+      # run.
+      backoffLimit: 0
+      template:
+        spec:
+          restartPolicy: Never
+          # No Service links in the environment: a Service named web would set
+          # WEB_PORT=tcp://..., which unitdb reads as its own port.
+          enableServiceLinks: false
+          containers:
+            - name: backup
+              image: unitdb:latest
+              command: ["/backup"]
+              args:
+                - -upload
+                - -nodes
+                - http://unitdb-0.unitdb.unitdb.svc.cluster.local:7374,http://unitdb-1.unitdb.unitdb.svc.cluster.local:7374,http://unitdb-2.unitdb.unitdb.svc.cluster.local:7374
+              env:
+                - name: CHECKPOINT_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: unitdb
+                      key: CHECKPOINT_TOKEN
+              resources:
+                requests:
+                  cpu: 10m
+                  memory: 16Mi
+                limits:
+                  memory: 64Mi
diff --git a/deploy/kubernetes/restore-node-job.yaml b/deploy/kubernetes/restore-node-job.yaml
new file mode 100644
index 0000000..7257c03
--- /dev/null
+++ b/deploy/kubernetes/restore-node-job.yaml
@@ -0,0 +1,60 @@
+# One node's part of a whole-cluster restore (docs/backup-restore.md,
+# runbook B): on the node's new, empty claim, its checkpoint of the run,
+# decrypted, where the node's store is (/var/udb/data/unitdb), and the
+# security journal since the run (/var/udb/data/restore-journal). Then the
+# node starts with -restored and -journal (runbook B, step 6).
+#
+# Per node: replace NODE (unitdb-0, unitdb-1, unitdb-2) and RUN (the run's
+# id, from `backup` runs listed in the bucket), and apply:
+#   sed -e s/NODE/unitdb-0/g -e s/RUN/20261004T213000Z/g restore-node-job.yaml | kubectl apply -f -
+#
+# It runs as the user unitdb runs as: the files it writes are the node's
+# store. If unitdb runs as another user than root, give the Job the same
+# securityContext.
+#
+# It reads the bucket and the backup key, so it needs the reader's identity
+# in the unitdb namespace for the restore only: the unitdb-backup-reader
+# Secret, made at runbook B's step 3 and deleted at its end.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: unitdb-restore-NODE
+  namespace: unitdb
+spec:
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      enableServiceLinks: false
+      initContainers:
+        - name: checkpoint
+          image: unitdb:latest
+          command: ["/backup"]
+          args: ["fetch", "-run=RUN", "-node=NODE", "-out=/var/udb/data/unitdb"]
+          envFrom:
+            - secretRef:
+                name: unitdb-backup-reader
+          env: &restoreEnv
+            - name: BACKUP_S3_BUCKET
+              value: unitdb-backups-production
+            - name: BACKUP_S3_REGION
+              value: <your region>
+            - name: BACKUP_CLUSTER
+              value: production
+          volumeMounts: &restoreMounts
+            - name: data
+              mountPath: /var/udb/data
+      containers:
+        - name: journal
+          image: unitdb:latest
+          command: ["/backup"]
+          args: ["fetch-journal", "-since=RUN", "-out=/var/udb/data/restore-journal"]
+          envFrom:
+            - secretRef:
+                name: unitdb-backup-reader
+          env: *restoreEnv
+          volumeMounts: *restoreMounts
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: data-NODE
diff --git a/deploy/kubernetes/restore-test.yaml b/deploy/kubernetes/restore-test.yaml
new file mode 100644
index 0000000..c122b96
--- /dev/null
+++ b/deploy/kubernetes/restore-test.yaml
@@ -0,0 +1,80 @@
+# unitdb's weekly restore test (docs/backup-restore.md): the backup
+# command's verify downloads the newest complete run from the bucket,
+# decrypts it, and opens each node's checkpoint on a scratch server in this
+# pod (no cluster, no clients), with the escrowed keyring: it must open and
+# be ready, hold what the manifest says, and hold the run's canary. It
+# pushes unitdb_backup_restore_verified_timestamp_seconds to the Pushgateway
+# only if every node passes; backup-alerts.yaml fires when that is more than
+# 8 days old.
+#
+# Its own namespace and identity: the reader's (deploy/aws/README.md, step
+# 5), the only one besides named admins that may read the bucket, the
+# backup key and the escrowed keyring. Nothing in the unitdb namespace can.
+#
+# Before applying:
+#   kubectl create namespace unitdb-restore-test
+#   kubectl -n unitdb-restore-test create secret generic unitdb-backup-reader \
+#     --from-literal=AWS_ACCESS_KEY_ID=<the reader's> \
+#     --from-literal=AWS_SECRET_ACCESS_KEY=<the reader's>
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: unitdb-restore-test
+  namespace: unitdb-restore-test
+spec:
+  schedule: "30 23 * * 6" # Saturdays 23:30 UTC, after that day's run
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      activeDeadlineSeconds: 21600
+      template:
+        spec:
+          restartPolicy: Never
+          enableServiceLinks: false
+          containers:
+            - name: verify
+              image: unitdb:latest
+              command: ["/backup"]
+              args:
+                - verify
+                - -run=latest
+                - -server=/unitdb
+                - -work=/scratch
+                - -pushgateway=http://pushgateway.monitoring.svc.cluster.local:9091
+              env:
+                - name: BACKUP_S3_BUCKET
+                  value: unitdb-backups-production
+                - name: BACKUP_S3_REGION
+                  value: <your region>
+                - name: BACKUP_CLUSTER
+                  value: production
+                - name: AWS_ACCESS_KEY_ID
+                  valueFrom:
+                    secretKeyRef:
+                      name: unitdb-backup-reader
+                      key: AWS_ACCESS_KEY_ID
+                - name: AWS_SECRET_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: unitdb-backup-reader
+                      key: AWS_SECRET_ACCESS_KEY
+              # One scratch server at a time; its memory store is the
+              # nodes' (mem_size, 500 MB).
+              resources:
+                requests:
+                  cpu: 500m
+                  memory: 1Gi
+                limits:
+                  memory: 2Gi
+              volumeMounts:
+                - name: scratch
+                  mountPath: /scratch
+          volumes:
+            # A run, decrypted: every node's checkpoint. Size it to three
+            # nodes' stores.
+            - name: scratch
+              emptyDir:
+                sizeLimit: 150Gi
diff --git a/docs/README.md b/docs/README.md
index 7d04fa4..bf8c035 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -8,5 +8,8 @@
   changes.
 - [Message log replication](message-log-replication.md): replicas of stored
   messages and session logs, hints, and rebuilding a node that lost its disk.
+- [Backup and restore](backup-restore.md): checkpoints, the backup run, copies
+  off the cluster, the security journal, reconciliation after a restore, the
+  weekly restore test, and the runbooks.
 - [Rolling deploys](rolling-deploys.md): upgrading a cluster node by node, and
   the maintenance-window upgrade from v0.3.0.
diff --git a/docs/backup-restore.md b/docs/backup-restore.md
new file mode 100644
index 0000000..76442ea
--- /dev/null
+++ b/docs/backup-restore.md
@@ -0,0 +1,198 @@
+# Backup and restore
+
+How a unitdb cluster's data is backed up and restored: checkpoints, the
+backup run and its manifest, copies off the cluster, the security journal,
+reconciliation after a restore, the weekly restore test, and the runbooks.
+Everything here is built; the deploy examples are in `deploy/kubernetes` and
+`deploy/aws`.
+
+## What a backup is
+
+- **A checkpoint** is a copy of a node's store that opens as the store was
+  at one moment (`server/internal/db/unitdb/checkpoint.go`). A copy of a
+  running store's files isn't one: they are copied at different moments,
+  and the WAL isn't fsynced, so a volume snapshot is like a power cut. The
+  adapter holds writes back, for up to about 1.5 s, until the DB has written
+  out the latest; then it syncs, copies the DB's files, and copies memdb's
+  records (not its files) into a fresh memdb. Reads go on. Last it writes
+  `checkpoint.json`: the node, the backup run, the time, the ring and
+  engine versions, the keyring's key ids (never keys), and the copy's
+  counts. A checkpoint without it is incomplete.
+- `POST /_checkpoint` on the monitor port (`monitor_listen`), with
+  `Authorization: Bearer <CHECKPOINT_TOKEN>`, takes one into `CHECKPOINT_DIR`;
+  off unless both are set. One runs at a time, one per
+  `CHECKPOINT_MIN_MINUTES` (10); the newest `CHECKPOINT_KEEP` (3) are kept.
+- **A backup run** (`server/cmd/backup`, `deploy/kubernetes/backups.yaml`)
+  checkpoints every node under one run id (`POST /_checkpoint?run=<id>`,
+  into `ckpt-<run>-<node>`), one node after another, each tried three
+  times, and keeps the run's `manifest.json` (each node's
+  `checkpoint.json`, or why it failed) in every node's checkpoint
+  (`PUT /_checkpoint/manifest`). A node that failed fails the run: a
+  restore needs every node of one run.
+- Just before its checkpoint of a run, a node writes **a canary**, the run's
+  id, into its store as a message and as a memdb record; the restore test
+  reads it back.
+
+## Copies off the cluster
+
+With `-upload`, the backup run has each node upload its own checkpoint (the
+checkpoints are on each node's volume): `POST /_checkpoint/upload?run=<id>`
+streams it, archived (tar), compressed (zstd) and encrypted with
+[age](https://age-encryption.org) to the backup key's public half, to S3,
+with the manifest (`server/internal/backup`):
+
+```
+<cluster>/<run>/manifest.json
+<cluster>/<run>/<node>.tar.zst.age
+<cluster>/journal/<date>/<node>-<time>.jsonl
+```
+
+- The bucket has Object Lock. Each object is locked in compliance mode by
+  its uploader for as long as its tier is kept, and tagged with the tier;
+  lifecycle rules expire it after: daily runs 8 days, weekly (Sundays) 29,
+  monthly (the 1st) and the journal 396.
+- The cluster's identity may only put objects: it can neither read, list
+  nor delete them, so a compromised cluster can't read or erase its
+  backups. The private half of the backup key is kept apart, readable by the
+  restore test's identity and named admins only.
+- **The keyring is escrowed** (`backup escrow-keyring`, in AWS Secrets
+  Manager): a backup restores only with the keyring of its time. Put it in
+  escrow before every change of the keyring; the escrow keeps every key it
+  ever held, retired ones as `read` keys, and refuses a key id with another
+  key.
+
+A node's settings (`server/internal/backup/backup.go`):
+
+| Variable | |
+| --- | --- |
+| `BACKUP_S3_BUCKET` | the bucket; uploads are off without it |
+| `BACKUP_S3_REGION` | the region; the AWS SDK's (`AWS_REGION`, a profile) if unset |
+| `BACKUP_S3_ENDPOINT` | another S3 (MinIO), path-style |
+| `BACKUP_CLUSTER` | the cluster's prefix in the bucket |
+| `BACKUP_AGE_RECIPIENT` | the backup key's public half (`age1...`) |
+| `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | the put-only identity, or any of the AWS SDK's credential sources |
+| `SECURITY_JOURNAL_DIR` | where the security journal waits for its upload |
+
+`deploy/aws/README.md` makes the bucket, its lifecycle, the identities and
+the keys.
+
+## The security journal
+
+A backup can't know what was revoked after its run. So each change of the
+security state a node makes (a revoked client id or topic key, a
+contract's not-before time) is written as a JSON line to
+`SECURITY_JOURNAL_DIR`, and fsynced, before it takes effect, and uploaded in
+a batch every 10 s (`server/internal/offsite.go`). A batch is deleted from
+the node only once it is in the bucket, so a restart uploads what was left.
+`unitdb_security_journal_lag_seconds` is the age of the oldest line not up
+yet.
+
+A restore replays it: `-restored -journal <dir>`, with the journal since the
+run (`backup fetch-journal`), before the node takes clients. Lines merge as
+the changes did, so lines older than the checkpoint, or replayed twice,
+change nothing. Nodes never read the bucket.
+
+## Restoring: one node, or the whole cluster
+
+- **One node lost, the others up:** don't restore it from a checkpoint. Start
+  it with an empty store: it copies the topics it holds from the other nodes
+  (`Cluster.rebuild`), up to the moment, and takes no clients until it has
+  (`/_readyz` says "catching up").
+- A node won't start at a checkpoint (a `db_path` with `checkpoint.json`)
+  while a peer answers that wasn't itself started at a checkpoint of the
+  same run (`Cluster.StartedFrom`): it would never get what was written
+  since. It says to start empty, or with `-restored`.
+- **The whole cluster lost:** start every node at its checkpoint of one run,
+  with `-restored`. Each reconciles every topic it holds with the topic's
+  other live holders before it takes clients (`/_readyz` says
+  "reconciling"; `server/internal/cluster_reconcile.go`): both list the
+  topic's messages as digests, `SHA-256(contract, topic, expiry, payload)`,
+  with how many times each occurs, and each gets what it holds fewer of. A
+  message one node took in the seconds between two nodes' checkpoints is
+  then on both; two identical publishes stay two. A message is sent with
+  how many its sender holds, and the receiver stores only what it lacks of
+  that count, so two restored nodes reconciling a topic with each other at
+  once settle on the larger count, not the sum. Hints are dropped: the
+  restored nodes are reconciled instead. Once done, `checkpoint.json`
+  becomes `restored-from.json`, and later restarts are ordinary ones.
+- `-restored` also brings one node at its checkpoint into a running
+  cluster: it catches up with the others the same way.
+
+| Failure | Data lost | Back in |
+| --- | --- | --- |
+| One node | none: rebuilt from the others | minutes |
+| The whole cluster | since the last run (a day, with one run a day); revocations since are replayed | under an hour, plus the download |
+
+## The weekly restore test
+
+`backup verify` (`deploy/kubernetes/restore-test.yaml`), with a reader's
+identity, downloads the newest complete run, decrypts it, and opens each
+node's checkpoint on a scratch server of its own (`-db_path` at it, no
+cluster, no clients), with the escrowed keyring. Each must be ready (its
+store probe writes and reads back), hold what the manifest says within 10%
+(what expired since), and hold the run's canary; and the escrow must hold
+every key id the run names. Only then it pushes
+`unitdb_backup_restore_verified_timestamp_seconds` to a Pushgateway.
+
+`deploy/kubernetes/backup-alerts.yaml` alerts when a checkpoint or an
+upload is over 26 h old, checkpoints fail, the backup job fails, the
+journal is over 5 minutes behind or fails to record, or the restore test
+hasn't passed in 8 days.
+
+## Runbooks
+
+For a cluster deployed as a StatefulSet `unitdb` in a namespace `unitdb`,
+with claims `data-unitdb-N`, as the examples in `deploy/kubernetes`; adjust
+the names. `backup` is `server/cmd/backup`. The e2e tests rehearse them:
+`TestRestoreLostNodeStartsEmpty` (A) and `TestRunbookWholeClusterRestore`
+(B, against MinIO).
+
+### A: one node lost
+
+1. Check that it is one node: the others answer `/_readyz`.
+2. Give it a new, empty claim; the StatefulSet makes the pod again:
+   ```sh
+   kubectl -n unitdb delete pvc data-unitdb-N --wait=false
+   kubectl -n unitdb delete pod unitdb-N
+   ```
+3. Wait for `/_readyz`: "catching up" until it has its topics, then ready.
+
+### B: the whole cluster lost
+
+1. **Stop what is left:** `kubectl -n unitdb scale statefulset unitdb
+   --replicas=0`. Snapshot a damaged volume first if it still exists.
+2. **The keyring:** if its Secret is gone, make it again from the escrow
+   (runbook C). Without the escrow there is no restore.
+3. **The reader's identity, for the restore only:** a Secret
+   `unitdb-backup-reader` with an admin's reader keys; deleted at step 9.
+4. **The run:** `backup runs` lists them, newest first; take the newest
+   `complete` one. `backup verify -run <run>` checks it first.
+5. **New claims, with the run:** delete the old claims, make new empty ones,
+   and fill each with its node's checkpoint and the journal since the run
+   (`deploy/kubernetes/restore-node-job.yaml`, per node, `NODE` and `RUN`
+   filled in). The Job runs as the user unitdb runs as: the files it writes
+   are the node's store.
+6. **Start every node restored:** add `-restored` and
+   `-journal=/var/udb/data/restore-journal` to the StatefulSet's args, and
+   scale it up. Each replays the journal, then reconciles (`/_readyz`:
+   "reconciling"); its log says `reconciled after a restore` with
+   `"failed":0`.
+7. **Take the flags out** once all are ready; the rolling restart is an
+   ordinary one. `-restored` on a node that isn't at a checkpoint is
+   refused.
+8. **Check:** every node ready, `unitdb_reconcile_failures_total` 0, the
+   data up to the run there, nothing revoked since back.
+9. **Clean up, and a new baseline:** delete the reader's Secret and the
+   restore Jobs, and run a backup now
+   (`kubectl create job --from=cronjob/unitdb-backup ...`).
+
+### C: the keyring lost
+
+There is no copy but the escrow. Read it with an admin's identity, make the
+cluster's Secret again from it (`UNITDB_KEYRING`, and a new
+`CHECKPOINT_TOKEN`), and restart the nodes.
+
+### D: one node at its checkpoint into the running cluster
+
+Start it with `-db_path` at its checkpoint and `-restored`: it reconciles
+with the running nodes before it takes clients. Then take `-restored` out.
diff --git a/go.mod b/go.mod
index f04d5b5..4b762e4 100644
--- a/go.mod
+++ b/go.mod
@@ -3,9 +3,16 @@ module github.com/unit-io/unitdb
 go 1.26.0
 
 require (
+	filippo.io/age v1.3.2
 	github.com/DisposaBoy/JsonConfigReader v0.0.0-20201129172854-99cf318d67e7
+	github.com/aws/aws-sdk-go-v2 v1.47.1
+	github.com/aws/aws-sdk-go-v2/config v1.33.6
+	github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.13
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.50.1
 	github.com/golang/snappy v1.0.0
 	github.com/gorilla/websocket v1.5.3
+	github.com/klauspost/compress v1.20.1
 	github.com/rs/zerolog v1.35.1
 	github.com/unit-io/bpool v0.1.0
 	golang.org/x/crypto v0.57.0
@@ -15,6 +22,22 @@ require (
 )
 
 require (
+	filippo.io/hpke v0.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.20.6 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect
+	github.com/aws/smithy-go v1.28.1 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	golang.org/x/sys v0.48.0 // indirect
diff --git a/go.sum b/go.sum
index 5929be2..3c0a2d1 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,51 @@
+c2sp.org/CCTV/age v0.0.0-20260829155415-4448f2097b2d h1:Blprhc2SbChNZtWcU+BLTM4YdoqYAS9V7cJgOwJKyAs=
+c2sp.org/CCTV/age v0.0.0-20260829155415-4448f2097b2d/go.mod h1:SrHC2C7r5GkDk8R+NFVzYy/sdj0Ypg9htaPXQq5Cqeo=
+filippo.io/age v1.3.2 h1:r6RSZLFSMm6rzKepZ7ZAYkKCu14f3/Me8c7uKYh7C8c=
+filippo.io/age v1.3.2/go.mod h1:TH/Yr2sSRhCKbaH4XPxpUV0Us8Gv6txYUpiZQWz8Evk=
+filippo.io/hpke v0.4.0 h1:p575VVQ6ted4pL+it6M00V/f2qTZITO0zgmdKCkd5+A=
+filippo.io/hpke v0.4.0/go.mod h1:EmAN849/P3qdeK+PCMkDpDm83vRHM5cDipBJ8xbQLVY=
 github.com/DisposaBoy/JsonConfigReader v0.0.0-20201129172854-99cf318d67e7 h1:AJKJCKcb/psppPl/9CUiQQnTG+Bce0/cIweD5w5Q7aQ=
 github.com/DisposaBoy/JsonConfigReader v0.0.0-20201129172854-99cf318d67e7/go.mod h1:GCzqZQHydohgVLSIqRKZeTt8IGb1Y4NaFfim3H40uUI=
+github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw=
+github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
+github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug=
+github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4=
+github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.13 h1:wO7TVbywHwdpHLUiX6DnmP2RDYOACVeJCb6zMfSFViU=
+github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.13/go.mod h1:Zc9r0r7wMid/NkbsLrkGxe5vZufWyP0CiC2dDXZ8ldk=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.50.1 h1:xYoGDAZtoSXI5wOfjv1jzG1AUOdXZthz4YL9DFvunrQ=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.50.1/go.mod h1:dgXxccOMNsXm/eOkrQbBfxm4a6H8IiRphA7z69RG8hM=
+github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU=
+github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0=
+github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM=
+github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ=
+github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
@@ -16,6 +62,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
+github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
 github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -43,6 +91,8 @@ golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
 golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
+golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
 golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
 golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
diff --git a/server/cmd/backup/main.go b/server/cmd/backup/main.go
new file mode 100644
index 0000000..d817d59
--- /dev/null
+++ b/server/cmd/backup/main.go
@@ -0,0 +1,209 @@
+// Command backup is unitdb's backup run (docs/backup-restore.md): it
+// checkpoints every node of a cluster, one after another, under one run id,
+// and writes the run's manifest into every node's checkpoint.
+//
+//	CHECKPOINT_TOKEN=... backup -nodes http://unitdb-0.unitdb:7374,http://unitdb-1.unitdb:7374,...
+//
+// A checkpoint holds a node's writes back for up to about 1.5 s, so nodes go
+// one at a time: the cluster pauses one node at once. Each node is tried
+// -tries times. A node whose checkpoint fails is named in the manifest, the
+// run is marked incomplete, and the command exits 1, after the others: a
+// restore needs every node of one run. With -upload, each node then sends
+// its checkpoint, with the manifest, to the bucket; a failed upload
+// fails the command too. It prints the manifest.
+//
+// The other commands are an operator's, or the restore test's, with a
+// reader's identity (tools.go): fetch, fetch-journal and escrow-keyring.
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+)
+
+// Manifest lists a backup run's checkpoints, one per node.
+type Manifest struct {
+	// Run is the run's id: when it began, in UTC.
+	Run string `json:"run"`
+	// Complete is set if every node's checkpoint was taken.
+	Complete bool      `json:"complete"`
+	Nodes    []NodeRun `json:"nodes"`
+	Finished time.Time `json:"finished"`
+}
+
+// NodeRun is one node's part of a run: its checkpoint, as the node
+// described it (POST /_checkpoint's answer), or why there is none.
+type NodeRun struct {
+	URL        string          `json:"url"`
+	Checkpoint json.RawMessage `json:"checkpoint,omitempty"`
+	Error      string          `json:"error,omitempty"`
+	// ManifestError is why the manifest couldn't be kept on the node.
+	ManifestError string `json:"manifest_error,omitempty"`
+	// Upload is the node's answer to the upload, or UploadError why it
+	// failed; neither is in the manifest the nodes keep.
+	Upload      json.RawMessage `json:"upload,omitempty"`
+	UploadError string          `json:"upload_error,omitempty"`
+}
+
+func main() {
+	if len(os.Args) > 1 {
+		if cmd, ok := tools[os.Args[1]]; ok {
+			if err := cmd(os.Args[2:]); err != nil {
+				fmt.Fprintf(os.Stderr, "backup %s: %v\n", os.Args[1], err)
+				os.Exit(1)
+			}
+			return
+		}
+	}
+	nodes := flag.String("nodes", os.Getenv("UNITDB_MONITOR_URLS"), "The nodes' monitor URLs, comma-separated, as http://unitdb-0.unitdb:7374.")
+	tries := flag.Int("tries", 3, "Attempts per node.")
+	wait := flag.Duration("retry_wait", 10*time.Second, "Time between a node's attempts.")
+	timeout := flag.Duration("timeout", 2*time.Minute, "Longest a node's checkpoint may take.")
+	upload := flag.Bool("upload", false, "Have each node upload its checkpoint and the manifest to the bucket (BACKUP_S3_* on the nodes).")
+	uploadTimeout := flag.Duration("upload_timeout", time.Hour, "Longest a node's upload may take.")
+	flag.Parse()
+	token := os.Getenv("CHECKPOINT_TOKEN")
+	if *nodes == "" || token == "" {
+		fmt.Fprintln(os.Stderr, "backup: -nodes (or UNITDB_MONITOR_URLS) and CHECKPOINT_TOKEN are needed")
+		os.Exit(2)
+	}
+	r := runner{token: token, tries: *tries, wait: *wait, client: &http.Client{Timeout: *timeout}}
+	m := r.run(time.Now().UTC(), strings.Split(*nodes, ","))
+	uploaded := true
+	if *upload && m.Complete {
+		r.client = &http.Client{Timeout: *uploadTimeout}
+		uploaded = r.uploadAll(m)
+	}
+	out, _ := json.MarshalIndent(m, "", "  ")
+	fmt.Println(string(out))
+	if !m.Complete || !uploaded {
+		os.Exit(1)
+	}
+}
+
+type runner struct {
+	token  string
+	tries  int
+	wait   time.Duration
+	client *http.Client
+}
+
+// run takes a backup run that began at start.
+func (r *runner) run(start time.Time, urls []string) *Manifest {
+	m := &Manifest{Run: start.Format("20060102T150405Z"), Complete: true}
+	for _, u := range urls {
+		u = strings.TrimRight(strings.TrimSpace(u), "/")
+		if u == "" {
+			continue
+		}
+		n := NodeRun{URL: u}
+		var err error
+		for i := 0; i < r.tries; i++ {
+			if i > 0 {
+				time.Sleep(r.wait)
+			}
+			if n.Checkpoint, err = r.checkpoint(u, m.Run); err == nil {
+				break
+			}
+		}
+		if err != nil {
+			n.Error = err.Error()
+			m.Complete = false
+			fmt.Fprintf(os.Stderr, "backup: run %s: %s: %v\n", m.Run, u, err)
+		}
+		m.Nodes = append(m.Nodes, n)
+	}
+	m.Finished = time.Now().UTC()
+	body, _ := json.MarshalIndent(m, "", "  ")
+	for i := range m.Nodes {
+		n := &m.Nodes[i]
+		if n.Error != "" {
+			continue
+		}
+		if err := r.putManifest(n.URL, m.Run, body); err != nil {
+			n.ManifestError = err.Error()
+			fmt.Fprintf(os.Stderr, "backup: run %s: manifest on %s: %v\n", m.Run, n.URL, err)
+		}
+	}
+	return m
+}
+
+// uploadAll has each node upload its checkpoint of the run, one after
+// another, each tried r.tries times, and reports whether all did.
+func (r *runner) uploadAll(m *Manifest) bool {
+	ok := true
+	for i := range m.Nodes {
+		n := &m.Nodes[i]
+		var err error
+		for t := 0; t < r.tries; t++ {
+			if t > 0 {
+				time.Sleep(r.wait)
+			}
+			req, rerr := http.NewRequest(http.MethodPost, n.URL+"/_checkpoint/upload?run="+url.QueryEscape(m.Run), nil)
+			if rerr != nil {
+				err = rerr
+				break
+			}
+			var b []byte
+			if b, err = r.do(req); err == nil {
+				n.Upload = json.RawMessage(b)
+				break
+			}
+		}
+		if err != nil {
+			n.UploadError = err.Error()
+			ok = false
+			fmt.Fprintf(os.Stderr, "backup: run %s: upload from %s: %v\n", m.Run, n.URL, err)
+		}
+	}
+	return ok
+}
+
+// checkpoint has the node at u take its checkpoint of run, and returns its
+// description of it.
+func (r *runner) checkpoint(u, run string) (json.RawMessage, error) {
+	req, err := http.NewRequest(http.MethodPost, u+"/_checkpoint?run="+url.QueryEscape(run), nil)
+	if err != nil {
+		return nil, err
+	}
+	b, err := r.do(req)
+	if err != nil {
+		return nil, err
+	}
+	if !json.Valid(b) {
+		return nil, fmt.Errorf("the answer isn't JSON: %.200s", b)
+	}
+	return json.RawMessage(b), nil
+}
+
+// putManifest keeps the run's manifest in the node's checkpoint of it.
+func (r *runner) putManifest(u, run string, manifest []byte) error {
+	req, err := http.NewRequest(http.MethodPut, u+"/_checkpoint/manifest?run="+url.QueryEscape(run), bytes.NewReader(manifest))
+	if err != nil {
+		return err
+	}
+	_, err = r.do(req)
+	return err
+}
+
+func (r *runner) do(req *http.Request) ([]byte, error) {
+	req.Header.Set("Authorization", "Bearer "+r.token)
+	resp, err := r.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if resp.StatusCode/100 != 2 {
+		return nil, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(b)))
+	}
+	return b, nil
+}
diff --git a/server/cmd/backup/tools.go b/server/cmd/backup/tools.go
new file mode 100644
index 0000000..b04a9b4
--- /dev/null
+++ b/server/cmd/backup/tools.go
@@ -0,0 +1,315 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"filippo.io/age"
+
+	"github.com/unit-io/unitdb/server/internal/backup"
+)
+
+// The operator's commands, run with a reader's identity (AWS credentials
+// allowed to read the bucket, and the backup key):
+//
+//	backup fetch -run <id> -out <dir> [-node <node>]
+//	    downloads a run, decrypts each node's checkpoint into <dir>/<node>,
+//	    or one node's into <dir>, to start each node at with -restored
+//	    (docs/backup-restore.md, runbook B).
+//	backup runs
+//	    lists the runs in the bucket, newest first, and whether each is
+//	    complete: a restore takes the newest complete one.
+//	backup fetch-journal -since <run id> -out <dir>
+//	    downloads the security journal from the run's day on, for -journal.
+//	backup escrow-keyring -keyring <file>
+//	    puts the keyring in use into the cluster's escrow in Secrets
+//	    Manager, before it goes into the cluster's Secret.
+//
+// The bucket and cluster come from BACKUP_S3_*, BACKUP_CLUSTER. The backup
+// key's private half is read from -identity (a file age-keygen wrote),
+// BACKUP_AGE_IDENTITY, or else Secrets Manager.
+var tools = map[string]func(args []string) error{
+	"fetch":          fetch,
+	"fetch-journal":  fetchJournal,
+	"escrow-keyring": escrowKeyring,
+	"verify":         verify,
+	"runs":           runs,
+}
+
+func storeFromEnv(ctx context.Context) (*backup.Store, error) {
+	cfg, err := backup.ConfigFromEnv()
+	if err != nil {
+		return nil, err
+	}
+	if cfg == nil {
+		return nil, errors.New("BACKUP_S3_BUCKET and BACKUP_CLUSTER name the bucket")
+	}
+	return backup.NewStore(ctx, cfg)
+}
+
+// identity returns the backup key's private half.
+func identity(ctx context.Context, file string, cfg *backup.Config) (age.Identity, error) {
+	switch {
+	case file != "":
+		b, err := os.ReadFile(file)
+		if err != nil {
+			return nil, err
+		}
+		return backup.ParseIdentity(string(b))
+	case os.Getenv("BACKUP_AGE_IDENTITY") != "":
+		return backup.ParseIdentity(os.Getenv("BACKUP_AGE_IDENTITY"))
+	}
+	ss, err := backup.NewAWSSecrets(ctx, cfg.Region)
+	if err != nil {
+		return nil, err
+	}
+	v, found, err := ss.Get(ctx, backup.BackupKeySecret(cfg.Cluster))
+	if err != nil {
+		return nil, err
+	}
+	if !found {
+		return nil, fmt.Errorf("no %s in Secrets Manager", backup.BackupKeySecret(cfg.Cluster))
+	}
+	return backup.ParseIdentity(v)
+}
+
+func fetch(args []string) error {
+	fs := flag.NewFlagSet("fetch", flag.ExitOnError)
+	run := fs.String("run", "", "The run's id, as 20060102T150405Z.")
+	out := fs.String("out", "", "Where to put each node's checkpoint: <out>/<node>; with -node, <out> itself.")
+	node := fs.String("node", "", "Only this node's checkpoint, into <out>: a restore Job's, on the node's new claim.")
+	idFile := fs.String("identity", "", "The backup key's private half, as age-keygen wrote it.")
+	fs.Parse(args)
+	if *run == "" || *out == "" {
+		return errors.New("-run and -out are needed")
+	}
+	ctx := context.Background()
+	st, err := storeFromEnv(ctx)
+	if err != nil {
+		return err
+	}
+	id, err := identity(ctx, *idFile, st.Config)
+	if err != nil {
+		return err
+	}
+	if *node != "" {
+		m, err := readManifest(ctx, st, *run)
+		if err != nil {
+			return err
+		}
+		if !m.Complete {
+			return fmt.Errorf("run %s is incomplete: restore every node from one complete run", *run)
+		}
+		found := false
+		for _, n := range m.Nodes {
+			found = found || n.Checkpoint.Node == *node
+		}
+		if !found {
+			return fmt.Errorf("run %s has no checkpoint of %s", *run, *node)
+		}
+		obj, err := st.Get(ctx, st.Config.CheckpointKey(*run, *node))
+		if err != nil {
+			return fmt.Errorf("%s's checkpoint: %v", *node, err)
+		}
+		defer obj.Close()
+		if err := backup.ExtractArchive(obj, id, *out); err != nil {
+			return fmt.Errorf("%s's checkpoint: %v", *node, err)
+		}
+		fmt.Println(*out)
+		return nil
+	}
+	_, dirs, err := fetchRun(ctx, st, id, *run, *out)
+	for _, d := range dirs {
+		fmt.Println(d)
+	}
+	return err
+}
+
+// runManifest is what fetch and verify read of a run's manifest.
+type runManifest struct {
+	Run      string `json:"run"`
+	Complete bool   `json:"complete"`
+	Nodes    []struct {
+		Checkpoint struct {
+			Node   string `json:"node"`
+			Time   string `json:"time"`
+			KeyIDs []int  `json:"key_ids"`
+			Stats  struct {
+				Messages   int64 `json:"messages"`
+				MemEntries int64 `json:"mem_entries"`
+			} `json:"stats"`
+		} `json:"checkpoint"`
+		Error string `json:"error"`
+	} `json:"nodes"`
+}
+
+func readManifest(ctx context.Context, st *backup.Store, run string) (*runManifest, error) {
+	body, err := st.Get(ctx, st.Config.ManifestKey(run))
+	if err != nil {
+		return nil, fmt.Errorf("run %s's manifest: %v", run, err)
+	}
+	defer body.Close()
+	var m runManifest
+	if err := json.NewDecoder(body).Decode(&m); err != nil {
+		return nil, fmt.Errorf("run %s's manifest: %v", run, err)
+	}
+	return &m, nil
+}
+
+// fetchRun downloads a complete run, and decrypts each node's checkpoint
+// into out/<node>. It returns the manifest and the directories.
+func fetchRun(ctx context.Context, st *backup.Store, id age.Identity, run, out string) (*runManifest, []string, error) {
+	m, err := readManifest(ctx, st, run)
+	if err != nil {
+		return nil, nil, err
+	}
+	if !m.Complete {
+		return m, nil, fmt.Errorf("run %s is incomplete: restore every node from one complete run", run)
+	}
+	var dirs []string
+	for _, n := range m.Nodes {
+		node := n.Checkpoint.Node
+		if node == "" {
+			node = "server"
+		}
+		obj, err := st.Get(ctx, st.Config.CheckpointKey(run, node))
+		if err != nil {
+			return m, dirs, fmt.Errorf("%s's checkpoint: %v", node, err)
+		}
+		dst := filepath.Join(out, node)
+		err = backup.ExtractArchive(obj, id, dst)
+		obj.Close()
+		if err != nil {
+			return m, dirs, fmt.Errorf("%s's checkpoint: %v", node, err)
+		}
+		dirs = append(dirs, dst)
+	}
+	return m, dirs, nil
+}
+
+func fetchJournal(args []string) error {
+	fs := flag.NewFlagSet("fetch-journal", flag.ExitOnError)
+	since := fs.String("since", "", "The run restored from (its id), or a date (2006-01-02): the journal from that day on.")
+	out := fs.String("out", "", "Where to put the journal's files.")
+	fs.Parse(args)
+	if *since == "" || *out == "" {
+		return errors.New("-since and -out are needed")
+	}
+	from, err := time.Parse("20060102T150405Z", *since)
+	if err != nil {
+		if from, err = time.Parse("2006-01-02", *since); err != nil {
+			return errors.New("-since is a run id or a date")
+		}
+	}
+	// A day early: a node's clock, or a batch uploaded after midnight.
+	day := from.UTC().Add(-24 * time.Hour).Format("2006-01-02")
+	ctx := context.Background()
+	st, err := storeFromEnv(ctx)
+	if err != nil {
+		return err
+	}
+	keys, err := st.List(ctx, st.Config.JournalPrefix())
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(*out, 0700); err != nil {
+		return err
+	}
+	n := 0
+	for _, key := range keys {
+		rest := strings.TrimPrefix(key, st.Config.JournalPrefix())
+		if date, _, _ := strings.Cut(rest, "/"); date < day {
+			continue
+		}
+		obj, err := st.Get(ctx, key)
+		if err != nil {
+			return err
+		}
+		f, err := os.Create(filepath.Join(*out, path.Base(key)))
+		if err == nil {
+			_, err = io.Copy(f, obj)
+			if cerr := f.Close(); err == nil {
+				err = cerr
+			}
+		}
+		obj.Close()
+		if err != nil {
+			return err
+		}
+		n++
+	}
+	fmt.Printf("%d journal files in %s\n", n, *out)
+	return nil
+}
+
+func escrowKeyring(args []string) error {
+	fs := flag.NewFlagSet("escrow-keyring", flag.ExitOnError)
+	file := fs.String("keyring", "", "The keyring, as UNITDB_KEYRING holds it; UNITDB_KEYRING if unset.")
+	fs.Parse(args)
+	keyring := os.Getenv("UNITDB_KEYRING")
+	if *file != "" {
+		b, err := os.ReadFile(*file)
+		if err != nil {
+			return err
+		}
+		keyring = string(b)
+	}
+	if keyring == "" {
+		return errors.New("-keyring or UNITDB_KEYRING is needed")
+	}
+	cluster := os.Getenv("BACKUP_CLUSTER")
+	if cluster == "" {
+		return errors.New("BACKUP_CLUSTER names the cluster")
+	}
+	region := os.Getenv("BACKUP_S3_REGION")
+	ctx := context.Background()
+	ss, err := backup.NewAWSSecrets(ctx, region)
+	if err != nil {
+		return err
+	}
+	if err := backup.EscrowKeyring(ctx, ss, cluster, keyring); err != nil {
+		return err
+	}
+	fmt.Printf("escrowed in %s\n", backup.KeyringSecret(cluster))
+	return nil
+}
+
+func runs(args []string) error {
+	ctx := context.Background()
+	st, err := storeFromEnv(ctx)
+	if err != nil {
+		return err
+	}
+	keys, err := st.List(ctx, st.Config.Cluster+"/")
+	if err != nil {
+		return err
+	}
+	var ids []string
+	for _, k := range keys {
+		parts := strings.Split(strings.TrimPrefix(k, st.Config.Cluster+"/"), "/")
+		if len(parts) == 2 && parts[1] == "manifest.json" && runDir.MatchString(parts[0]) {
+			ids = append(ids, parts[0])
+		}
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(ids)))
+	for _, id := range ids {
+		status := "complete"
+		if m, err := readManifest(ctx, st, id); err != nil {
+			status = "unreadable: " + err.Error()
+		} else if !m.Complete {
+			status = "incomplete"
+		}
+		fmt.Printf("%s\t%s\n", id, status)
+	}
+	return nil
+}
diff --git a/server/cmd/backup/verify.go b/server/cmd/backup/verify.go
new file mode 100644
index 0000000..bae66fa
--- /dev/null
+++ b/server/cmd/backup/verify.go
@@ -0,0 +1,359 @@
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"math"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/unit-io/unitdb/server/internal/backup"
+)
+
+// verify is the weekly restore test (docs/backup-restore.md),
+// run in its own namespace with a reader's identity, the only one besides
+// named admins that may read the backup key and the escrowed keyring:
+//
+//	backup verify -run latest -server /unitdb -pushgateway http://...
+//
+// It downloads the newest complete run, decrypts it, and opens each node's
+// checkpoint on a scratch server of its own (-db_path at it, no cluster,
+// no clients), with the escrowed keyring. Each must:
+//
+//   - open, and answer /_readyz, its checks passing (the store probe
+//     writes and reads back);
+//   - hold what the manifest says it held, within -stats_tolerance (what
+//     expired since);
+//   - hold the run's canary, as a message and as a record, both opened
+//     with the escrowed keyring.
+//
+// And the escrow must hold every key the run's manifest names. Only if all
+// pass, it pushes unitdb_backup_restore_verified_timestamp_seconds to the
+// Pushgateway; the alert on its age (deploy/kubernetes/backup-alerts.yaml)
+// catches a test that fails, or doesn't run.
+func verify(args []string) error {
+	fs := flag.NewFlagSet("verify", flag.ExitOnError)
+	runFlag := fs.String("run", "latest", "The run to test, or latest: the newest complete one.")
+	server := fs.String("server", "/unitdb", "The unitdb server binary.")
+	keyringFile := fs.String("keyring", "", "The keyring, as UNITDB_KEYRING holds it; the escrowed one if unset.")
+	idFile := fs.String("identity", "", "The backup key's private half; BACKUP_AGE_IDENTITY, or Secrets Manager, if unset.")
+	work := fs.String("work", "", "A scratch directory; a temporary one if unset.")
+	pushgateway := fs.String("pushgateway", "", "The Pushgateway to push the result to, as http://pushgateway:9091.")
+	tolerance := fs.Float64("stats_tolerance", 0.1, "How far a restored store's counts may be from the manifest's, as a fraction (what expired since).")
+	startTimeout := fs.Duration("start_timeout", 5*time.Minute, "Longest a scratch server may take to be ready.")
+	fs.Parse(args)
+
+	ctx := context.Background()
+	st, err := storeFromEnv(ctx)
+	if err != nil {
+		return err
+	}
+	run := *runFlag
+	if run == "latest" {
+		if run, err = latestCompleteRun(ctx, st); err != nil {
+			return err
+		}
+	}
+	id, err := identity(ctx, *idFile, st.Config)
+	if err != nil {
+		return err
+	}
+	keyring, err := escrowedKeyring(ctx, *keyringFile, st.Config)
+	if err != nil {
+		return err
+	}
+	dir := *work
+	if dir == "" {
+		if dir, err = os.MkdirTemp("", "unitdb-verify-"); err != nil {
+			return err
+		}
+		defer os.RemoveAll(dir)
+	}
+
+	m, dirs, err := fetchRun(ctx, st, id, run, filepath.Join(dir, "run"))
+	if err != nil {
+		return err
+	}
+	var failures []string
+	if missing := missingKeys(m, keyring); len(missing) > 0 {
+		failures = append(failures, fmt.Sprintf("the escrow lacks key ids %v that the run names", missing))
+	}
+	for i, n := range m.Nodes {
+		if err := checkNode(*server, dirs[i], keyring, run, n.Checkpoint.Stats.Messages, n.Checkpoint.Stats.MemEntries, *tolerance, *startTimeout); err != nil {
+			failures = append(failures, fmt.Sprintf("%s: %v", n.Checkpoint.Node, err))
+		} else {
+			fmt.Printf("run %s, %s: verified\n", run, n.Checkpoint.Node)
+		}
+	}
+	if len(failures) > 0 {
+		return fmt.Errorf("run %s:\n  %s", run, strings.Join(failures, "\n  "))
+	}
+	if *pushgateway != "" {
+		if err := pushVerified(*pushgateway, st.Config.Cluster, run); err != nil {
+			return fmt.Errorf("run %s verified, but the Pushgateway: %v", run, err)
+		}
+	}
+	fmt.Printf("run %s verified: %d nodes\n", run, len(m.Nodes))
+	return nil
+}
+
+var runDir = regexp.MustCompile(`^[0-9]{8}T[0-9]{6}Z$`)
+
+// latestCompleteRun returns the newest run whose manifest says it is
+// complete.
+func latestCompleteRun(ctx context.Context, st *backup.Store) (string, error) {
+	keys, err := st.List(ctx, st.Config.Cluster+"/")
+	if err != nil {
+		return "", err
+	}
+	seen := map[string]bool{}
+	var runs []string
+	for _, k := range keys {
+		parts := strings.Split(strings.TrimPrefix(k, st.Config.Cluster+"/"), "/")
+		if len(parts) == 2 && parts[1] == "manifest.json" && runDir.MatchString(parts[0]) && !seen[parts[0]] {
+			seen[parts[0]] = true
+			runs = append(runs, parts[0])
+		}
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(runs)))
+	for _, run := range runs {
+		if m, err := readManifest(ctx, st, run); err == nil && m.Complete {
+			return run, nil
+		}
+	}
+	return "", errors.New("no complete run in the bucket")
+}
+
+// escrowedKeyring returns the keyring to open the checkpoints with: the
+// file's, or the escrowed one.
+func escrowedKeyring(ctx context.Context, file string, cfg *backup.Config) (string, error) {
+	if file != "" {
+		b, err := os.ReadFile(file)
+		return string(b), err
+	}
+	ss, err := backup.NewAWSSecrets(ctx, cfg.Region)
+	if err != nil {
+		return "", err
+	}
+	v, found, err := ss.Get(ctx, backup.KeyringSecret(cfg.Cluster))
+	if err != nil {
+		return "", err
+	}
+	if !found {
+		return "", fmt.Errorf("no %s in Secrets Manager", backup.KeyringSecret(cfg.Cluster))
+	}
+	return v, nil
+}
+
+// missingKeys returns the key ids the run's checkpoints name that the
+// keyring lacks.
+func missingKeys(m *runManifest, keyring string) []int {
+	have, err := backup.EscrowedKeyIDs(keyring)
+	if err != nil {
+		return []int{-1}
+	}
+	var missing []int
+	seen := map[int]bool{}
+	for _, n := range m.Nodes {
+		for _, id := range n.Checkpoint.KeyIDs {
+			if !have[id] && !seen[id] {
+				seen[id] = true
+				missing = append(missing, id)
+			}
+		}
+	}
+	return missing
+}
+
+// checkNode opens one node's checkpoint on a scratch server, and checks it.
+func checkNode(server, dbPath, keyring, run string, messages, memEntries int64, tolerance float64, startTimeout time.Duration) error {
+	ports := make([]int, 3)
+	for i := range ports {
+		p, err := freePort()
+		if err != nil {
+			return err
+		}
+		ports[i] = p
+	}
+	scratch := filepath.Dir(dbPath)
+	node := filepath.Base(dbPath)
+	conf := filepath.Join(scratch, node+".conf")
+	if err := os.WriteFile(conf, []byte(fmt.Sprintf(`{
+  "listen": "127.0.0.1:%d",
+  "grpc_listen": "127.0.0.1:%d",
+  "monitor_listen": "127.0.0.1:%d",
+  "logging_level": "Error",
+  "encryption_config": {"identifier": "local", "sealed": false, "timestamp": 1522325758},
+  "cluster_config": {"self": ""},
+  "store_config": {"reset": false, "adapters": {"unitdb": {"database": "unitdb", "mem_size": 500000000}}}
+}`, ports[0], ports[1], ports[2])), 0600); err != nil {
+		return err
+	}
+	tokenBytes := make([]byte, 16)
+	rand.Read(tokenBytes)
+	token := hex.EncodeToString(tokenBytes)
+	// The server reads -config from beside its binary: the path relative to it.
+	confArg, err := filepath.Rel(filepath.Dir(server), conf)
+	if err != nil {
+		return err
+	}
+	cmd := exec.Command(server, "-config", confArg, "-db_path", dbPath)
+	// Only what the scratch server needs: no bucket, no cluster.
+	cmd.Env = []string{
+		"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME"),
+		"UNITDB_KEYRING=" + keyring,
+		"CHECKPOINT_DIR=" + filepath.Join(scratch, node+"-checkpoints"),
+		"CHECKPOINT_TOKEN=" + token,
+	}
+	logs := &strings.Builder{}
+	cmd.Stdout, cmd.Stderr = logs, logs
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	exited := make(chan struct{})
+	go func() { cmd.Wait(); close(exited) }()
+	defer func() {
+		cmd.Process.Signal(syscall.SIGTERM)
+		select {
+		case <-exited:
+		case <-time.After(30 * time.Second):
+			cmd.Process.Kill()
+			<-exited
+		}
+	}()
+	monitor := fmt.Sprintf("http://127.0.0.1:%d", ports[2])
+	fail := func(format string, a ...interface{}) error {
+		return fmt.Errorf(format+"\n    server logs: %.2000s", append(a, logs.String())...)
+	}
+
+	// It opens, and is ready: its checks, the store probe among them, pass.
+	deadline := time.Now().Add(startTimeout)
+	for {
+		select {
+		case <-exited:
+			return fail("the scratch server exited")
+		default:
+		}
+		if code, _, _ := httpGet(monitor+"/_readyz", ""); code == http.StatusOK {
+			break
+		}
+		if time.Now().After(deadline) {
+			_, status, _ := httpGet(monitor+"/_status", "")
+			return fail("not ready after %s: %s", startTimeout, status)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	if code, status, err := httpGet(monitor+"/_status", ""); err != nil || code != http.StatusOK {
+		return fail("/_status %d: %s %v", code, status, err)
+	}
+
+	// It holds what it held.
+	_, metrics, err := httpGet(monitor+"/_metrics", "")
+	if err != nil {
+		return fail("/_metrics: %v", err)
+	}
+	for _, c := range []struct {
+		name string
+		want int64
+	}{{"unitdb_store_messages", messages}, {"unitdb_store_mem_entries", memEntries}} {
+		got, ok := metricValue(metrics, c.name)
+		if !ok {
+			return fail("no %s", c.name)
+		}
+		if slack := math.Max(tolerance*float64(c.want), 5); math.Abs(got-float64(c.want)) > slack {
+			return fail("%s is %v, the manifest says %d", c.name, got, c.want)
+		}
+	}
+
+	// Its canary is the run's.
+	code, body, err := httpGet(monitor+"/_checkpoint/canary", token)
+	if err != nil || code != http.StatusOK {
+		return fail("the canary: %d %s %v", code, body, err)
+	}
+	var canary struct{ Message, Record string }
+	if err := json.Unmarshal([]byte(body), &canary); err != nil || canary.Message != run || canary.Record != run {
+		return fail("the canary is %s, not run %s's", body, run)
+	}
+	return nil
+}
+
+func httpGet(url, token string) (int, string, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return 0, "", err
+	}
+	if token != "" {
+		req.Header.Set("Authorization", "Bearer "+token)
+	}
+	resp, err := (&http.Client{Timeout: 10 * time.Second}).Do(req)
+	if err != nil {
+		return 0, "", err
+	}
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	return resp.StatusCode, string(b), nil
+}
+
+// metricValue returns an unlabelled metric's value from a /_metrics page.
+func metricValue(page, name string) (float64, bool) {
+	for _, line := range strings.Split(page, "\n") {
+		if strings.HasPrefix(line, name+" ") {
+			v, err := strconv.ParseFloat(strings.TrimSpace(strings.TrimPrefix(line, name+" ")), 64)
+			return v, err == nil
+		}
+	}
+	return 0, false
+}
+
+func freePort() (int, error) {
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		return 0, err
+	}
+	defer l.Close()
+	return l.Addr().(*net.TCPAddr).Port, nil
+}
+
+// pushVerified pushes the test's success to the Pushgateway, under the
+// cluster's name.
+func pushVerified(gateway, cluster, run string) error {
+	runAt, _ := time.Parse("20060102T150405Z", run)
+	body := fmt.Sprintf(`# TYPE unitdb_backup_restore_verified_timestamp_seconds gauge
+# HELP unitdb_backup_restore_verified_timestamp_seconds When the weekly restore test last verified a run, in unix seconds.
+unitdb_backup_restore_verified_timestamp_seconds %d
+# TYPE unitdb_backup_restore_verified_run_timestamp_seconds gauge
+# HELP unitdb_backup_restore_verified_run_timestamp_seconds When the run it verified began, in unix seconds.
+unitdb_backup_restore_verified_run_timestamp_seconds %d
+`, time.Now().Unix(), runAt.Unix())
+	url := strings.TrimRight(gateway, "/") + "/metrics/job/unitdb_backup_restore_test/cluster/" + cluster
+	req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(body))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "text/plain; version=0.0.4")
+	resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<10))
+		return fmt.Errorf("%s: %s", resp.Status, b)
+	}
+	return nil
+}
diff --git a/server/e2e/backup_run_test.go b/server/e2e/backup_run_test.go
new file mode 100644
index 0000000..be4d64e
--- /dev/null
+++ b/server/e2e/backup_run_test.go
@@ -0,0 +1,160 @@
+package e2e
+
+// The backup run (docs/backup-restore.md): the backup command
+// checkpoints every node under one run id, and keeps the run's manifest in
+// each node's checkpoint; a node that is down makes the run incomplete.
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// buildBackup builds the backup command, once per test.
+func buildBackup(t *testing.T) string {
+	t.Helper()
+	bin := filepath.Join(t.TempDir(), "backup")
+	cmd := exec.Command("go", "build", "-o", bin, "./cmd/backup")
+	cmd.Dir = serverSourceDir(t)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("build backup: %v\n%s", err, out)
+	}
+	return bin
+}
+
+type backupManifest struct {
+	Run      string `json:"run"`
+	Complete bool   `json:"complete"`
+	Nodes    []struct {
+		URL        string `json:"url"`
+		Error      string `json:"error"`
+		Checkpoint struct {
+			Dir  string `json:"dir"`
+			Node string `json:"node"`
+			Run  string `json:"run"`
+		} `json:"checkpoint"`
+	} `json:"nodes"`
+}
+
+// runBackup runs the backup command against the nodes, and returns its
+// manifest and whether it exited 0.
+func runBackup(t *testing.T, bin string, nodes []*clusterNode) (backupManifest, bool) {
+	t.Helper()
+	var urls []string
+	for _, n := range nodes {
+		urls = append(urls, "http://"+n.monitorAddr)
+	}
+	cmd := exec.Command(bin, "-nodes", strings.Join(urls, ","), "-tries", "2", "-retry_wait", "200ms", "-timeout", "30s")
+	cmd.Env = append(os.Environ(), "CHECKPOINT_TOKEN="+checkpointToken)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout, cmd.Stderr = &stdout, &stderr
+	err := cmd.Run()
+	var m backupManifest
+	if jerr := json.Unmarshal(stdout.Bytes(), &m); jerr != nil {
+		t.Fatalf("backup printed %q (%v), stderr:\n%s", stdout.String(), err, stderr.String())
+	}
+	t.Logf("backup run %s: complete %v, exit %v\n%s", m.Run, m.Complete, err, stderr.String())
+	return m, err == nil
+}
+
+func TestBackupRunCheckpointsEveryNode(t *testing.T) {
+	bin := buildBackup(t)
+	c := startClusterWith(t, clusterOpts{env: checkpointEnv(t)}, names...)
+	if _, err := c.waitLeader(c.nodes, 10*time.Second); err != nil {
+		t.Fatal(err)
+	}
+	cid := newClientID(0x0e2e8c8c)
+	publishAcked(t, c.nodes[0], cid, 10, "groups.run.x")
+
+	m, ok := runBackup(t, bin, c.nodes)
+	if !ok || !m.Complete || len(m.Nodes) != len(c.nodes) {
+		t.Fatalf("a run of %d nodes: %+v", len(c.nodes), m)
+	}
+	for i, n := range m.Nodes {
+		ck := n.Checkpoint
+		if want := "ckpt-" + m.Run + "-" + c.nodes[i].name; filepath.Base(ck.Dir) != want || ck.Run != m.Run || ck.Node != c.nodes[i].name {
+			t.Errorf("node %s: checkpoint %+v, want %s of run %s", c.nodes[i].name, ck, want, m.Run)
+			continue
+		}
+		// Each node's checkpoint holds the whole run's manifest.
+		b, err := os.ReadFile(filepath.Join(ck.Dir, "manifest.json"))
+		var kept backupManifest
+		if err != nil || json.Unmarshal(b, &kept) != nil || kept.Run != m.Run || len(kept.Nodes) != len(c.nodes) {
+			t.Errorf("manifest in %s: %v %s", ck.Dir, err, b)
+		}
+		info, err := os.ReadFile(filepath.Join(ck.Dir, "checkpoint.json"))
+		if err != nil || !strings.Contains(string(info), `"run": "`+m.Run+`"`) {
+			t.Errorf("checkpoint.json in %s: %v %s", ck.Dir, err, info)
+		}
+	}
+
+	// A node down: the others are checkpointed, the run is incomplete, and
+	// the command fails, naming it.
+	time.Sleep(1100 * time.Millisecond) // a new run id
+	down := c.nodes[1]
+	down.stop()
+	m2, ok := runBackup(t, bin, c.nodes)
+	if ok || m2.Complete {
+		t.Fatalf("a run with %s down: exit ok %v, %+v", down.name, ok, m2)
+	}
+	for i, n := range m2.Nodes {
+		if (i == 1) != (n.Error != "") {
+			t.Errorf("node %s in the incomplete run: error %q", c.nodes[i].name, n.Error)
+		}
+	}
+}
+
+// TestRestoreWholeClusterFromRunID restores every node at its checkpoint of
+// one backup run, with run ids: a node at another run's checkpoint is
+// refused, however close in time.
+func TestRestoreWholeClusterFromRunID(t *testing.T) {
+	bin := buildBackup(t)
+	c := startClusterWith(t, clusterOpts{env: checkpointEnv(t)}, names...)
+	if _, err := c.waitLeader(c.nodes, 10*time.Second); err != nil {
+		t.Fatal(err)
+	}
+	contract := uint32(0x0e2e9d9d)
+	cid := newClientID(contract)
+	var topics []string
+	for _, n := range c.nodes {
+		topics = append(topics, topicOwnedBy(n.name, contract, "groups.runid.owned", names...))
+	}
+	publishAcked(t, c.nodes[0], cid, 20, topics...)
+	time.Sleep(500 * time.Millisecond) // replication is asynchronous
+
+	first, ok := runBackup(t, bin, c.nodes)
+	if !ok {
+		t.Fatal("the first run is incomplete")
+	}
+	time.Sleep(1100 * time.Millisecond) // a new run id
+	second, ok := runBackup(t, bin, c.nodes)
+	if !ok {
+		t.Fatal("the second run is incomplete")
+	}
+	for _, n := range c.nodes {
+		n.stop()
+	}
+	last := len(c.nodes) - 1
+	for i, n := range c.nodes[:last] {
+		if err := n.restartAt(second.Nodes[i].Checkpoint.Dir); err != nil {
+			t.Fatalf("start %s at run %s: %v\nlogs:\n%s", n.name, second.Run, err, n.logs.String())
+		}
+	}
+	if logs := c.nodes[last].refusedAt(t, first.Nodes[last].Checkpoint.Dir); !strings.Contains(logs, "not the same backup run") {
+		t.Errorf("a checkpoint of run %s among run %s's, it says:\n%s", first.Run, second.Run, logs)
+	}
+	if err := c.nodes[last].restartAt(second.Nodes[last].Checkpoint.Dir); err != nil {
+		t.Fatalf("start %s at run %s: %v", c.nodes[last].name, second.Run, err)
+	}
+	for _, n := range c.nodes {
+		waitReadyz(t, n, 30*time.Second)
+	}
+	for _, n := range c.nodes {
+		checkStored(t, n, cid, 20, topics...)
+	}
+}
diff --git a/server/e2e/offsite_test.go b/server/e2e/offsite_test.go
new file mode 100644
index 0000000..6a1ba7a
--- /dev/null
+++ b/server/e2e/offsite_test.go
@@ -0,0 +1,268 @@
+package e2e
+
+// Copies off the cluster (docs/backup-restore.md), against
+// MinIO: a run's checkpoints go to a bucket with Object Lock, encrypted;
+// the security journal follows; a whole-cluster restore from the bucket
+// replays the journal, so a client id revoked after the run stays revoked.
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"filippo.io/age"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+
+	"github.com/unit-io/unitdb/server/internal/backup"
+	"github.com/unit-io/unitdb/server/internal/types"
+)
+
+const (
+	minioUser     = "e2e-minio"
+	minioPassword = "e2e-minio-password"
+	offsiteBucket = "unitdb-backups-e2e"
+)
+
+// startMinIO starts a MinIO server for the test, with a bucket that has
+// Object Lock, and returns its endpoint.
+func startMinIO(t *testing.T) string {
+	t.Helper()
+	bin, err := exec.LookPath("minio")
+	if err != nil {
+		t.Skip("minio isn't installed")
+	}
+	port, console := freePort(t), freePort(t)
+	endpoint := fmt.Sprintf("http://127.0.0.1:%d", port)
+	cmd := exec.Command(bin, "server", t.TempDir(), "--address", fmt.Sprintf("127.0.0.1:%d", port), "--console-address", fmt.Sprintf("127.0.0.1:%d", console), "--quiet")
+	cmd.Env = append(os.Environ(), "MINIO_ROOT_USER="+minioUser, "MINIO_ROOT_PASSWORD="+minioPassword)
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		cmd.Process.Kill()
+		cmd.Wait()
+	})
+	for deadline := time.Now().Add(20 * time.Second); ; time.Sleep(100 * time.Millisecond) {
+		if resp, err := http.Get(endpoint + "/minio/health/live"); err == nil {
+			resp.Body.Close()
+			if resp.StatusCode == http.StatusOK {
+				break
+			}
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("minio didn't start")
+		}
+	}
+	// This test's own client: the root user, a reader.
+	t.Setenv("AWS_ACCESS_KEY_ID", minioUser)
+	t.Setenv("AWS_SECRET_ACCESS_KEY", minioPassword)
+	st := offsiteStore(t, endpoint)
+	if _, err := st.Client().CreateBucket(context.Background(), &s3.CreateBucketInput{Bucket: aws.String(offsiteBucket), ObjectLockEnabledForBucket: aws.Bool(true)}); err != nil {
+		t.Fatal(err)
+	}
+	return endpoint
+}
+
+func offsiteStore(t *testing.T, endpoint string) *backup.Store {
+	t.Helper()
+	st, err := backup.NewStore(context.Background(), &backup.Config{Bucket: offsiteBucket, Region: "us-east-1", Endpoint: endpoint, Cluster: "e2e"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return st
+}
+
+// offsiteEnv is a node's environment for checkpoints, uploads to endpoint,
+// and its security journal.
+func offsiteEnv(t *testing.T, endpoint, recipient string) map[string][]string {
+	env := checkpointEnv(t)
+	for name := range env {
+		env[name] = append(env[name],
+			"BACKUP_S3_BUCKET="+offsiteBucket,
+			"BACKUP_S3_ENDPOINT="+endpoint, "BACKUP_S3_REGION=us-east-1",
+			"BACKUP_CLUSTER=e2e",
+			"BACKUP_AGE_RECIPIENT="+recipient,
+			"AWS_ACCESS_KEY_ID="+minioUser,
+			"AWS_SECRET_ACCESS_KEY="+minioPassword,
+			"SECURITY_JOURNAL_DIR="+t.TempDir(),
+			"SECURITY_JOURNAL_EVERY=500ms",
+		)
+	}
+	return env
+}
+
+// runBackupTool runs the backup command's tool with the reader's
+// environment.
+func runBackupTool(t *testing.T, bin, endpoint, identity string, args ...string) string {
+	t.Helper()
+	cmd := exec.Command(bin, args...)
+	cmd.Env = append(os.Environ(),
+		"BACKUP_S3_BUCKET="+offsiteBucket, "BACKUP_S3_ENDPOINT="+endpoint, "BACKUP_S3_REGION=us-east-1", "BACKUP_CLUSTER=e2e",
+		"AWS_ACCESS_KEY_ID="+minioUser, "AWS_SECRET_ACCESS_KEY="+minioPassword, "BACKUP_AGE_IDENTITY="+identity)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("backup %v: %v\n%s", args, err, out)
+	}
+	return string(out)
+}
+
+func TestBackupOffsiteRestoreReplaysJournal(t *testing.T) {
+	endpoint := startMinIO(t)
+	bin := buildBackup(t)
+	id, err := age.GenerateX25519Identity()
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := startClusterWith(t, clusterOpts{env: offsiteEnv(t, endpoint, id.Recipient().String())}, names...)
+	if _, err := c.waitLeader(c.nodes, 10*time.Second); err != nil {
+		t.Fatal(err)
+	}
+	contract := uint32(0x0e2edfdf)
+	cid := newClientID(contract)
+	var topics []string
+	for _, n := range c.nodes {
+		topics = append(topics, topicOwnedBy(n.name, contract, "groups.offsite.owned", names...))
+	}
+	publishAcked(t, c.nodes[0], cid, 20, topics...)
+	time.Sleep(500 * time.Millisecond) // replication is asynchronous
+
+	// The run, uploaded.
+	m := runBackupUpload(t, bin, c.nodes)
+	st := offsiteStore(t, endpoint)
+	keys, err := st.List(context.Background(), "e2e/"+m.Run+"/")
+	if err != nil || len(keys) != len(c.nodes)+1 {
+		t.Fatalf("the run's objects: %v (%v)", keys, err)
+	}
+
+	// Locked: not even the bucket's root user deletes a version, and the
+	// object isn't readable without the backup key.
+	ckKey := "e2e/" + m.Run + "/" + c.nodes[0].name + ".tar.zst.age"
+	head, err := st.Client().HeadObject(context.Background(), &s3.HeadObjectInput{Bucket: aws.String(offsiteBucket), Key: aws.String(ckKey)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if head.ObjectLockMode != s3types.ObjectLockModeCompliance || head.ObjectLockRetainUntilDate == nil || head.ObjectLockRetainUntilDate.Before(time.Now().Add(7*24*time.Hour)) {
+		t.Errorf("%s: lock %q until %v", ckKey, head.ObjectLockMode, head.ObjectLockRetainUntilDate)
+	}
+	if _, err := st.Client().DeleteObject(context.Background(), &s3.DeleteObjectInput{Bucket: aws.String(offsiteBucket), Key: aws.String(ckKey), VersionId: head.VersionId}); err == nil {
+		t.Errorf("deleted a locked version of %s", ckKey)
+	}
+	obj, err := st.Get(context.Background(), ckKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var raw bytes.Buffer
+	raw.ReadFrom(obj)
+	obj.Close()
+	if !bytes.HasPrefix(raw.Bytes(), []byte("age-encryption.org/v1")) {
+		t.Errorf("%s isn't an age file: %.40q", ckKey, raw.Bytes())
+	}
+
+	// After the run: a client id revoked, and messages written; the
+	// journal goes up.
+	revokedID := newClientID(contract)
+	uid, _, err := openClientID(revokedID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pub, err := connectTo(t, c.nodes[1].tcpAddr, connectOpts{clientID: primaryClientID(contract)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	revoke(t, pub, types.RevokeRequest{Uuid: strconv.FormatUint(uid.Uuid(), 10)})
+	if _, err := connectTo(t, c.nodes[1].tcpAddr, connectOpts{clientID: revokedID}); err == nil {
+		t.Fatal("control: the revoked client id connects")
+	}
+	publishBodies(t, c.nodes[0], cid, topics[0], bodies("lost", 5)...)
+	for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(250 * time.Millisecond) {
+		if js, _ := st.List(context.Background(), "e2e/journal/"); len(js) > 0 && journalLagZero(t, c.nodes[1]) {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("the journal didn't go up")
+		}
+	}
+
+	// The whole cluster lost; restored from the bucket, with the journal.
+	for _, n := range c.nodes {
+		n.stop()
+	}
+	restoreDir := t.TempDir()
+	runBackupTool(t, bin, endpoint, id.String(), "fetch", "-run", m.Run, "-out", filepath.Join(restoreDir, "run"))
+	journalDir := filepath.Join(restoreDir, "journal")
+	out := runBackupTool(t, bin, endpoint, id.String(), "fetch-journal", "-since", m.Run, "-out", journalDir)
+	if strings.HasPrefix(out, "0 ") {
+		t.Fatalf("fetch-journal found nothing: %s", out)
+	}
+	// Control: the run predates the revocation. A copy of a node's
+	// checkpoint, opened alone without the journal, takes the client id.
+	copied := filepath.Join(t.TempDir(), "copy")
+	if out, err := exec.Command("cp", "-R", filepath.Join(restoreDir, "run", c.nodes[1].name), copied).CombinedOutput(); err != nil {
+		t.Fatalf("copy: %v %s", err, out)
+	}
+	lone := startServerWith(t, serverOpts{args: []string{"-db_path", copied}})
+	if _, err := connectTo(t, lone.tcpAddr, connectOpts{clientID: revokedID}); err != nil {
+		t.Fatalf("control: without the journal, the run's checkpoint refuses the client id revoked after it: %v", err)
+	}
+	lone.stop()
+
+	for _, n := range c.nodes {
+		if err := n.restartAt(filepath.Join(restoreDir, "run", n.name), "-restored", "-journal", journalDir); err != nil {
+			t.Fatalf("restore %s: %v\nlogs:\n%s", n.name, err, n.logs.String())
+		}
+	}
+	for _, n := range c.nodes {
+		waitReadyz(t, n, 60*time.Second)
+	}
+
+	for _, n := range c.nodes {
+		if _, err := connectTo(t, n.tcpAddr, connectOpts{clientID: revokedID}); err == nil {
+			t.Errorf("the client id revoked after the run connects to %s after the restore", n.name)
+		}
+	}
+	for _, n := range c.nodes {
+		checkStored(t, n, cid, 20, topics...)
+	}
+	if got := relayCounts(t, c.nodes[0], cid, topics[0]); got["lost0"] != 0 {
+		t.Errorf("a message written after the run came back: %v", got)
+	}
+}
+
+// runBackupUpload runs a backup run that uploads, and returns its manifest.
+func runBackupUpload(t *testing.T, bin string, nodes []*clusterNode) backupManifest {
+	t.Helper()
+	var urls []string
+	for _, n := range nodes {
+		urls = append(urls, "http://"+n.monitorAddr)
+	}
+	cmd := exec.Command(bin, "-nodes", strings.Join(urls, ","), "-tries", "2", "-retry_wait", "200ms", "-timeout", "30s", "-upload")
+	cmd.Env = append(os.Environ(), "CHECKPOINT_TOKEN="+checkpointToken)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout, cmd.Stderr = &stdout, &stderr
+	if err := cmd.Run(); err != nil {
+		t.Fatalf("backup -upload: %v\n%s\n%s", err, stdout.String(), stderr.String())
+	}
+	var m backupManifest
+	if err := json.Unmarshal(stdout.Bytes(), &m); err != nil {
+		t.Fatal(err)
+	}
+	return m
+}
+
+// journalLagZero reports whether n's security journal has nothing left to
+// upload.
+func journalLagZero(t *testing.T, n *clusterNode) bool {
+	_, metrics := monitorGet(t, n.server, "/_metrics")
+	return strings.Contains(metrics, "unitdb_security_journal_lag_seconds 0\n")
+}
diff --git a/server/e2e/reconcile_test.go b/server/e2e/reconcile_test.go
new file mode 100644
index 0000000..a6709b1
--- /dev/null
+++ b/server/e2e/reconcile_test.go
@@ -0,0 +1,263 @@
+package e2e
+
+// Reconciliation after a restore (docs/backup-restore.md):
+// nodes started with -restored at one run's checkpoints settle each topic
+// on the union of what their checkpoints hold; a node started with
+// -restored into a running cluster catches up with it.
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	rh "github.com/unit-io/unitdb/server/internal/pkg/hash"
+)
+
+// relayCounts relays topic on node and returns how many times each message
+// body came, all of them, until the relay is quiet.
+func relayCounts(t *testing.T, node *clusterNode, cid, topic string) map[string]int {
+	t.Helper()
+	sub, err := dial(context.Background(), node.tcpAddr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer sub.close()
+	if _, err := sub.connect(cid, true, nextSess()); err != nil {
+		t.Fatal(err)
+	}
+	rid, err := sub.relay(topic, "24h")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !sub.waitAck(rid, 5*time.Second) {
+		t.Fatalf("no relay ack from %s", node.name)
+	}
+	got := map[string]int{}
+	for {
+		msg, ok := sub.waitPub(time.Second)
+		if !ok {
+			return got
+		}
+		for _, m := range msg.Messages {
+			if _, body, ok := decodePayload(m.Payload); ok {
+				got[string(body)]++
+			}
+		}
+	}
+}
+
+// publishBodies publishes each body on topic through node, acknowledged.
+func publishBodies(t *testing.T, node *clusterNode, cid, topic string, bodies ...string) {
+	t.Helper()
+	p, err := dial(context.Background(), node.tcpAddr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer p.close()
+	if _, err := p.connect(cid, true, nextSess()); err != nil {
+		t.Fatal(err)
+	}
+	for _, b := range bodies {
+		id, err := p.publish(1, topic, encodePayload(0, b), "24h")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !p.waitAck(id, 5*time.Second) {
+			t.Fatalf("publish %q on %s: no RECEIPT", b, topic)
+		}
+	}
+}
+
+func bodies(prefix string, n int) []string {
+	var b []string
+	for i := 0; i < n; i++ {
+		b = append(b, fmt.Sprintf("%s%d", prefix, i))
+	}
+	return b
+}
+
+// sameCounts fails t unless got holds exactly want's bodies, as many times.
+func sameCounts(t *testing.T, what string, got map[string]int, want []string) {
+	t.Helper()
+	w := map[string]int{}
+	for _, b := range want {
+		w[b]++
+	}
+	var diff []string
+	for b, n := range w {
+		if got[b] != n {
+			diff = append(diff, fmt.Sprintf("%s: %d, want %d", b, got[b], n))
+		}
+	}
+	for b, n := range got {
+		if w[b] == 0 {
+			diff = append(diff, fmt.Sprintf("%s: %d, want none", b, n))
+		}
+	}
+	if len(diff) > 0 {
+		sort.Strings(diff)
+		t.Errorf("%s:\n  %s", what, strings.Join(diff, "\n  "))
+	}
+}
+
+// TestRestoreReconcilesOneRun restores a cluster from one run whose nodes
+// were checkpointed apart, writes going on between: with -restored, each
+// topic's owner and replica settle on the union, each message once (two
+// identical publishes stay two), and nothing written after the run comes
+// back.
+func TestRestoreReconcilesOneRun(t *testing.T) {
+	c := startClusterWith(t, clusterOpts{env: checkpointEnv(t)}, names...)
+	if _, err := c.waitLeader(c.nodes, 10*time.Second); err != nil {
+		t.Fatal(err)
+	}
+	first := c.nodes[0] // checkpointed before the others
+	contract := uint32(0x0e2ebcbc)
+	cid := newClientID(contract)
+	ring := rh.NewRing(clusterHashReplicas, nil)
+	ring.Add(names...)
+	holders := func(topic string) []string { return ring.GetN(fmt.Sprintf("%d/%s", contract, topic), 2) }
+	// A topic the first node owns, and one it is the replica of.
+	owned := topicOwnedBy(first.name, contract, "groups.reconcile.owned", names...)
+	var replicated string
+	for i := 0; ; i++ {
+		topic := fmt.Sprintf("groups.reconcile.replica.t%d", i)
+		if h := holders(topic); h[1] == first.name {
+			replicated = topic
+			break
+		}
+	}
+	topics := []string{owned, replicated}
+
+	// Before every checkpoint, with a message published twice.
+	before := append(bodies("a", 10), "twice", "twice")
+	between := bodies("b", 10)
+	after := bodies("c", 5)
+	for _, topic := range topics {
+		publishBodies(t, c.nodes[1], cid, topic, before...)
+	}
+	time.Sleep(500 * time.Millisecond) // replication is asynchronous
+	dirs := map[string]string{}
+	dirs[first.name], _ = clusterCheckpoint(t, first)
+	for _, topic := range topics {
+		publishBodies(t, c.nodes[1], cid, topic, between...)
+	}
+	time.Sleep(500 * time.Millisecond)
+	for _, n := range c.nodes[1:] {
+		dirs[n.name], _ = clusterCheckpoint(t, n)
+	}
+	for _, topic := range topics {
+		publishBodies(t, c.nodes[1], cid, topic, after...)
+	}
+
+	// The whole cluster lost, and restored from the run with -restored, the
+	// first node first.
+	for _, n := range c.nodes {
+		n.stop()
+	}
+	for _, n := range c.nodes {
+		if err := n.restartAt(dirs[n.name], "-restored"); err != nil {
+			t.Fatalf("start %s with -restored: %v\nlogs:\n%s", n.name, err, n.logs.String())
+		}
+	}
+	for _, n := range c.nodes {
+		t.Logf("%s before it was ready: %v", n.name, waitReadyz(t, n, 60*time.Second))
+	}
+	want := append(append([]string(nil), before...), between...)
+
+	// Each holder of each topic answers alone: the owner first, then the
+	// replica with the owner stopped.
+	for _, topic := range topics {
+		h := holders(topic)
+		owner, replica := c.node(h[0]), c.node(h[1])
+		sameCounts(t, fmt.Sprintf("%s on its owner %s", topic, owner.name), relayCounts(t, owner, cid, topic), want)
+		owner.stop()
+		deadline := time.Now().Add(15 * time.Second)
+		for {
+			got := relayCounts(t, replica, cid, topic)
+			if len(got) > 0 || time.Now().After(deadline) {
+				sameCounts(t, fmt.Sprintf("%s on its replica %s", topic, replica.name), got, want)
+				break
+			}
+			time.Sleep(250 * time.Millisecond) // until the ring drops the owner
+		}
+		if err := owner.restartAt(owner.dbPathArg()); err != nil {
+			t.Fatalf("restart %s: %v", owner.name, err)
+		}
+		for _, n := range c.nodes {
+			waitReadyz(t, n, 30*time.Second)
+		}
+	}
+
+	// Reconciled, a node's checkpoint.json is its restored-from.json: its
+	// next start is an ordinary one.
+	for _, n := range c.nodes {
+		if _, err := readFile(dirs[n.name], "restored-from.json"); err != nil {
+			t.Errorf("%s: no restored-from.json after reconciling: %v", n.name, err)
+		}
+		if _, err := readFile(dirs[n.name], "checkpoint.json"); err == nil {
+			t.Errorf("%s: checkpoint.json is still there after reconciling", n.name)
+		}
+	}
+	if _, metrics := monitorGet(t, first.server, "/_metrics"); !strings.Contains(metrics, "unitdb_reconcile_topics_total ") {
+		t.Errorf("no reconciliation metrics:\n%s", metrics)
+	}
+}
+
+func readFile(dir, name string) ([]byte, error) {
+	return os.ReadFile(filepath.Join(dir, name))
+}
+
+// TestClusterRebuildFromRestartedNode rebuilds a node from a peer that has
+// itself restarted: the peer's list of the topics it stores, loaded from its
+// store when it opened, must name them. It didn't when records were sealed
+// at rest: the list was loaded before the store could open them, and a
+// rebuild (or a reconciliation, or a ring switch) skipped every topic the
+// peer had stored before it restarted.
+func TestClusterRebuildFromRestartedNode(t *testing.T) {
+	c := startCluster(t, names...)
+	leader, err := c.waitLeader(c.nodes, 10*time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	contract := uint32(0x0e2ecdcd)
+	cid := newClientID(contract)
+	ring := rh.NewRing(clusterHashReplicas, nil)
+	ring.Add(names...)
+	// Neither the victim nor the donor leads: a leader restarted gracefully
+	// is TestClusterGracefulRestartRejoins's case, not this one's.
+	victim := c.node(followerOf(leader))
+	var topic string
+	var donor *clusterNode
+	for i := 0; ; i++ {
+		topic = fmt.Sprintf("groups.donor.t%d", i)
+		if h := ring.GetN(fmt.Sprintf("%d/%s", contract, topic), 2); h[0] == victim.name && h[1] != leader {
+			donor = c.node(h[1])
+			break
+		}
+	}
+	publishAcked(t, c.nodes[2], cid, 20, topic)
+	time.Sleep(500 * time.Millisecond) // replication is asynchronous
+
+	// The donor restarts; then the victim loses its store.
+	donor.shutdown()
+	if err := donor.start(); err != nil {
+		t.Fatal(err)
+	}
+	waitReadyz(t, donor, 30*time.Second)
+	victim.stop()
+	if err := os.RemoveAll(filepath.Join(victim.dbPath, "db")); err != nil {
+		t.Fatal(err)
+	}
+	if err := victim.start(); err != nil {
+		t.Fatal(err)
+	}
+	waitReadyz(t, victim, 30*time.Second)
+
+	// The victim, the topic's owner, answers from its own store.
+	checkStored(t, victim, cid, 20, topic)
+}
diff --git a/server/e2e/restore_cluster_test.go b/server/e2e/restore_cluster_test.go
new file mode 100644
index 0000000..607ba5c
--- /dev/null
+++ b/server/e2e/restore_cluster_test.go
@@ -0,0 +1,322 @@
+package e2e
+
+// Restoring a cluster's nodes (docs/backup-restore.md):
+// a lost node starts empty and rebuilds from the others; a node started at a
+// checkpoint while its peers run is refused; a whole cluster starts at one
+// backup run's checkpoints.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	rh "github.com/unit-io/unitdb/server/internal/pkg/hash"
+)
+
+// restartAt restarts s with -db_path at dbPath and extra arguments (the
+// last start's -restored and -journal dropped), and waits until it takes
+// connections.
+func (s *server) restartAt(dbPath string, extra ...string) error {
+	s.stop()
+	args := []string{s.cmd.Args[0]}
+	for i := 1; i < len(s.cmd.Args); i++ {
+		switch a := s.cmd.Args[i]; a {
+		case "-restored": // a flag of the last start only
+		case "-journal": // with its value, as -restored's companion
+			i++
+		case "-db_path":
+			args = append(args, "-db_path", dbPath)
+			i++
+		default:
+			args = append(args, a)
+		}
+	}
+	s.cmd.Args = append(args, extra...)
+	return s.start()
+}
+
+// dbPathArg returns the -db_path s was started with.
+func (s *server) dbPathArg() string {
+	for i, a := range s.cmd.Args {
+		if a == "-db_path" && i+1 < len(s.cmd.Args) {
+			return s.cmd.Args[i+1]
+		}
+	}
+	return ""
+}
+
+// refusedAt starts s with -db_path at dbPath and extra arguments, expecting
+// it to refuse: it returns what the server logged before it exited.
+func (s *server) refusedAt(t *testing.T, dbPath string, extra ...string) string {
+	t.Helper()
+	saved := append([]string(nil), s.cmd.Args...)
+	s.noWait = true
+	defer func() { s.cmd.Args, s.noWait = saved, false }()
+	before := len(s.logs.String())
+	if err := s.restartAt(dbPath, extra...); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case <-s.exited:
+	case <-time.After(20 * time.Second):
+		s.stop()
+		t.Fatalf("started at %s %v; want it refused\nlogs:\n%s", dbPath, extra, s.logs.String()[before:])
+	}
+	return s.logs.String()[before:]
+}
+
+// clusterCheckpoint takes a checkpoint on n and returns its directory and
+// description.
+func clusterCheckpoint(t *testing.T, n *clusterNode) (string, map[string]interface{}) {
+	t.Helper()
+	code, body := checkpointPost(t, n.server, checkpointToken)
+	if code != http.StatusOK {
+		t.Fatalf("checkpoint on %s: %d %s", n.name, code, body)
+	}
+	var ck map[string]interface{}
+	if err := json.Unmarshal([]byte(body), &ck); err != nil {
+		t.Fatalf("checkpoint answer %q", body)
+	}
+	dir, _ := ck["dir"].(string)
+	return dir, ck
+}
+
+// checkpointEnv has every node take checkpoints into a directory of its own.
+func checkpointEnv(t *testing.T) map[string][]string {
+	env := map[string][]string{}
+	for _, name := range names {
+		env[name] = []string{"CHECKPOINT_DIR=" + t.TempDir(), "CHECKPOINT_TOKEN=" + checkpointToken, "CHECKPOINT_MIN_MINUTES=0"}
+	}
+	return env
+}
+
+// publishAcked publishes n messages on each topic through node, each
+// acknowledged.
+func publishAcked(t *testing.T, node *clusterNode, cid string, n int, topics ...string) {
+	t.Helper()
+	p, err := dial(context.Background(), node.tcpAddr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer p.close()
+	if _, err := p.connect(cid, true, nextSess()); err != nil {
+		t.Fatal(err)
+	}
+	for _, topic := range topics {
+		for i := 0; i < n; i++ {
+			id, err := p.publish(1, topic, encodePayload(i, fmt.Sprintf("%s/%d", topic, i)), "24h")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !p.waitAck(id, 5*time.Second) {
+				t.Fatalf("publish %d on %s: no RECEIPT", i, topic)
+			}
+		}
+	}
+}
+
+// checkStored fails t unless node relays all n messages of each topic.
+func checkStored(t *testing.T, node *clusterNode, cid string, n int, topics ...string) {
+	t.Helper()
+	for _, topic := range topics {
+		got := storedOn(t, node.server, cid, topic, n)
+		for i := 0; i < n; i++ {
+			if want := fmt.Sprintf("%s/%d", topic, i); got[i] != want {
+				t.Errorf("%s relays message %d of %s as %q, want %q (has %d)", node.name, i, topic, got[i], want, len(got))
+				break
+			}
+		}
+	}
+}
+
+// TestRestoreLostNodeStartsEmpty is the drill for restoring a lost node,
+// the other two up: its store is deleted and it starts empty. It takes no
+// clients until it has every message acknowledged before (/_readyz), and
+// then has them on its own, the topics' other holders stopped.
+func TestRestoreLostNodeStartsEmpty(t *testing.T) {
+	c := startCluster(t, names...)
+	leader, err := c.waitLeader(c.nodes, 10*time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	victim := c.node(followerOf(leader))
+	var live *clusterNode
+	for _, n := range c.nodes {
+		if n != victim {
+			live = n
+			break
+		}
+	}
+	contract := uint32(0x0e2e5e5e)
+	cid := newClientID(contract)
+	ring := rh.NewRing(clusterHashReplicas, nil)
+	ring.Add(names...)
+	holders := func(topic string) []string { return ring.GetN(fmt.Sprintf("%d/%s", contract, topic), 2) }
+	owned := topicOwnedBy(victim.name, contract, "groups.lost.owned", names...)
+	var replicated string
+	for i := 0; ; i++ {
+		topic := fmt.Sprintf("groups.lost.replica.t%d", i)
+		if h := holders(topic); h[0] != victim.name && h[1] == victim.name {
+			replicated = topic
+			break
+		}
+	}
+	n := scaled(200)
+	publishAcked(t, live, cid, n, owned, replicated)
+	time.Sleep(500 * time.Millisecond) // replication is asynchronous
+
+	// The node is lost with its store, and starts empty.
+	victim.stop()
+	if err := os.RemoveAll(filepath.Join(victim.dbPath, "db")); err != nil {
+		t.Fatal(err)
+	}
+	if err := victim.start(); err != nil {
+		t.Fatalf("restart %s: %v", victim.name, err)
+	}
+	t.Logf("%s before it was ready: %v", victim.name, waitReadyz(t, victim, 30*time.Second))
+
+	// Ready, it has them all, alone.
+	for _, topic := range []string{owned, replicated} {
+		for _, h := range holders(topic) {
+			if h != victim.name {
+				c.node(h).stop()
+			}
+		}
+	}
+	checkStored(t, victim, cid, n, owned, replicated)
+}
+
+// TestRestoreCheckpointRefusedInRunningCluster checks the guard: a node
+// started at its own checkpoint while its peers run is refused, and says to
+// start empty or with -restored. With -restored it catches up with them:
+// what was written since its checkpoint is on it, alone. -restored on a
+// store that isn't a checkpoint is refused. Started empty, it rebuilds.
+func TestRestoreCheckpointRefusedInRunningCluster(t *testing.T) {
+	c := startClusterWith(t, clusterOpts{env: checkpointEnv(t)}, names...)
+	leader, err := c.waitLeader(c.nodes, 10*time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	victim := c.node(followerOf(leader))
+	cid := newClientID(0x0e2e6a4d)
+	topic := topicOwnedBy(victim.name, 0x0e2e6a4d, "groups.guard.owned", names...)
+	publishAcked(t, victim, cid, 20, topic)
+
+	dir, ck := clusterCheckpoint(t, victim)
+	if ck["node"] != victim.name || ck["engine"] == "" || ck["time"] == nil {
+		t.Errorf("checkpoint answer %v", ck)
+	}
+	if stats, _ := ck["stats"].(map[string]interface{}); stats == nil || stats["messages"] == nil || stats["messages"].(float64) < 20 {
+		t.Errorf("checkpoint stats %v", ck["stats"])
+	}
+	if _, err := os.Stat(filepath.Join(dir, "checkpoint.json")); err != nil {
+		t.Errorf("no checkpoint.json in %s: %v", dir, err)
+	}
+
+	logs := victim.refusedAt(t, dir)
+	if !strings.Contains(logs, "Refusing to start") || !strings.Contains(logs, "empty store") {
+		t.Errorf("refused at a checkpoint, it says:\n%s", logs)
+	}
+	logs = victim.refusedAt(t, filepath.Join(victim.dbPath, "db"), "-restored")
+	if !strings.Contains(logs, "isn't a checkpoint") {
+		t.Errorf("refused with -restored on its own store, it says:\n%s", logs)
+	}
+
+	// Written since its checkpoint, while it was down: the topic's replica
+	// took it.
+	publishBodies(t, c.node(leader), cid, topic, bodies("since", 10)...)
+	time.Sleep(500 * time.Millisecond)
+	if err := victim.restartAt(dir, "-restored"); err != nil {
+		t.Fatalf("start with -restored: %v\nlogs:\n%s", err, victim.logs.String())
+	}
+	waitReadyz(t, victim, 30*time.Second)
+	var want []string
+	for i := 0; i < 20; i++ {
+		want = append(want, fmt.Sprintf("%s/%d", topic, i))
+	}
+	sameCounts(t, "the owner, restored into the running cluster", relayCounts(t, victim, cid, topic), append(want, bodies("since", 10)...))
+
+	// Its own store, which isn't a checkpoint, starts as before; so does an
+	// empty one.
+	if err := victim.restartAt(filepath.Join(victim.dbPath, "db")); err != nil {
+		t.Fatalf("restart on its own store: %v\nlogs:\n%s", err, victim.logs.String())
+	}
+	if err := victim.restartAt(filepath.Join(victim.dbPath, "empty")); err != nil {
+		t.Fatalf("restart empty: %v\nlogs:\n%s", err, victim.logs.String())
+	}
+	waitReadyz(t, victim, 30*time.Second)
+	sameCounts(t, "the owner, rebuilt", relayCounts(t, victim, cid, topic), append(want, bodies("since", 10)...))
+}
+
+// TestRestoreWholeClusterFromOneRun restores every node at its checkpoint of
+// one run, started one after another: each finds its peers restored from
+// the same run, and starts. A node whose checkpoint is of another run is
+// refused.
+func TestRestoreWholeClusterFromOneRun(t *testing.T) {
+	c := startClusterWith(t, clusterOpts{env: checkpointEnv(t)}, names...)
+	if _, err := c.waitLeader(c.nodes, 10*time.Second); err != nil {
+		t.Fatal(err)
+	}
+	contract := uint32(0x0e2e7b0b)
+	cid := newClientID(contract)
+	var topics []string
+	for _, n := range c.nodes {
+		topics = append(topics, topicOwnedBy(n.name, contract, "groups.whole.owned", names...))
+	}
+	publishAcked(t, c.nodes[0], cid, 20, topics...)
+	time.Sleep(500 * time.Millisecond) // replication is asynchronous
+
+	// One run: a checkpoint of every node; then every node is lost.
+	dirs := map[string]string{}
+	for _, n := range c.nodes {
+		dirs[n.name], _ = clusterCheckpoint(t, n)
+	}
+	for _, n := range c.nodes {
+		n.stop()
+	}
+
+	// The last node's checkpoint is made to be of another run: refused once
+	// the first two are up.
+	last := c.nodes[len(c.nodes)-1]
+	infoPath := filepath.Join(dirs[last.name], "checkpoint.json")
+	orig, err := os.ReadFile(infoPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var info map[string]interface{}
+	json.Unmarshal(orig, &info)
+	taken, _ := time.Parse(time.RFC3339Nano, info["time"].(string))
+	info["time"] = taken.Add(-2 * time.Hour).Format(time.RFC3339Nano)
+	other, _ := json.Marshal(info)
+
+	for _, n := range c.nodes[:len(c.nodes)-1] {
+		if err := n.restartAt(dirs[n.name]); err != nil {
+			t.Fatalf("start %s at its checkpoint: %v\nlogs:\n%s", n.name, err, n.logs.String())
+		}
+	}
+	if err := os.WriteFile(infoPath, other, 0600); err != nil {
+		t.Fatal(err)
+	}
+	if logs := last.refusedAt(t, dirs[last.name]); !strings.Contains(logs, "not the same backup run") {
+		t.Errorf("a checkpoint of another run, it says:\n%s", logs)
+	}
+	if err := os.WriteFile(infoPath, orig, 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := last.restartAt(dirs[last.name]); err != nil {
+		t.Fatalf("start %s at its checkpoint: %v\nlogs:\n%s", last.name, err, last.logs.String())
+	}
+
+	for _, n := range c.nodes {
+		waitReadyz(t, n, 30*time.Second)
+	}
+	for _, n := range c.nodes {
+		checkStored(t, n, cid, 20, topics...)
+	}
+}
diff --git a/server/e2e/restore_test_run_test.go b/server/e2e/restore_test_run_test.go
new file mode 100644
index 0000000..88f9c94
--- /dev/null
+++ b/server/e2e/restore_test_run_test.go
@@ -0,0 +1,107 @@
+package e2e
+
+// The weekly restore test (docs/backup-restore.md): the
+// backup command's verify opens each node's checkpoint of the newest run on
+// a scratch server and checks it, and pushes its success; a corrupted
+// archive, or an escrow without the run's key, fails it, and nothing is
+// pushed.
+
+import (
+	"context"
+	"encoding/base64"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"filippo.io/age"
+
+	"github.com/unit-io/unitdb/server/internal/backup"
+)
+
+func TestBackupRestoreTestVerifiesRun(t *testing.T) {
+	endpoint := startMinIO(t)
+	bin := buildBackup(t)
+	id, err := age.GenerateX25519Identity()
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := startClusterWith(t, clusterOpts{env: offsiteEnv(t, endpoint, id.Recipient().String())}, names...)
+	if _, err := c.waitLeader(c.nodes, 10*time.Second); err != nil {
+		t.Fatal(err)
+	}
+	cid := newClientID(0x0e2ee0e0)
+	publishAcked(t, c.nodes[0], cid, 30, "groups.verify.a", "groups.verify.b")
+	time.Sleep(500 * time.Millisecond)
+	m := runBackupUpload(t, bin, c.nodes)
+
+	// The Pushgateway, as the test sees it.
+	var mu sync.Mutex
+	var pushes []string
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, _ := io.ReadAll(r.Body)
+		mu.Lock()
+		pushes = append(pushes, r.Method+" "+r.URL.Path+"\n"+string(b))
+		mu.Unlock()
+	}))
+	defer gw.Close()
+	keyringFile := filepath.Join(t.TempDir(), "keyring.json")
+	os.WriteFile(keyringFile, []byte(`[{"id":0,"key":"`+base64.StdEncoding.EncodeToString([]byte(testKey))+`","use":"issue"}]`), 0600)
+	verify := func(keyring string) (string, error) {
+		cmd := exec.Command(bin, "verify", "-run", "latest", "-server", serverBinary(t), "-keyring", keyring, "-pushgateway", gw.URL, "-start_timeout", "60s")
+		cmd.Env = append(os.Environ(),
+			"BACKUP_S3_BUCKET="+offsiteBucket, "BACKUP_S3_ENDPOINT="+endpoint, "BACKUP_S3_REGION=us-east-1", "BACKUP_CLUSTER=e2e",
+			"AWS_ACCESS_KEY_ID="+minioUser, "AWS_SECRET_ACCESS_KEY="+minioPassword, "BACKUP_AGE_IDENTITY="+id.String())
+		out, err := cmd.CombinedOutput()
+		return string(out), err
+	}
+
+	out, err := verify(keyringFile)
+	if err != nil {
+		t.Fatalf("verify: %v\n%s", err, out)
+	}
+	if !strings.Contains(out, "run "+m.Run+" verified: 3 nodes") {
+		t.Errorf("verify said:\n%s", out)
+	}
+	mu.Lock()
+	if len(pushes) != 1 || !strings.Contains(pushes[0], "PUT /metrics/job/unitdb_backup_restore_test/cluster/e2e") ||
+		!strings.Contains(pushes[0], "unitdb_backup_restore_verified_timestamp_seconds ") {
+		t.Errorf("pushed %q", pushes)
+	}
+	mu.Unlock()
+
+	// An escrow without the run's key fails it.
+	other := filepath.Join(t.TempDir(), "other.json")
+	os.WriteFile(other, []byte(`[{"id":9,"key":"`+base64.StdEncoding.EncodeToString([]byte("abcdefghijabcdefghijabcdefghij12"))+`","use":"issue"}]`), 0600)
+	if out, err := verify(other); err == nil || !strings.Contains(out, "lacks key ids [0]") {
+		t.Errorf("verify with the wrong escrow: %v\n%s", err, out)
+	}
+
+	// A corrupted archive, put as the object's newest version, fails it.
+	st := offsiteStore(t, endpoint)
+	key := "e2e/" + m.Run + "/" + c.nodes[1].name + ".tar.zst.age"
+	obj, err := st.Get(context.Background(), key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, _ := io.ReadAll(obj)
+	obj.Close()
+	b[len(b)/2] ^= 0xff
+	if err := st.Put(context.Background(), key, b, backup.RunRetention(time.Now())); err != nil {
+		t.Fatal(err)
+	}
+	if out, err := verify(keyringFile); err == nil || !strings.Contains(out, c.nodes[1].name+"'s checkpoint") {
+		t.Errorf("verify of a corrupted archive: %v\n%s", err, out)
+	}
+	mu.Lock()
+	if len(pushes) != 1 {
+		t.Errorf("a failed test pushed: %d pushes", len(pushes))
+	}
+	mu.Unlock()
+}
diff --git a/server/e2e/runbook_test.go b/server/e2e/runbook_test.go
new file mode 100644
index 0000000..0657c70
--- /dev/null
+++ b/server/e2e/runbook_test.go
@@ -0,0 +1,126 @@
+package e2e
+
+// Runbook B, rehearsed (docs/backup-restore.md): the whole cluster lost,
+// restored from the bucket as the runbook says, step by step, and timed.
+// The drill on staging (docs/backup-restore.md) runs the same
+// steps on Kubernetes.
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"filippo.io/age"
+
+	"github.com/unit-io/unitdb/server/internal/types"
+)
+
+func TestRunbookWholeClusterRestore(t *testing.T) {
+	endpoint := startMinIO(t)
+	bin := buildBackup(t)
+	id, err := age.GenerateX25519Identity()
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := startClusterWith(t, clusterOpts{env: offsiteEnv(t, endpoint, id.Recipient().String())}, names...)
+	if _, err := c.waitLeader(c.nodes, 10*time.Second); err != nil {
+		t.Fatal(err)
+	}
+	contract := uint32(0x0e2ef1f1)
+	cid := newClientID(contract)
+	var topics []string
+	for _, n := range c.nodes {
+		topics = append(topics, topicOwnedBy(n.name, contract, "groups.runbook.owned", names...))
+	}
+	publishAcked(t, c.nodes[0], cid, 50, topics...)
+	time.Sleep(500 * time.Millisecond)
+	m := runBackupUpload(t, bin, c.nodes)
+	// After the run: a revocation, which the journal keeps.
+	revokedID := newClientID(contract)
+	uid, _, _ := openClientID(revokedID)
+	pub, err := connectTo(t, c.nodes[0].tcpAddr, connectOpts{clientID: primaryClientID(contract)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	revoke(t, pub, types.RevokeRequest{Uuid: strconv.FormatUint(uid.Uuid(), 10)})
+	for deadline := time.Now().Add(15 * time.Second); !journalLagZero(t, c.nodes[0]); time.Sleep(250 * time.Millisecond) {
+		if time.Now().After(deadline) {
+			t.Fatal("the journal didn't go up")
+		}
+	}
+
+	// Step 1: the cluster is lost.
+	for _, n := range c.nodes {
+		n.stop()
+	}
+	tool := func(args ...string) string {
+		t.Helper()
+		cmd := exec.Command(bin, args...)
+		cmd.Env = append(os.Environ(),
+			"BACKUP_S3_BUCKET="+offsiteBucket, "BACKUP_S3_ENDPOINT="+endpoint, "BACKUP_S3_REGION=us-east-1", "BACKUP_CLUSTER=e2e",
+			"AWS_ACCESS_KEY_ID="+minioUser, "AWS_SECRET_ACCESS_KEY="+minioPassword, "BACKUP_AGE_IDENTITY="+id.String())
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("backup %v: %v\n%s", args, err, out)
+		}
+		return string(out)
+	}
+	start := time.Now()
+
+	// Step 4: the newest complete run.
+	runs := tool("runs")
+	if first := strings.SplitN(runs, "\n", 2)[0]; first != m.Run+"\tcomplete" {
+		t.Fatalf("backup runs: %q, want %s first, complete", runs, m.Run)
+	}
+
+	// Step 5: each node's new claim gets its checkpoint and the journal.
+	claims := map[string]string{}
+	for _, n := range c.nodes {
+		claim := t.TempDir()
+		claims[n.name] = claim
+		tool("fetch", "-run="+m.Run, "-node="+n.name, "-out="+filepath.Join(claim, "unitdb"))
+		tool("fetch-journal", "-since="+m.Run, "-out="+filepath.Join(claim, "restore-journal"))
+	}
+	fetched := time.Since(start)
+
+	// Step 6: every node with -restored and -journal; ready once
+	// reconciled.
+	for _, n := range c.nodes {
+		claim := claims[n.name]
+		if err := n.restartAt(filepath.Join(claim, "unitdb"), "-restored", "-journal", filepath.Join(claim, "restore-journal")); err != nil {
+			t.Fatalf("step 6, %s: %v\nlogs:\n%s", n.name, err, n.logs.String())
+		}
+	}
+	for _, n := range c.nodes {
+		waitReadyz(t, n, 60*time.Second)
+	}
+	ready := time.Since(start)
+	for _, n := range c.nodes {
+		if _, err := os.Stat(filepath.Join(claims[n.name], "unitdb", "restored-from.json")); err != nil {
+			t.Errorf("step 6, %s: not marked restored: %v", n.name, err)
+		}
+	}
+
+	// Step 7: the flags come out; the nodes restart as usual.
+	for _, n := range c.nodes {
+		if err := n.restartAt(filepath.Join(claims[n.name], "unitdb")); err != nil {
+			t.Fatalf("step 7, %s: %v\nlogs:\n%s", n.name, err, n.logs.String())
+		}
+	}
+	for _, n := range c.nodes {
+		waitReadyz(t, n, 60*time.Second)
+	}
+
+	// Step 8: checked.
+	for _, n := range c.nodes {
+		checkStored(t, n, cid, 50, topics...)
+		if _, err := connectTo(t, n.tcpAddr, connectOpts{clientID: revokedID}); err == nil {
+			t.Errorf("step 8: the client id revoked after the run connects to %s", n.name)
+		}
+	}
+	t.Logf("runbook B: fetched in %s, ready in %s (%d nodes, run %s)", fetched.Round(time.Millisecond), ready.Round(time.Millisecond), len(c.nodes), m.Run)
+}
diff --git a/server/internal/backup/archive.go b/server/internal/backup/archive.go
new file mode 100644
index 0000000..91d01b9
--- /dev/null
+++ b/server/internal/backup/archive.go
@@ -0,0 +1,145 @@
+package backup
+
+import (
+	"archive/tar"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"filippo.io/age"
+	"github.com/klauspost/compress/zstd"
+)
+
+// ParseRecipient parses the backup key's public half.
+func ParseRecipient(s string) (age.Recipient, error) {
+	r, err := age.ParseX25519Recipient(strings.TrimSpace(s))
+	if err != nil {
+		return nil, fmt.Errorf("backup: the backup key's public half: %v", err)
+	}
+	return r, nil
+}
+
+// ParseIdentity parses the backup key's private half, as age-keygen writes
+// it (comment lines are skipped).
+func ParseIdentity(s string) (age.Identity, error) {
+	ids, err := age.ParseIdentities(strings.NewReader(s))
+	if err != nil || len(ids) == 0 {
+		return nil, fmt.Errorf("backup: the backup key's private half: %v", err)
+	}
+	return ids[0], nil
+}
+
+// WriteArchive writes the files under dir to w as a tar archive, compressed
+// with zstd and encrypted to rcpt. Only regular files and directories go in.
+func WriteArchive(w io.Writer, dir string, rcpt age.Recipient) error {
+	enc, err := age.Encrypt(w, rcpt)
+	if err != nil {
+		return err
+	}
+	zw, err := zstd.NewWriter(enc)
+	if err != nil {
+		return err
+	}
+	tw := tar.NewWriter(zw)
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, err := filepath.Rel(dir, path)
+		if err != nil || rel == "." {
+			return err
+		}
+		if !info.IsDir() && !info.Mode().IsRegular() {
+			return nil
+		}
+		hdr, err := tar.FileInfoHeader(info, "")
+		if err != nil {
+			return err
+		}
+		hdr.Name = filepath.ToSlash(rel)
+		if info.IsDir() {
+			hdr.Name += "/"
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		f, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		_, err = io.Copy(tw, f)
+		return err
+	})
+	if err != nil {
+		return err
+	}
+	if err := tw.Close(); err != nil {
+		return err
+	}
+	if err := zw.Close(); err != nil {
+		return err
+	}
+	return enc.Close()
+}
+
+// ExtractArchive decrypts r with id and extracts it into dst, which must
+// not exist or be empty. A name that would land outside dst is refused.
+func ExtractArchive(r io.Reader, id age.Identity, dst string) error {
+	if entries, err := os.ReadDir(dst); err == nil && len(entries) > 0 {
+		return fmt.Errorf("backup: %s is not empty", dst)
+	}
+	if err := os.MkdirAll(dst, 0700); err != nil {
+		return err
+	}
+	dec, err := age.Decrypt(r, id)
+	if err != nil {
+		return fmt.Errorf("backup: decrypt: %v", err)
+	}
+	zr, err := zstd.NewReader(dec)
+	if err != nil {
+		return err
+	}
+	defer zr.Close()
+	tr := tar.NewReader(zr)
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("backup: archive: %v", err)
+		}
+		target := filepath.Join(dst, filepath.FromSlash(hdr.Name))
+		if !strings.HasPrefix(target, filepath.Clean(dst)+string(os.PathSeparator)) {
+			return fmt.Errorf("backup: archive: %q is outside it", hdr.Name)
+		}
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, 0700); err != nil {
+				return err
+			}
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(target), 0700); err != nil {
+				return err
+			}
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, os.FileMode(hdr.Mode).Perm()|0600)
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(f, tr); err != nil {
+				f.Close()
+				return err
+			}
+			if err := f.Close(); err != nil {
+				return err
+			}
+		}
+	}
+}
diff --git a/server/internal/backup/backup.go b/server/internal/backup/backup.go
new file mode 100644
index 0000000..6a1cc87
--- /dev/null
+++ b/server/internal/backup/backup.go
@@ -0,0 +1,119 @@
+// Package backup keeps unitdb's backups off the cluster
+// (docs/backup-restore.md): each node's checkpoint of a run,
+// archived (tar + zstd) and encrypted to the backup key's public half (age),
+// and the security journal, in an S3 bucket with Object Lock. The cluster
+// writes with an identity that may only put objects: it can neither read
+// nor erase its backups. Reading them (fetch, the restore test) needs the
+// backup key's private half and a reader's identity, kept apart.
+//
+// The bucket's layout, under one prefix per cluster:
+//
+//	<cluster>/<run>/manifest.json
+//	<cluster>/<run>/<node>.tar.zst.age
+//	<cluster>/journal/<date>/<node>-<time>.jsonl
+package backup
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+)
+
+// Config is where a cluster's backups go, from the environment:
+//
+//	BACKUP_S3_BUCKET     the bucket; uploads are off without it
+//	BACKUP_S3_REGION     the region; the AWS SDK's (AWS_REGION, a profile) if unset
+//	BACKUP_S3_ENDPOINT   another S3 endpoint (MinIO), path-style; AWS if unset
+//	BACKUP_CLUSTER       the cluster's prefix in the bucket, e.g. production
+//	BACKUP_AGE_RECIPIENT the backup key's public half (age1...)
+//
+// Credentials come from the AWS SDK's chain: AWS_ACCESS_KEY_ID and
+// AWS_SECRET_ACCESS_KEY from a Secret, or a profile.
+type Config struct {
+	Bucket    string
+	Region    string
+	Endpoint  string
+	Cluster   string
+	Recipient string
+}
+
+// ConfigFromEnv returns the configuration the environment sets, or nil if
+// there is no bucket.
+func ConfigFromEnv() (*Config, error) {
+	c := &Config{
+		Bucket:    os.Getenv("BACKUP_S3_BUCKET"),
+		Region:    os.Getenv("BACKUP_S3_REGION"),
+		Endpoint:  os.Getenv("BACKUP_S3_ENDPOINT"),
+		Cluster:   os.Getenv("BACKUP_CLUSTER"),
+		Recipient: os.Getenv("BACKUP_AGE_RECIPIENT"),
+	}
+	if c.Bucket == "" {
+		return nil, nil
+	}
+	if c.Cluster == "" || strings.ContainsAny(c.Cluster, "/ ") {
+		return nil, fmt.Errorf("backup: BACKUP_CLUSTER names the cluster's prefix in the bucket, without / or spaces")
+	}
+	return c, nil
+}
+
+// RunPrefix is where a run's objects are.
+func (c *Config) RunPrefix(run string) string {
+	return c.Cluster + "/" + run + "/"
+}
+
+// CheckpointKey is the object of a node's checkpoint of a run.
+func (c *Config) CheckpointKey(run, node string) string {
+	return c.RunPrefix(run) + node + ".tar.zst.age"
+}
+
+// ManifestKey is the object of a run's manifest.
+func (c *Config) ManifestKey(run string) string {
+	return c.RunPrefix(run) + "manifest.json"
+}
+
+// JournalPrefix is where the security journal is.
+func (c *Config) JournalPrefix() string {
+	return c.Cluster + "/journal/"
+}
+
+// JournalKey is the object of one batch of a node's journal, uploaded at t.
+func (c *Config) JournalKey(node string, t time.Time) string {
+	t = t.UTC()
+	return fmt.Sprintf("%s%s/%s-%s.jsonl", c.JournalPrefix(), t.Format("2006-01-02"), node, t.Format("20060102T150405.000000000Z"))
+}
+
+// Retention is how long an object can't be deleted or overwritten, and the
+// tag the bucket's lifecycle rules expire it by.
+type Retention struct {
+	Tier  string // daily, weekly, monthly or journal
+	Until time.Time
+}
+
+// Retentions kept: 7 daily, 4 weekly and 12 monthly runs, and the journal
+// as long as the oldest run. Each object is locked a day longer than it is
+// kept, and the lifecycle rules expire it after (deploy/aws/README.md).
+const (
+	dailyKeep   = 7 * 24 * time.Hour
+	weeklyKeep  = 28 * 24 * time.Hour
+	monthlyKeep = 395 * 24 * time.Hour // 13 months
+)
+
+// RunRetention returns the retention of a run that began at t: the first
+// run of a month is monthly, of a week (Sunday) weekly, any other daily.
+func RunRetention(t time.Time) Retention {
+	t = t.UTC()
+	switch {
+	case t.Day() == 1:
+		return Retention{Tier: "monthly", Until: t.Add(monthlyKeep + 24*time.Hour)}
+	case t.Weekday() == time.Sunday:
+		return Retention{Tier: "weekly", Until: t.Add(weeklyKeep + 24*time.Hour)}
+	}
+	return Retention{Tier: "daily", Until: t.Add(dailyKeep + 24*time.Hour)}
+}
+
+// JournalRetention is the retention of a journal batch uploaded at t: as
+// long as a monthly run, since a restore of the oldest run replays it.
+func JournalRetention(t time.Time) Retention {
+	return Retention{Tier: "journal", Until: t.UTC().Add(monthlyKeep + 24*time.Hour)}
+}
diff --git a/server/internal/backup/backup_test.go b/server/internal/backup/backup_test.go
new file mode 100644
index 0000000..145dcb3
--- /dev/null
+++ b/server/internal/backup/backup_test.go
@@ -0,0 +1,120 @@
+package backup
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"filippo.io/age"
+)
+
+func TestArchiveRoundTrip(t *testing.T) {
+	src := t.TempDir()
+	os.MkdirAll(filepath.Join(src, "unitdb", "sub"), 0700)
+	os.WriteFile(filepath.Join(src, "checkpoint.json"), []byte(`{"node":"a"}`), 0600)
+	big := bytes.Repeat([]byte("unitdb "), 100000)
+	os.WriteFile(filepath.Join(src, "unitdb", "sub", "data"), big, 0600)
+	id, err := age.GenerateX25519Identity()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var buf bytes.Buffer
+	if err := WriteArchive(&buf, src, id.Recipient()); err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Contains(buf.Bytes(), []byte(`"node":"a"`)) || bytes.Contains(buf.Bytes(), []byte("unitdb unitdb")) {
+		t.Fatal("the archive isn't encrypted")
+	}
+	if buf.Len() > len(big)/10 {
+		t.Errorf("the archive is %d bytes for %d compressible ones", buf.Len(), len(big))
+	}
+
+	dst := filepath.Join(t.TempDir(), "out")
+	if err := ExtractArchive(bytes.NewReader(buf.Bytes()), id, dst); err != nil {
+		t.Fatal(err)
+	}
+	if b, _ := os.ReadFile(filepath.Join(dst, "unitdb", "sub", "data")); !bytes.Equal(b, big) {
+		t.Error("the extracted file differs")
+	}
+	if b, _ := os.ReadFile(filepath.Join(dst, "checkpoint.json")); string(b) != `{"node":"a"}` {
+		t.Errorf("checkpoint.json: %q", b)
+	}
+
+	// Another key doesn't open it; a directory that isn't empty is refused.
+	other, _ := age.GenerateX25519Identity()
+	if err := ExtractArchive(bytes.NewReader(buf.Bytes()), other, filepath.Join(t.TempDir(), "x")); err == nil {
+		t.Error("another key opened the archive")
+	}
+	if err := ExtractArchive(bytes.NewReader(buf.Bytes()), id, dst); err == nil {
+		t.Error("extracted into a directory that isn't empty")
+	}
+}
+
+func TestRunRetention(t *testing.T) {
+	day := func(s string) time.Time { d, _ := time.Parse("2006-01-02T15:04", s); return d }
+	for _, tc := range []struct {
+		run  string
+		tier string
+		days int
+	}{
+		{"2026-10-01T21:30", "monthly", 396}, // the 1st, a Thursday
+		{"2026-10-04T21:30", "weekly", 29},   // a Sunday
+		{"2026-11-01T21:30", "monthly", 396}, // a Sunday and the 1st
+		{"2026-10-06T21:30", "daily", 8},
+	} {
+		r := RunRetention(day(tc.run))
+		if got := int(r.Until.Sub(day(tc.run)).Hours() / 24); r.Tier != tc.tier || got != tc.days {
+			t.Errorf("%s: %s for %d days, want %s for %d", tc.run, r.Tier, got, tc.tier, tc.days)
+		}
+	}
+}
+
+func TestMergeKeyring(t *testing.T) {
+	cur := `[{"id":1,"key":"k1","use":"read"},{"id":2,"key":"k2","use":"issue"}]`
+	got, err := MergeKeyring(`[{"id":0,"key":"k0","use":"issue"},{"id":1,"key":"k1","use":"issue"}]`, cur)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := `[{"id":0,"key":"k0","use":"read"},{"id":1,"key":"k1","use":"read"},{"id":2,"key":"k2","use":"issue"}]`
+	if got != want {
+		t.Errorf("merged %s\nwant   %s", got, want)
+	}
+	if _, err := MergeKeyring(`[{"id":2,"key":"other","use":"issue"}]`, cur); err == nil || !strings.Contains(err.Error(), "never reused") {
+		t.Errorf("a key id reused: %v", err)
+	}
+	if got, err := MergeKeyring("", cur); err != nil || got != `[{"id":1,"key":"k1","use":"read"},{"id":2,"key":"k2","use":"issue"}]` {
+		t.Errorf("into an empty escrow: %s %v", got, err)
+	}
+}
+
+type fakeSecrets map[string]string
+
+func (f fakeSecrets) Get(_ context.Context, name string) (string, bool, error) {
+	v, ok := f[name]
+	return v, ok, nil
+}
+
+func (f fakeSecrets) Put(_ context.Context, name, value string) error {
+	f[name] = value
+	return nil
+}
+
+func TestEscrowKeyring(t *testing.T) {
+	ss := fakeSecrets{}
+	ctx := context.Background()
+	if err := EscrowKeyring(ctx, ss, "staging", `[{"id":0,"key":"k0","use":"issue"}]`); err != nil {
+		t.Fatal(err)
+	}
+	// The key rotates: the old one stays in escrow, as a read key.
+	if err := EscrowKeyring(ctx, ss, "staging", `[{"id":1,"key":"k1","use":"issue"}]`); err != nil {
+		t.Fatal(err)
+	}
+	ids, err := EscrowedKeyIDs(ss[KeyringSecret("staging")])
+	if err != nil || !ids[0] || !ids[1] {
+		t.Errorf("escrowed %s (%v)", ss[KeyringSecret("staging")], err)
+	}
+}
diff --git a/server/internal/backup/escrow.go b/server/internal/backup/escrow.go
new file mode 100644
index 0000000..a2a1985
--- /dev/null
+++ b/server/internal/backup/escrow.go
@@ -0,0 +1,150 @@
+package backup
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
+	smtypes "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types"
+)
+
+// The keyring is escrowed apart from the backups, in AWS Secrets Manager:
+// a backup restores only with the keyring of its time (records sealed at
+// rest, client ids and topic keys). Every change of the keyring is put
+// there first, and a key leaves the escrow only when no backup within
+// retention names it (the manifests' key_ids): the escrow holds every key,
+// those retired from the keyring as "read" ones.
+
+// KeyringSecret is the name of a cluster's escrowed keyring.
+func KeyringSecret(cluster string) string {
+	return "unitdb/" + cluster + "/keyring"
+}
+
+// BackupKeySecret is the name of the backup key's private half.
+func BackupKeySecret(cluster string) string {
+	return "unitdb/" + cluster + "/backup-key"
+}
+
+// SecretStore holds secrets by name.
+type SecretStore interface {
+	// Get returns the secret's value; found is false if there is none.
+	Get(ctx context.Context, name string) (value string, found bool, err error)
+	// Put sets the secret's value, making it if there is none.
+	Put(ctx context.Context, name, value string) error
+}
+
+// keyringKey is a key of the keyring, as UNITDB_KEYRING holds it.
+type keyringKey struct {
+	ID  int    `json:"id"`
+	Key string `json:"key"`
+	Use string `json:"use"`
+}
+
+// MergeKeyring returns the escrow after current, the keyring in use, is put
+// in it: every key of both, by id, with current's use; a key the keyring
+// no longer has is kept as a "read" key. The same id with another key is
+// an error: a key is never replaced.
+func MergeKeyring(escrowed, current string) (string, error) {
+	var esc, cur []keyringKey
+	if escrowed != "" {
+		if err := json.Unmarshal([]byte(escrowed), &esc); err != nil {
+			return "", fmt.Errorf("backup: the escrowed keyring: %v", err)
+		}
+	}
+	if err := json.Unmarshal([]byte(current), &cur); err != nil || len(cur) == 0 {
+		return "", fmt.Errorf("backup: the keyring: %v", err)
+	}
+	byID := make(map[int]keyringKey)
+	for _, k := range esc {
+		k.Use = "read"
+		byID[k.ID] = k
+	}
+	for _, k := range cur {
+		if old, ok := byID[k.ID]; ok && old.Key != k.Key {
+			return "", fmt.Errorf("backup: key %d differs from the escrowed one: a key id is never reused", k.ID)
+		}
+		byID[k.ID] = k
+	}
+	merged := make([]keyringKey, 0, len(byID))
+	for _, k := range byID {
+		merged = append(merged, k)
+	}
+	sort.Slice(merged, func(i, j int) bool { return merged[i].ID < merged[j].ID })
+	b, err := json.Marshal(merged)
+	return string(b), err
+}
+
+// EscrowKeyring puts the keyring in use into the cluster's escrow.
+func EscrowKeyring(ctx context.Context, ss SecretStore, cluster, keyring string) error {
+	name := KeyringSecret(cluster)
+	old, _, err := ss.Get(ctx, name)
+	if err != nil {
+		return err
+	}
+	merged, err := MergeKeyring(old, keyring)
+	if err != nil {
+		return err
+	}
+	if merged == old {
+		return nil
+	}
+	return ss.Put(ctx, name, merged)
+}
+
+// EscrowedKeyIDs returns the ids of the keys in an escrowed keyring.
+func EscrowedKeyIDs(escrowed string) (map[int]bool, error) {
+	var esc []keyringKey
+	if err := json.Unmarshal([]byte(escrowed), &esc); err != nil {
+		return nil, err
+	}
+	ids := make(map[int]bool, len(esc))
+	for _, k := range esc {
+		ids[k.ID] = true
+	}
+	return ids, nil
+}
+
+// AWSSecrets is AWS Secrets Manager.
+type AWSSecrets struct {
+	client *secretsmanager.Client
+}
+
+// NewAWSSecrets returns Secrets Manager in region (the AWS SDK's if empty),
+// with credentials from the AWS SDK's chain.
+func NewAWSSecrets(ctx context.Context, region string) (*AWSSecrets, error) {
+	var opts []func(*awsconfig.LoadOptions) error
+	if region != "" {
+		opts = append(opts, awsconfig.WithRegion(region))
+	}
+	cfg, err := awsconfig.LoadDefaultConfig(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return &AWSSecrets{client: secretsmanager.NewFromConfig(cfg)}, nil
+}
+
+func (a *AWSSecrets) Get(ctx context.Context, name string) (string, bool, error) {
+	out, err := a.client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{SecretId: aws.String(name)})
+	var nf *smtypes.ResourceNotFoundException
+	if errors.As(err, &nf) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return aws.ToString(out.SecretString), true, nil
+}
+
+func (a *AWSSecrets) Put(ctx context.Context, name, value string) error {
+	_, err := a.client.PutSecretValue(ctx, &secretsmanager.PutSecretValueInput{SecretId: aws.String(name), SecretString: aws.String(value)})
+	var nf *smtypes.ResourceNotFoundException
+	if errors.As(err, &nf) {
+		_, err = a.client.CreateSecret(ctx, &secretsmanager.CreateSecretInput{Name: aws.String(name), SecretString: aws.String(value)})
+	}
+	return err
+}
diff --git a/server/internal/backup/s3.go b/server/internal/backup/s3.go
new file mode 100644
index 0000000..55f6dab
--- /dev/null
+++ b/server/internal/backup/s3.go
@@ -0,0 +1,98 @@
+package backup
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager"
+	tmtypes "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager/types"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+// Store is the bucket a cluster's backups go to.
+type Store struct {
+	Config *Config
+	client *s3.Client
+}
+
+// NewStore returns the store c names, with credentials from the AWS SDK's
+// chain.
+func NewStore(ctx context.Context, c *Config) (*Store, error) {
+	var opts []func(*awsconfig.LoadOptions) error
+	if c.Region != "" {
+		opts = append(opts, awsconfig.WithRegion(c.Region))
+	}
+	awsCfg, err := awsconfig.LoadDefaultConfig(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+	client := s3.NewFromConfig(awsCfg, func(o *s3.Options) {
+		if c.Endpoint != "" {
+			o.BaseEndpoint = aws.String(c.Endpoint)
+			o.UsePathStyle = true
+		}
+	})
+	return &Store{Config: c, client: client}, nil
+}
+
+// Client is the store's S3 client, for tests and tools.
+func (s *Store) Client() *s3.Client {
+	return s.client
+}
+
+// Upload writes r to key, in parts as it comes, locked in compliance mode
+// until ret.Until and tagged with its tier. A put-only identity can do it:
+// s3:PutObject, s3:PutObjectRetention, s3:PutObjectTagging and, for a
+// failed upload, s3:AbortMultipartUpload.
+func (s *Store) Upload(ctx context.Context, key string, r io.Reader, ret Retention) error {
+	_, err := transfermanager.New(s.client).UploadObject(ctx, &transfermanager.UploadObjectInput{
+		Bucket:                    aws.String(s.Config.Bucket),
+		Key:                       aws.String(key),
+		Body:                      r,
+		ObjectLockMode:            tmtypes.ObjectLockModeCompliance,
+		ObjectLockRetainUntilDate: aws.Time(ret.Until),
+		Tagging:                   aws.String("tier=" + ret.Tier),
+		// Object Lock needs a checksum of every part.
+		ChecksumAlgorithm: tmtypes.ChecksumAlgorithmCrc32,
+	})
+	return err
+}
+
+// Put writes b to key, as Upload does.
+func (s *Store) Put(ctx context.Context, key string, b []byte, ret Retention) error {
+	return s.Upload(ctx, key, bytes.NewReader(b), ret)
+}
+
+// JournalKey is the object of one batch of a node's journal, uploaded at t.
+func (s *Store) JournalKey(node string, t time.Time) string {
+	return s.Config.JournalKey(node, t)
+}
+
+// Get returns the object at key. It needs a reader's identity.
+func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
+	out, err := s.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(s.Config.Bucket), Key: aws.String(key)})
+	if err != nil {
+		return nil, err
+	}
+	return out.Body, nil
+}
+
+// List returns the keys under prefix. It needs a reader's identity.
+func (s *Store) List(ctx context.Context, prefix string) ([]string, error) {
+	var keys []string
+	p := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{Bucket: aws.String(s.Config.Bucket), Prefix: aws.String(prefix)})
+	for p.HasMorePages() {
+		page, err := p.NextPage(ctx)
+		if err != nil {
+			return nil, err
+		}
+		for _, o := range page.Contents {
+			keys = append(keys, aws.ToString(o.Key))
+		}
+	}
+	return keys, nil
+}
diff --git a/server/internal/checkpoint.go b/server/internal/checkpoint.go
index 4c044ab..343aaf8 100644
--- a/server/internal/checkpoint.go
+++ b/server/internal/checkpoint.go
@@ -4,9 +4,11 @@ import (
 	"crypto/subtle"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -35,6 +37,31 @@ import (
 
 const checkpointPrefix = "ckpt-"
 
+// A backup run's id: when it began, in UTC. The job sends it with each of
+// the run's checkpoints (POST /_checkpoint?run=<id>), which are named
+// ckpt-<run>-<node>, so that a restore takes every node from one run.
+var runID = regexp.MustCompile(`^[0-9]{8}T[0-9]{6}Z$`)
+
+// runLayout is the format of a run id.
+const runLayout = "20060102T150405Z"
+
+// parseRun returns the run id raw names, as a new string formatted from
+// the time it parses to, and when the run began: only that string goes into
+// a path, never what a request sent. ok is false if raw isn't a run id.
+func parseRun(raw string) (run string, at time.Time, ok bool) {
+	if !runID.MatchString(raw) {
+		return "", time.Time{}, false
+	}
+	at, err := time.Parse(runLayout, raw)
+	if err != nil {
+		return "", time.Time{}, false
+	}
+	return at.UTC().Format(runLayout), at, true
+}
+
+// maxManifest is the largest manifest a node keeps.
+const maxManifest = 1 << 20
+
 type checkpointer struct {
 	dir      string
 	token    string
@@ -81,6 +108,14 @@ func (c *checkpointer) handle(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "a checkpoint needs its token", http.StatusUnauthorized)
 		return
 	}
+	var run string
+	if raw := r.URL.Query().Get("run"); raw != "" {
+		var ok bool
+		if run, _, ok = parseRun(raw); !ok {
+			http.Error(w, "a run id is a UTC time, as 20060102T150405Z", http.StatusBadRequest)
+			return
+		}
+	}
 	if !c.mu.TryLock() {
 		http.Error(w, "a checkpoint is running", http.StatusConflict)
 		return
@@ -92,9 +127,26 @@ func (c *checkpointer) handle(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	c.last = now
-	dst := filepath.Join(c.dir, checkpointPrefix+now.Format("20060102T150405Z"))
+	name := checkpointPrefix + now.Format("20060102T150405Z")
+	if run != "" {
+		name = runCheckpointName(run)
+	}
+	dst := filepath.Join(c.dir, name)
+	described := describeCheckpoint(now)
+	described.Run = run
 	start := time.Now()
-	err := store.Checkpoint(dst)
+	var info store.CheckpointInfo
+	var err error
+	// A run's checkpoint holds the run's canary, for the restore test
+	// to read back.
+	if run != "" {
+		if err = store.WriteCanary(run); err != nil {
+			err = fmt.Errorf("store checkpoint: the canary: %v", err)
+		}
+	}
+	if err == nil {
+		info, err = store.Checkpoint(dst, described)
+	}
 	took := time.Since(start)
 	c.seconds.Store(took.Milliseconds())
 	if err != nil {
@@ -108,10 +160,98 @@ func (c *checkpointer) handle(w http.ResponseWriter, r *http.Request) {
 	log.Info("checkpoint", fmt.Sprintf("checkpoint %s in %s", dst, took.Round(time.Millisecond)))
 	c.prune()
 	w.Header().Set("content-type", "application/json")
-	json.NewEncoder(w).Encode(map[string]interface{}{
-		"dir":     dst,
-		"seconds": took.Seconds(),
-	})
+	json.NewEncoder(w).Encode(checkpointResp{Dir: dst, Seconds: took.Seconds(), CheckpointInfo: info})
+}
+
+// checkpointResp is the answer to POST /_checkpoint: where the checkpoint
+// is, how long it took, and its checkpoint.json.
+type checkpointResp struct {
+	Dir     string  `json:"dir"`
+	Seconds float64 `json:"seconds"`
+	store.CheckpointInfo
+}
+
+// describeCheckpoint returns what this node knows of a checkpoint it takes
+// at t: the store fills in the rest.
+func describeCheckpoint(t time.Time) store.CheckpointInfo {
+	info := store.CheckpointInfo{Time: t}
+	if c := Globals.Cluster; c != nil {
+		info.Node = c.thisNodeName
+		info.RingVersion = c.getRingVersion()
+	}
+	if s := Globals.Service; s != nil && s.keys != nil {
+		for id := range s.keys.StoreKeys() {
+			info.KeyIDs = append(info.KeyIDs, int(id))
+		}
+		sort.Ints(info.KeyIDs)
+	}
+	return info
+}
+
+// handleCanary takes GET /_checkpoint/canary: the run ids of the store's
+// canary message and record, for the restore test, which opens a
+// checkpoint on a scratch server and reads them back.
+func (c *checkpointer) handleCanary(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "GET only", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.authorized(r) {
+		http.Error(w, "the canary needs the checkpoint token", http.StatusUnauthorized)
+		return
+	}
+	msg, rec, err := store.ReadCanary()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+	w.Header().Set("content-type", "application/json")
+	json.NewEncoder(w).Encode(map[string]string{"message": msg, "record": rec})
+}
+
+// runCheckpointName is the name of this node's checkpoint of run.
+func runCheckpointName(run string) string {
+	node := "server"
+	if c := Globals.Cluster; c != nil && c.thisNodeName != "" {
+		node = c.thisNodeName
+	}
+	return checkpointPrefix + run + "-" + node
+}
+
+// handleManifest takes PUT /_checkpoint/manifest?run=<id>: the backup job's
+// manifest of the run, kept in this node's checkpoint of it, so that each
+// node's copy of its checkpoint (a volume snapshot, an upload) names the
+// whole run.
+func (c *checkpointer) handleManifest(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPut {
+		http.Error(w, "PUT only", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.authorized(r) {
+		http.Error(w, "a manifest needs the checkpoint token", http.StatusUnauthorized)
+		return
+	}
+	run, _, ok := parseRun(r.URL.Query().Get("run"))
+	if !ok {
+		http.Error(w, "a run id is a UTC time, as 20060102T150405Z", http.StatusBadRequest)
+		return
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, maxManifest+1))
+	if err != nil || len(body) > maxManifest || !json.Valid(body) {
+		http.Error(w, "a manifest is JSON, up to 1 MiB", http.StatusBadRequest)
+		return
+	}
+	dir := filepath.Join(c.dir, runCheckpointName(run))
+	if fi, err := os.Stat(filepath.Join(dir, store.CheckpointInfoFile)); err != nil || fi.IsDir() {
+		http.Error(w, "this node has no complete checkpoint of run "+run, http.StatusNotFound)
+		return
+	}
+	if err := store.WriteManifest(dir, body); err != nil {
+		log.Error("checkpoint", "manifest: "+err.Error())
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
 }
 
 // prune removes all but the newest keep checkpoints.
diff --git a/server/internal/cluster.go b/server/internal/cluster.go
index e32ce72..7f1a253 100644
--- a/server/internal/cluster.go
+++ b/server/internal/cluster.go
@@ -646,6 +646,9 @@ type Cluster struct {
 	// Set while this node, started with an empty store, copies its topics'
 	// messages from the other nodes.
 	rebuilding atomic.Bool
+	// Set while this node, started with -restored, reconciles its topics'
+	// messages with the other nodes (cluster_reconcile.go).
+	reconciling atomic.Bool
 	// Set once this node, shutting down, leaves the cluster.
 	leaving atomic.Bool
 	// Set once the cluster is shut down.
@@ -823,7 +826,7 @@ func (c *Cluster) takes(req *ClusterReq) bool {
 			}
 		}
 	case message.RELAY:
-		if c.rebuilding.Load() {
+		if c.catchingUp() {
 			return false
 		}
 		for _, r := range req.RelayMsg.RelayRequests {
@@ -1015,11 +1018,17 @@ func (c *Cluster) routeToTopic(msg lp.MessagePack, contract uint32, topic string
 	}
 }
 
+// catchingUp reports whether this node's messages are not all here yet: it
+// rebuilds an empty store, or reconciles a restored one.
+func (c *Cluster) catchingUp() bool {
+	return c.rebuilding.Load() || c.reconciling.Load()
+}
+
 // relayFromHolder forwards a relay on topic to the first node of the topic's
 // replica set that takes it: the owner, then its replicas. It reports false if
 // this node comes first, or none takes it, for the relay to be answered here.
 func (c *Cluster) relayFromHolder(msg *utp.Relay, contract uint32, topic string, conn *_Conn) bool {
-	rebuilding := c.rebuilding.Load()
+	rebuilding := c.catchingUp()
 	for _, holder := range c.getRing().GetN(topicRingKey(contract, topic), c.replicas) {
 		n := c.nodes[holder]
 		if n == nil {
@@ -1769,9 +1778,10 @@ func (c *Cluster) Replicate(req *ReplicateReq, unused *bool) error {
 			return err
 		}
 	}
-	if req.Handoff && c.rebuilding.Load() {
-		// The node's rebuild copies these messages with the rest. It does
-		// not copy session logs: their changes are applied below.
+	if req.Handoff && c.catchingUp() {
+		// The node's rebuild, or its reconciliation, copies these messages
+		// with the rest. It does not copy session logs: their changes are
+		// applied below.
 		req.Entries = nil
 	}
 	for _, e := range req.Entries {
@@ -2092,6 +2102,19 @@ func (c *Cluster) Start() {
 		c.rebuilding.Store(true)
 		go c.rebuild()
 	}
+	// Restored from a backup: the hints this node kept for the others are
+	// as old as its checkpoint, and the others are reconciled with instead.
+	if restoreRequested && c.replicas >= 2 && hasCapability(capReconcile) {
+		c.reconciling.Store(true)
+		for name, n := range c.nodes {
+			n.handoffMu.Lock()
+			c.dropHints(name)
+			n.handoffMu.Unlock()
+		}
+		go c.reconcileAll()
+	} else if restoreRequested {
+		markRestored(restoredPath)
+	}
 
 	if c.fo != nil {
 		go c.run()
diff --git a/server/internal/cluster_caps.go b/server/internal/cluster_caps.go
index 1dd1876..67bf367 100644
--- a/server/internal/cluster_caps.go
+++ b/server/internal/cluster_caps.go
@@ -75,7 +75,7 @@ const (
 	capRevocations = "revocations"
 )
 
-var allCapabilities = []string{capReplicate, capDeliver, capSessions, capResync, capService, capV2Keys, capRevocations}
+var allCapabilities = []string{capReplicate, capDeliver, capSessions, capResync, capService, capV2Keys, capRevocations, capReconcile}
 
 // ownCapabilities are what this node can do: all of them, unless the
 // UNITDB_CLUSTER_CAPS environment variable lists fewer ("none" for none), so
diff --git a/server/internal/cluster_health.go b/server/internal/cluster_health.go
index 134817d..925c5fd 100644
--- a/server/internal/cluster_health.go
+++ b/server/internal/cluster_health.go
@@ -36,6 +36,8 @@ func (c *Cluster) readiness() (bool, string) {
 		return false, "leaving the cluster"
 	case c.rebuilding.Load():
 		return false, "catching up: copying its topics from the other nodes"
+	case c.reconciling.Load():
+		return false, "catching up: reconciling its restored topics with the other nodes"
 	}
 	ring := c.getRingNodes()
 	configured := len(c.nodes) + 1
@@ -73,6 +75,7 @@ func (c *Cluster) writeMetrics(m *metricsWriter) {
 	m.one("unitdb_cluster_members", "gauge", "Nodes in the ring this node routes by.", float64(len(c.getRingNodes())))
 	m.one("unitdb_cluster_ring_version", "gauge", "The ring version last seen from the leader; 0 before any.", float64(c.clusterRing.Load()))
 	m.one("unitdb_cluster_rebuilding", "gauge", "Whether this node is copying its topics from the others.", b(c.rebuilding.Load()))
+	c.writeReconcileMetrics(m)
 	m.one("unitdb_cluster_leaving", "gauge", "Whether this node is leaving the cluster.", b(c.leaving.Load()))
 	if last := c.health.lastLeader.Load(); last != 0 {
 		m.one("unitdb_cluster_leader_age_seconds", "gauge", "Seconds since this node last heard from a leader, or was one.", time.Since(time.Unix(0, last)).Seconds())
diff --git a/server/internal/cluster_reconcile.go b/server/internal/cluster_reconcile.go
new file mode 100644
index 0000000..0d83bb0
--- /dev/null
+++ b/server/internal/cluster_reconcile.go
@@ -0,0 +1,425 @@
+package internal
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+	"sort"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/unit-io/unitdb/server/internal/pkg/log"
+	"github.com/unit-io/unitdb/server/internal/store"
+)
+
+// Reconciliation after a restore (docs/backup-restore.md): a
+// node started with -restored, at a checkpoint, settles each topic it holds
+// with the topic's other holders on the union of their messages, before it
+// takes clients. A message one took in the seconds between two nodes'
+// checkpoints of a run, or since this node's checkpoint while the others ran
+// on, survives if either has it.
+//
+// Stored messages carry no identity, so they are compared by content: each
+// side lists the topic's messages as digests,
+// SHA-256(contract, topic, expiry, payload), with how many times each
+// occurs (two identical publishes stay two), and each side gets what it
+// holds fewer of. Expired messages are skipped. What is sent is a message
+// and how many the sender holds of it; the receiver, under the topic's
+// lock, stores only as many as it lacks of that count. So two nodes both
+// restored, reconciling the same topic with each other at once, settle on
+// the larger count, not the sum. Nothing is deleted: clients can't delete
+// messages, and the security state merges by itself, as every node holds it
+// all.
+//
+// A topic's holders are those of the full ring, as for a rebuild; a holder
+// that isn't live is skipped: when it starts, with -restored, it reconciles
+// with this node. A peer is asked for the topics it holds that this node
+// should too, for a topic this node's checkpoint predates.
+
+// capReconcile: the RPCs of a reconciliation (ReconcileTopics, Digests,
+// Reconcile).
+const capReconcile = "reconcile"
+
+// reconcileBatch is the most distinct messages sent in one Reconcile.
+const reconcileBatch = 1000
+
+// restoreRequested is set by -restored (CheckStart): the cluster reconciles
+// when it starts.
+var restoreRequested bool
+
+// restoredPath is the db_path a -restored node started at: its
+// checkpoint.json becomes restored-from.json once reconciled.
+var restoredPath string
+
+// reconcileStats counts what reconciliations did on this node.
+var reconcileStats struct {
+	topics   atomic.Int64
+	sent     atomic.Int64
+	received atomic.Int64
+	failed   atomic.Int64
+	millis   atomic.Int64
+}
+
+// MessageDigest identifies a stored message by its content.
+type MessageDigest [sha256.Size]byte
+
+// DigestCount is how many of a topic's messages have one digest.
+type DigestCount struct {
+	Digest MessageDigest
+	Count  int
+}
+
+// ReconcileTopicsReq asks a node for the topics it holds that the node
+// asking should hold too.
+type ReconcileTopicsReq struct {
+	Node string
+}
+
+// ReconcileTopicsResp lists them.
+type ReconcileTopicsResp struct {
+	Topics []store.TopicRef
+}
+
+// DigestsReq asks a node for a topic's messages, as digests.
+type DigestsReq struct {
+	Node  string
+	Topic store.TopicRef
+}
+
+// DigestsResp lists them, with their counts.
+type DigestsResp struct {
+	Digests []DigestCount
+}
+
+// CountedEntry is a message and how many the sender holds of it.
+type CountedEntry struct {
+	Entry store.HistoryEntry
+	Count int
+}
+
+// ReconcileReq sends a node the messages of a topic it holds fewer of
+// (Put), and asks for those it holds more of (Want).
+type ReconcileReq struct {
+	Node  string
+	Topic store.TopicRef
+	Put   []CountedEntry
+	Want  []MessageDigest
+}
+
+// ReconcileResp holds the messages asked for.
+type ReconcileResp struct {
+	Entries []CountedEntry
+}
+
+// topicLocks serializes, per topic, a reconciliation's counting and storing
+// on this node.
+var topicLocks sync.Map // store.TopicRef -> *sync.Mutex
+
+func topicLock(t store.TopicRef) *sync.Mutex {
+	mu, _ := topicLocks.LoadOrStore(t, &sync.Mutex{})
+	return mu.(*sync.Mutex)
+}
+
+// messageDigest returns the digest of a message of topic t.
+func messageDigest(t store.TopicRef, e store.HistoryEntry) MessageDigest {
+	h := sha256.New()
+	var b [8]byte
+	binary.LittleEndian.PutUint32(b[:4], t.Contract)
+	h.Write(b[:4])
+	binary.LittleEndian.PutUint32(b[:4], uint32(len(t.Topic)))
+	h.Write(b[:4])
+	h.Write([]byte(t.Topic))
+	binary.LittleEndian.PutUint64(b[:], uint64(e.ExpiresAt))
+	h.Write(b[:])
+	if e.Known {
+		h.Write([]byte{1})
+	} else {
+		h.Write([]byte{0})
+	}
+	h.Write(e.Payload)
+	var d MessageDigest
+	h.Sum(d[:0])
+	return d
+}
+
+// byDigest groups a topic's messages by digest.
+func byDigest(t store.TopicRef, entries []store.HistoryEntry) map[MessageDigest][]store.HistoryEntry {
+	m := make(map[MessageDigest][]store.HistoryEntry, len(entries))
+	for _, e := range entries {
+		d := messageDigest(t, e)
+		m[d] = append(m[d], e)
+	}
+	return m
+}
+
+// nodeHoldsTopic reports whether node is one of t's holders on the full ring.
+func (c *Cluster) nodeHoldsTopic(node string, t store.TopicRef) bool {
+	for _, h := range c.getFullRing().GetN(topicRingKey(t.Contract, t.Topic), c.replicas) {
+		if h == node {
+			return true
+		}
+	}
+	return false
+}
+
+// ReconcileTopics returns the topics this node holds that req.Node holds
+// too, and drops the message hints kept for it, as RebuildTopics does.
+// Called by a node reconciling.
+func (c *Cluster) ReconcileTopics(req *ReconcileTopicsReq, resp *ReconcileTopicsResp) error {
+	if err := refuse(capReconcile); err != nil {
+		return err
+	}
+	// The hints of messages kept for the node are as stale as its
+	// checkpoint: it gets the messages from the reconciliation, and a hint
+	// handed to it after would store one again.
+	if n := c.nodes[req.Node]; n != nil {
+		n.handoffMu.Lock()
+		c.dropHints(req.Node)
+		n.handoffMu.Unlock()
+	}
+	for _, t := range store.Message.Topics() {
+		if c.nodeHoldsTopic(req.Node, t) {
+			resp.Topics = append(resp.Topics, t)
+		}
+	}
+	return nil
+}
+
+// Digests returns a topic's messages on this node, as digests with their
+// counts. Called by a node reconciling.
+func (c *Cluster) Digests(req *DigestsReq, resp *DigestsResp) error {
+	if err := refuse(capReconcile); err != nil {
+		return err
+	}
+	entries, err := store.Message.History(req.Topic.Contract, req.Topic.Topic)
+	if err != nil {
+		return err
+	}
+	for d, es := range byDigest(req.Topic, entries) {
+		resp.Digests = append(resp.Digests, DigestCount{Digest: d, Count: len(es)})
+	}
+	return nil
+}
+
+// Reconcile stores the messages a reconciling node sends, and returns those
+// it asks for. Called by a node reconciling.
+func (c *Cluster) Reconcile(req *ReconcileReq, resp *ReconcileResp) error {
+	if err := refuse(capReconcile); err != nil {
+		return err
+	}
+	stored, err := c.storeReconciled(req.Topic, req.Put)
+	reconcileStats.received.Add(int64(stored))
+	if err != nil {
+		return err
+	}
+	if len(req.Want) == 0 {
+		return nil
+	}
+	entries, err := store.Message.History(req.Topic.Contract, req.Topic.Topic)
+	if err != nil {
+		return err
+	}
+	have := byDigest(req.Topic, entries)
+	for _, d := range req.Want {
+		if es := have[d]; len(es) > 0 {
+			resp.Entries = append(resp.Entries, CountedEntry{Entry: es[0], Count: len(es)})
+		}
+	}
+	return nil
+}
+
+// storeReconciled stores, of each message of topic t a peer holds Count of,
+// as many as this node lacks, as a rebuild does: a message stored without
+// its expiry is kept for rebuildTTL. It counts under the topic's lock, so
+// what another reconciliation stored meanwhile counts.
+func (c *Cluster) storeReconciled(t store.TopicRef, entries []CountedEntry) (int, error) {
+	if len(entries) == 0 {
+		return 0, nil
+	}
+	mu := topicLock(t)
+	mu.Lock()
+	defer mu.Unlock()
+	local, err := store.Message.History(t.Contract, t.Topic)
+	if err != nil {
+		return 0, err
+	}
+	have := byDigest(t, local)
+	stored := 0
+	for _, ce := range entries {
+		e := ce.Entry
+		missing := ce.Count - len(have[messageDigest(t, e)])
+		expiresAt := e.ExpiresAt
+		if !e.Known {
+			expiresAt = time.Now().Add(c.rebuildTTL).Unix()
+		}
+		for i := 0; i < missing; i++ {
+			if err := store.Message.PutReplica(t.Contract, t.Topic, e.Payload, expiresAt); err != nil {
+				return stored, err
+			}
+			stored++
+		}
+	}
+	return stored, nil
+}
+
+// reconcileAll reconciles every topic this node holds, or a live peer
+// holds that it should too, with the topic's live holders, then lets the
+// node take clients. Run once, when a -restored node's cluster starts.
+func (c *Cluster) reconcileAll() {
+	defer c.reconciling.Store(false)
+	start := time.Now()
+	c.waitInRing(rebuildTimeout)
+
+	live := make(map[string]bool)
+	for _, n := range c.getRingNodes() {
+		live[n] = true
+	}
+	topics := make(map[store.TopicRef]bool)
+	for _, t := range store.Message.Topics() {
+		topics[t] = true
+	}
+	for name, n := range c.nodes {
+		if !live[name] || !n.supports(capReconcile) {
+			continue
+		}
+		var resp ReconcileTopicsResp
+		if err := n.callTimeout("Cluster.ReconcileTopics", &ReconcileTopicsReq{Node: c.thisNodeName}, &resp, rebuildTimeout); err != nil {
+			log.ErrLogger.Error().Err(err).Str("context", "cluster.reconcileAll").Msg("no topics from " + name)
+			reconcileStats.failed.Add(1)
+			continue
+		}
+		for _, t := range resp.Topics {
+			topics[t] = true
+		}
+	}
+
+	failed := 0
+	var peers []string
+	for name := range live {
+		if name != c.thisNodeName {
+			peers = append(peers, name)
+		}
+	}
+	sort.Strings(peers)
+	for t := range topics {
+		for _, h := range c.getFullRing().GetN(topicRingKey(t.Contract, t.Topic), c.replicas) {
+			n := c.nodes[h]
+			if n == nil || !live[h] || !n.supports(capReconcile) {
+				continue // this node, or one that reconciles when it starts
+			}
+			if err := c.reconcileTopic(n, t); err != nil {
+				failed++
+				reconcileStats.failed.Add(1)
+				log.ErrLogger.Error().Err(err).Str("context", "cluster.reconcileAll").Str("topic", t.Topic).Msg("topic not reconciled with " + h)
+			}
+		}
+		reconcileStats.topics.Add(1)
+	}
+	took := time.Since(start)
+	reconcileStats.millis.Store(took.Milliseconds())
+	log.ErrLogger.Info().Str("context", "cluster.reconcileAll").Strs("live", peers).Int("topics", len(topics)).Int("failed", failed).
+		Int64("sent", reconcileStats.sent.Load()).Int64("received", reconcileStats.received.Load()).Dur("took", took).Msg("reconciled after a restore")
+	if failed == 0 {
+		markRestored(restoredPath)
+	}
+}
+
+// reconcileTopic settles topic t with node n on the union of their
+// messages.
+func (c *Cluster) reconcileTopic(n *ClusterNode, t store.TopicRef) error {
+	var theirs DigestsResp
+	if err := n.callTimeout("Cluster.Digests", &DigestsReq{Node: c.thisNodeName, Topic: t}, &theirs, rebuildTimeout); err != nil {
+		return err
+	}
+	entries, err := store.Message.History(t.Contract, t.Topic)
+	if err != nil {
+		return err
+	}
+	mine := byDigest(t, entries)
+	theirCount := make(map[MessageDigest]int, len(theirs.Digests))
+	var want []MessageDigest
+	for _, dc := range theirs.Digests {
+		theirCount[dc.Digest] = dc.Count
+		if dc.Count > len(mine[dc.Digest]) {
+			want = append(want, dc.Digest)
+		}
+	}
+	var put []CountedEntry
+	for d, es := range mine {
+		if len(es) > theirCount[d] {
+			put = append(put, CountedEntry{Entry: es[0], Count: len(es)})
+		}
+	}
+	for first := true; first || len(put) > 0; first = false {
+		batch := put
+		if len(batch) > reconcileBatch {
+			batch = batch[:reconcileBatch]
+		}
+		put = put[len(batch):]
+		req := &ReconcileReq{Node: c.thisNodeName, Topic: t, Put: batch}
+		if first {
+			req.Want = want
+		}
+		if len(req.Put) == 0 && len(req.Want) == 0 {
+			return nil
+		}
+		var resp ReconcileResp
+		if err := n.callTimeout("Cluster.Reconcile", req, &resp, rebuildTimeout); err != nil {
+			return err
+		}
+		for _, ce := range batch {
+			reconcileStats.sent.Add(int64(ce.Count - theirCount[messageDigest(t, ce.Entry)]))
+		}
+		stored, err := c.storeReconciled(t, resp.Entries)
+		reconcileStats.received.Add(int64(stored))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// waitInRing waits until this node is in the ring and has heard from a
+// leader, for up to d: the live nodes it reconciles with are those of the
+// ring.
+func (c *Cluster) waitInRing(d time.Duration) {
+	deadline := time.Now().Add(d)
+	for time.Now().Before(deadline) {
+		if containsNode(c.getRingNodes(), c.thisNodeName) && (c.fo == nil || c.health.lastLeader.Load() != 0) {
+			return
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	log.ErrLogger.Warn().Str("context", "cluster.waitInRing").Msg("not in the ring yet: reconciling with the nodes it has")
+}
+
+// markRestored renames the checkpoint.json of a node's db_path once it is
+// reconciled: its next start is an ordinary one, not a restore.
+func markRestored(dbPath string) {
+	if dbPath == "" {
+		return
+	}
+	from := filepath.Join(dbPath, store.CheckpointInfoFile)
+	if err := os.Rename(from, filepath.Join(dbPath, restoredFromFile)); err != nil {
+		log.ErrLogger.Error().Err(err).Str("context", "cluster.markRestored").Msg("unable to mark the store restored")
+	}
+}
+
+// restoredFromFile is what a restored node's checkpoint.json becomes.
+const restoredFromFile = "restored-from.json"
+
+// writeReconcileMetrics writes what reconciliations did on this node.
+func (c *Cluster) writeReconcileMetrics(m *metricsWriter) {
+	b := 0.0
+	if c.reconciling.Load() {
+		b = 1
+	}
+	m.one("unitdb_cluster_reconciling", "gauge", "Whether this node, restored, is settling its topics with the others.", b)
+	m.one("unitdb_reconcile_topics_total", "counter", "Topics this node reconciled after a restore.", float64(reconcileStats.topics.Load()))
+	m.one("unitdb_reconcile_messages_sent_total", "counter", "Messages this node offered a peer that held fewer, in its reconciliations.", float64(reconcileStats.sent.Load()))
+	m.one("unitdb_reconcile_messages_received_total", "counter", "Messages this node stored from a peer in reconciliations.", float64(reconcileStats.received.Load()))
+	m.one("unitdb_reconcile_failures_total", "counter", "Topics or peers a reconciliation failed on.", float64(reconcileStats.failed.Load()))
+	m.one("unitdb_reconcile_duration_seconds", "gauge", "How long the last reconciliation took.", float64(reconcileStats.millis.Load())/1000)
+}
diff --git a/server/internal/cluster_tls.go b/server/internal/cluster_tls.go
index 773bb23..a9e09dd 100644
--- a/server/internal/cluster_tls.go
+++ b/server/internal/cluster_tls.go
@@ -325,3 +325,31 @@ func (p *peerRPC) Revocations(req *RevocationsReq, resp *RevocationsResp) error
 	}
 	return p.c.Revocations(req, resp)
 }
+
+func (p *peerRPC) StartedFrom(req *StartedFromReq, resp *StartedFromResp) error {
+	if err := p.is(req.Node); err != nil {
+		return err
+	}
+	return p.c.StartedFrom(req, resp)
+}
+
+func (p *peerRPC) ReconcileTopics(req *ReconcileTopicsReq, resp *ReconcileTopicsResp) error {
+	if err := p.is(req.Node); err != nil {
+		return err
+	}
+	return p.c.ReconcileTopics(req, resp)
+}
+
+func (p *peerRPC) Digests(req *DigestsReq, resp *DigestsResp) error {
+	if err := p.is(req.Node); err != nil {
+		return err
+	}
+	return p.c.Digests(req, resp)
+}
+
+func (p *peerRPC) Reconcile(req *ReconcileReq, resp *ReconcileResp) error {
+	if err := p.is(req.Node); err != nil {
+		return err
+	}
+	return p.c.Reconcile(req, resp)
+}
diff --git a/server/internal/cluster_tls_test.go b/server/internal/cluster_tls_test.go
index 6f98221..ac9cfe5 100644
--- a/server/internal/cluster_tls_test.go
+++ b/server/internal/cluster_tls_test.go
@@ -239,17 +239,21 @@ func TestPeerRPCSenders(t *testing.T) {
 	p := &peerRPC{c: nil, peer: "two"}
 	var b bool
 	calls := map[string]func() error{
-		"Ping":           func() error { return p.Ping(&ClusterPing{Leader: "three"}, &ClusterPong{}) },
-		"Vote":           func() error { return p.Vote(&ClusterVoteRequest{Node: "three"}, &ClusterVoteResponse{}) },
-		"Master":         func() error { return p.Master(&ClusterReq{Node: "three", Conn: &ClusterSess{}}, &b) },
-		"Deliver":        func() error { return p.Deliver(&DeliverReq{Node: "three"}, &b) },
-		"RebuildTopics":  func() error { return p.RebuildTopics(&RebuildReq{Node: "three"}, &RebuildTopicsResp{}) },
-		"RebuildHistory": func() error { return p.RebuildHistory(&RebuildHistoryReq{Node: "three"}, &RebuildHistoryResp{}) },
-		"FetchSession":   func() error { return p.FetchSession(&FetchSessionReq{Node: "three"}, &FetchSessionResp{}) },
-		"ForgetSession":  func() error { return p.ForgetSession(&ForgetSessionReq{Node: "three"}, &b) },
-		"Replicate":      func() error { return p.Replicate(&ReplicateReq{Node: "three"}, &b) },
-		"Resync":         func() error { return p.Resync(&ResyncReq{Node: "three"}, &b) },
-		"Revocations":    func() error { return p.Revocations(&RevocationsReq{Node: "three"}, &RevocationsResp{}) },
+		"Ping":            func() error { return p.Ping(&ClusterPing{Leader: "three"}, &ClusterPong{}) },
+		"Vote":            func() error { return p.Vote(&ClusterVoteRequest{Node: "three"}, &ClusterVoteResponse{}) },
+		"Master":          func() error { return p.Master(&ClusterReq{Node: "three", Conn: &ClusterSess{}}, &b) },
+		"Deliver":         func() error { return p.Deliver(&DeliverReq{Node: "three"}, &b) },
+		"RebuildTopics":   func() error { return p.RebuildTopics(&RebuildReq{Node: "three"}, &RebuildTopicsResp{}) },
+		"RebuildHistory":  func() error { return p.RebuildHistory(&RebuildHistoryReq{Node: "three"}, &RebuildHistoryResp{}) },
+		"FetchSession":    func() error { return p.FetchSession(&FetchSessionReq{Node: "three"}, &FetchSessionResp{}) },
+		"ForgetSession":   func() error { return p.ForgetSession(&ForgetSessionReq{Node: "three"}, &b) },
+		"Replicate":       func() error { return p.Replicate(&ReplicateReq{Node: "three"}, &b) },
+		"Resync":          func() error { return p.Resync(&ResyncReq{Node: "three"}, &b) },
+		"Revocations":     func() error { return p.Revocations(&RevocationsReq{Node: "three"}, &RevocationsResp{}) },
+		"StartedFrom":     func() error { return p.StartedFrom(&StartedFromReq{Node: "three"}, &StartedFromResp{}) },
+		"ReconcileTopics": func() error { return p.ReconcileTopics(&ReconcileTopicsReq{Node: "three"}, &ReconcileTopicsResp{}) },
+		"Digests":         func() error { return p.Digests(&DigestsReq{Node: "three"}, &DigestsResp{}) },
+		"Reconcile":       func() error { return p.Reconcile(&ReconcileReq{Node: "three"}, &ReconcileResp{}) },
 	}
 	for name, call := range calls {
 		if err := call(); err == nil || !strings.Contains(err.Error(), `names "three"`) {
diff --git a/server/internal/db/adapter.go b/server/internal/db/adapter.go
index 1bd26dd..e65621d 100644
--- a/server/internal/db/adapter.go
+++ b/server/internal/db/adapter.go
@@ -91,8 +91,10 @@ type Adapter interface {
 
 	// Checkpoint writes a copy of the store into dst, a directory that
 	// doesn't exist or is empty. The copy opens as the store was at one
-	// moment, as after a clean shutdown. Writes wait while it runs.
-	Checkpoint(dst string) error
+	// moment, as after a clean shutdown. Writes wait while it runs. Last, it
+	// writes info into the copy as CheckpointInfoFile, with the copy's
+	// stats and the engine's version, and returns what it wrote.
+	Checkpoint(dst string, info CheckpointInfo) (CheckpointInfo, error)
 
 	// Stats returns the size of the store.
 	Stats() Stats
@@ -101,12 +103,12 @@ type Adapter interface {
 // Stats is the size of a store.
 type Stats struct {
 	// Messages is the number of messages in the store.
-	Messages uint64
+	Messages uint64 `json:"messages"`
 	// DiskBytes is the size of the store's files.
-	DiskBytes int64
+	DiskBytes int64 `json:"disk_bytes"`
 	// MemEntries is the number of records in memory (sessions, logs and the
 	// like), and MemSize the configured size of that memory (mem_size); 0
 	// if not configured.
-	MemEntries int64
-	MemSize    int64
+	MemEntries int64 `json:"mem_entries"`
+	MemSize    int64 `json:"mem_size"`
 }
diff --git a/server/internal/db/checkpoint_info.go b/server/internal/db/checkpoint_info.go
new file mode 100644
index 0000000..92500bb
--- /dev/null
+++ b/server/internal/db/checkpoint_info.go
@@ -0,0 +1,105 @@
+package adapter
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// CheckpointInfoFile is the file in a checkpoint that describes it
+// (docs/backup-restore.md). A store's own directory never
+// holds one: a db_path that does is a checkpoint.
+const CheckpointInfoFile = "checkpoint.json"
+
+// CheckpointInfo describes a checkpoint: what a backup's manifest lists for
+// each node, and what a node started at the checkpoint tells its peers.
+type CheckpointInfo struct {
+	// Node is the cluster node the checkpoint is of; empty for a single
+	// server.
+	Node string `json:"node"`
+	// Run is the backup run the checkpoint is of (its id, the run's start
+	// as 20060102T150405Z); empty for a checkpoint taken on its own.
+	Run string `json:"run,omitempty"`
+	// Time is when the checkpoint was taken, in UTC.
+	Time time.Time `json:"time"`
+	// RingVersion is the version of the ring the cluster routed by; 0 for a
+	// single server.
+	RingVersion int `json:"ring_version"`
+	// Engine is the version of the unitdb module that wrote the store.
+	Engine string `json:"engine"`
+	// KeyIDs are the ids of the keyring's keys (never the keys): the
+	// checkpoint opens only with a keyring that holds them.
+	KeyIDs []int `json:"key_ids"`
+	// Stats is the size of the store the checkpoint copied.
+	Stats Stats `json:"stats"`
+}
+
+// ManifestFile is the file in a checkpoint of a backup run that lists the
+// run's checkpoints, on every node: the backup job writes it.
+const ManifestFile = "manifest.json"
+
+// ReadCheckpointInfo returns the description of the checkpoint in dir, or
+// nil if dir holds none: a store that isn't a checkpoint.
+func ReadCheckpointInfo(dir string) (*CheckpointInfo, error) {
+	b, err := os.ReadFile(filepath.Join(dir, CheckpointInfoFile))
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var info CheckpointInfo
+	if err := json.Unmarshal(b, &info); err != nil {
+		return nil, fmt.Errorf("%s: %v", filepath.Join(dir, CheckpointInfoFile), err)
+	}
+	return &info, nil
+}
+
+// WriteCheckpointInfo writes info into dir as CheckpointInfoFile, whole or
+// not at all, and fsyncs it and dir.
+func WriteCheckpointInfo(dir string, info CheckpointInfo) error {
+	b, err := json.MarshalIndent(info, "", "  ")
+	if err != nil {
+		return err
+	}
+	return writeSynced(dir, CheckpointInfoFile, append(b, '\n'))
+}
+
+// WriteManifest writes a backup run's manifest into dir, a checkpoint of
+// the run, as ManifestFile, whole or not at all.
+func WriteManifest(dir string, manifest []byte) error {
+	return writeSynced(dir, ManifestFile, manifest)
+}
+
+// writeSynced writes b into dir as name, through a temporary file renamed
+// into place, and fsyncs it and dir.
+func writeSynced(dir, name string, b []byte) error {
+	tmp := filepath.Join(dir, name+".tmp")
+	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+	if err != nil {
+		return err
+	}
+	if _, err := f.Write(b); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	if err := os.Rename(tmp, filepath.Join(dir, name)); err != nil {
+		return err
+	}
+	d, err := os.Open(dir)
+	if err != nil {
+		return err
+	}
+	defer d.Close()
+	return d.Sync()
+}
diff --git a/server/internal/db/unitdb/checkpoint.go b/server/internal/db/unitdb/checkpoint.go
index 8fc707b..26d82ce 100644
--- a/server/internal/db/unitdb/checkpoint.go
+++ b/server/internal/db/unitdb/checkpoint.go
@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"runtime/debug"
 	"time"
 
 	"github.com/unit-io/unitdb/memdb"
@@ -35,10 +36,11 @@ import (
 const settleTime = 1500 * time.Millisecond
 
 // Checkpoint writes a copy of the store into dst, which must not exist or be
-// empty.
-func (a *adapter) Checkpoint(dst string) error {
-	fail := func(what string, err error) error {
-		return fmt.Errorf("store checkpoint: %s: %v", what, err)
+// empty, and then info, with the copy's stats and the engine's version, as
+// dbapi.CheckpointInfoFile: a checkpoint without one is incomplete.
+func (a *adapter) Checkpoint(dst string, info dbapi.CheckpointInfo) (dbapi.CheckpointInfo, error) {
+	fail := func(what string, err error) (dbapi.CheckpointInfo, error) {
+		return info, fmt.Errorf("store checkpoint: %s: %v", what, err)
 	}
 	if a.db == nil {
 		return fail("the store", fmt.Errorf("is not open"))
@@ -62,6 +64,9 @@ func (a *adapter) Checkpoint(dst string) error {
 			return fail("sync", err)
 		}
 	}
+	// The DB as the copy has it: writes wait until the copy is done.
+	info.Stats = a.Stats()
+	info.Engine = engineVersion()
 	src := filepath.Join(a.path, defaultDatabase)
 	if err := copyTree(src, filepath.Join(dst, defaultDatabase)); err != nil {
 		return fail("copy the DB", err)
@@ -76,6 +81,7 @@ func (a *adapter) Checkpoint(dst string) error {
 	if err != nil {
 		return fail("open the copy's memdb", err)
 	}
+	copied := 0
 	for _, key := range a.Keys() {
 		b, err := a.mem.Get(key)
 		if err != nil {
@@ -85,14 +91,44 @@ func (a *adapter) Checkpoint(dst string) error {
 			mem.Close()
 			return fail("copy memdb", err)
 		}
+		copied++
 	}
+	// The records the copy holds: memdb's Size counts a key put again as
+	// another record (unitdb v0.6.0), so a live store's overstates them,
+	// and a restore would be measured against too many.
+	info.Stats.MemEntries = int64(copied)
 	if err := mem.Close(); err != nil {
 		return fail("close the copy's memdb", err)
 	}
 	if err := syncTree(dst); err != nil {
 		return fail("fsync the copy", err)
 	}
-	return nil
+	if err := dbapi.WriteCheckpointInfo(dst, info); err != nil {
+		return fail("describe the copy", err)
+	}
+	return info, nil
+}
+
+// unitdbModule is the module of the engine the adapter stores with.
+const unitdbModule = "github.com/unit-io/unitdb"
+
+// engineVersion returns the version of the unitdb module this binary was
+// built with, as go.mod requires it, or "unknown".
+func engineVersion() string {
+	bi, ok := debug.ReadBuildInfo()
+	if !ok {
+		return "unknown"
+	}
+	for _, m := range bi.Deps {
+		if m.Path != unitdbModule {
+			continue
+		}
+		if m.Replace != nil {
+			return m.Version + " => " + m.Replace.Path + " " + m.Replace.Version
+		}
+		return m.Version
+	}
+	return "unknown"
 }
 
 // Stats returns the size of the store.
diff --git a/server/internal/db/unitdb/checkpoint_test.go b/server/internal/db/unitdb/checkpoint_test.go
index 7e24cea..870320c 100644
--- a/server/internal/db/unitdb/checkpoint_test.go
+++ b/server/internal/db/unitdb/checkpoint_test.go
@@ -9,6 +9,9 @@ import (
 	"sync"
 	"sync/atomic"
 	"testing"
+	"time"
+
+	dbapi "github.com/unit-io/unitdb/server/internal/db"
 )
 
 const testConfig = `{"mem_size": 16777216}`
@@ -61,7 +64,8 @@ func TestCheckpoint(t *testing.T) {
 		}
 	}()
 	dst := filepath.Join(t.TempDir(), "checkpoint")
-	err := a.Checkpoint(dst)
+	taken := time.Date(2026, 10, 3, 21, 30, 0, 0, time.UTC)
+	wrote, err := a.Checkpoint(dst, dbapi.CheckpointInfo{Node: "unitdb-1", Time: taken, RingVersion: 2, KeyIDs: []int{1, 2}})
 	close(stop)
 	wg.Wait()
 	if err != nil {
@@ -100,8 +104,36 @@ func TestCheckpoint(t *testing.T) {
 		t.Errorf("stats of the copy: %+v", s)
 	}
 
+	// The copy describes itself: what it was given, its stats and the
+	// engine's version.
+	info, err := dbapi.ReadCheckpointInfo(dst)
+	if err != nil || info == nil {
+		t.Fatalf("checkpoint.json: %+v, %v", info, err)
+	}
+	if info.Node != "unitdb-1" || !info.Time.Equal(taken) || info.RingVersion != 2 || fmt.Sprint(info.KeyIDs) != "[1 2]" {
+		t.Errorf("checkpoint.json says %+v", info)
+	}
+	if info.Engine == "" {
+		t.Error("checkpoint.json has no engine version")
+	}
+	if info.Stats != wrote.Stats || info.Engine != wrote.Engine {
+		t.Errorf("checkpoint.json says %+v, the checkpoint returned %+v", info, wrote)
+	}
+	// Its records are the copy's keys, though one was written twice:
+	// memdb's own count takes a key put again for another record.
+	if got := int64(len(b.Keys())); info.Stats.MemEntries != got {
+		t.Errorf("checkpoint.json counts %d records; the copy holds %d", info.Stats.MemEntries, got)
+	}
+	if got := b.Stats().Messages; info.Stats.Messages != got || info.Stats.MemEntries == 0 {
+		t.Errorf("checkpoint.json counts %+v; the copy holds %d messages", info.Stats, got)
+	}
+	// The store it was taken from isn't a checkpoint.
+	if info, err := dbapi.ReadCheckpointInfo(a.path); info != nil || err != nil {
+		t.Errorf("the live store reads as a checkpoint: %+v, %v", info, err)
+	}
+
 	// A directory that isn't empty is refused.
-	if err := a.Checkpoint(dst); err == nil || !strings.HasPrefix(err.Error(), "store checkpoint: ") {
+	if _, err := a.Checkpoint(dst, dbapi.CheckpointInfo{}); err == nil || !strings.HasPrefix(err.Error(), "store checkpoint: ") {
 		t.Errorf("a checkpoint into a directory that isn't empty: %v", err)
 	}
 	if _, err := os.Stat(filepath.Join(dst, defaultDatabase, "unitdb.lock")); err != nil {
diff --git a/server/internal/health.go b/server/internal/health.go
index 381ed40..c726d79 100644
--- a/server/internal/health.go
+++ b/server/internal/health.go
@@ -178,6 +178,9 @@ func (h *healthMonitor) handler() http.Handler {
 	})
 	if h.checkpoints != nil {
 		mux.HandleFunc("/_checkpoint", h.checkpoints.handle)
+		mux.HandleFunc("/_checkpoint/manifest", h.checkpoints.handleManifest)
+		mux.HandleFunc("/_checkpoint/upload", h.checkpoints.handleUpload)
+		mux.HandleFunc("/_checkpoint/canary", h.checkpoints.handleCanary)
 	}
 	mux.HandleFunc("/_metrics", func(w http.ResponseWriter, r *http.Request) {
 		if h.metrics == nil {
diff --git a/server/internal/metrics.go b/server/internal/metrics.go
index b043d41..7178a32 100644
--- a/server/internal/metrics.go
+++ b/server/internal/metrics.go
@@ -142,6 +142,7 @@ func (s *_Service) writeMetrics(w io.Writer) {
 	if s.health != nil && s.health.checkpoints != nil {
 		s.health.checkpoints.writeMetrics(m)
 	}
+	writeOffsiteMetrics(m)
 
 	if c := Globals.Cluster; c != nil {
 		c.writeMetrics(m)
diff --git a/server/internal/offsite.go b/server/internal/offsite.go
new file mode 100644
index 0000000..fa66764
--- /dev/null
+++ b/server/internal/offsite.go
@@ -0,0 +1,411 @@
+package internal
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"filippo.io/age"
+
+	"github.com/unit-io/unitdb/server/internal/pkg/log"
+	"github.com/unit-io/unitdb/server/internal/backup"
+	"github.com/unit-io/unitdb/server/internal/store"
+)
+
+// Copies off the cluster (docs/backup-restore.md). A node
+// uploads what only it has, with an identity that may only put objects
+// (internal/backup):
+//
+//   - its checkpoint of a backup run, when the backup job asks
+//     (POST /_checkpoint/upload?run=<id>): archived, compressed and
+//     encrypted as it streams, with the run's manifest;
+//   - the security journal: each change of the security state this node
+//     makes (a revoked client id or topic key, a contract's not-before
+//     time), as a JSON line, kept on its volume before the change takes
+//     effect and uploaded in a batch every 10 s. A restore from a run replays the journal since
+//     (-restored -journal <dir>), so that nothing revoked after the run comes
+//     back.
+//
+// Nodes never read the bucket: the journal to replay is downloaded by an
+// operator, with a reader's identity (the backup command's fetch-journal).
+
+// journalEvery is how often the journal is uploaded: 10 s, or
+// SECURITY_JOURNAL_EVERY (a duration), for tests.
+var journalEvery = func() time.Duration {
+	if d, err := time.ParseDuration(os.Getenv("SECURITY_JOURNAL_EVERY")); err == nil && d > 0 {
+		return d
+	}
+	return 10 * time.Second
+}()
+
+// offsite is this node's link to the bucket; nil fields when it has none.
+var offsite struct {
+	store     *backup.Store
+	recipient age.Recipient
+	node      string
+	uploading sync.Mutex // one checkpoint upload at a time
+
+	lastUpload     atomic.Int64 // unix seconds of the last checkpoint upload
+	uploadFailures atomic.Int64
+	uploadMillis   atomic.Int64
+}
+
+// journal is this node's security journal, or nil.
+var journal atomic.Pointer[securityJournal]
+
+// InitOffsite reads where backups go (BACKUP_S3_*), and starts the security
+// journal when SECURITY_JOURNAL_DIR is set. Run it after ClusterInit, before
+// the service starts: a change made before it isn't journalled.
+func InitOffsite() error {
+	node := "server"
+	if c := Globals.Cluster; c != nil && c.thisNodeName != "" {
+		node = c.thisNodeName
+	}
+	offsite.node = node
+	cfg, err := backup.ConfigFromEnv()
+	if err != nil {
+		return err
+	}
+	if cfg != nil {
+		if offsite.store, err = backup.NewStore(context.Background(), cfg); err != nil {
+			return fmt.Errorf("backup: %v", err)
+		}
+		if cfg.Recipient == "" {
+			return fmt.Errorf("backup: BACKUP_AGE_RECIPIENT, the backup key's public half, is needed with BACKUP_S3_BUCKET")
+		}
+		if offsite.recipient, err = backup.ParseRecipient(cfg.Recipient); err != nil {
+			return err
+		}
+	}
+	if dir := os.Getenv("SECURITY_JOURNAL_DIR"); dir != "" {
+		var up journalUploader
+		if offsite.store != nil {
+			up = offsite.store
+		}
+		j, err := newSecurityJournal(dir, node, up)
+		if err != nil {
+			return err
+		}
+		if offsite.store == nil {
+			log.ErrLogger.Warn().Str("context", "InitOffsite").Msg("SECURITY_JOURNAL_DIR is set, but no bucket: the journal is kept on this node only")
+		}
+		journal.Store(j)
+		go j.run()
+	}
+	return nil
+}
+
+// handleUpload takes POST /_checkpoint/upload?run=<id>: this node's
+// checkpoint of the run goes to the bucket, with the run's manifest.
+func (c *checkpointer) handleUpload(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "POST only", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.authorized(r) {
+		http.Error(w, "an upload needs the checkpoint token", http.StatusUnauthorized)
+		return
+	}
+	run, runAt, ok := parseRun(r.URL.Query().Get("run"))
+	if !ok {
+		http.Error(w, "a run id is a UTC time, as 20060102T150405Z", http.StatusBadRequest)
+		return
+	}
+	if offsite.store == nil {
+		http.Error(w, "this node has no bucket to upload to (BACKUP_S3_BUCKET)", http.StatusConflict)
+		return
+	}
+	dir := filepath.Join(c.dir, runCheckpointName(run))
+	if _, err := os.Stat(filepath.Join(dir, store.CheckpointInfoFile)); err != nil {
+		http.Error(w, "this node has no complete checkpoint of run "+run, http.StatusNotFound)
+		return
+	}
+	if !offsite.uploading.TryLock() {
+		http.Error(w, "an upload is running", http.StatusConflict)
+		return
+	}
+	defer offsite.uploading.Unlock()
+
+	start := time.Now()
+	cfg := offsite.store.Config
+	key := cfg.CheckpointKey(run, offsite.node)
+	ret := backup.RunRetention(runAt)
+	err := uploadCheckpoint(r.Context(), dir, key, ret)
+	if err == nil {
+		if b, rerr := os.ReadFile(filepath.Join(dir, store.ManifestFile)); rerr == nil {
+			err = offsite.store.Put(r.Context(), cfg.ManifestKey(run), b, ret)
+		}
+	}
+	took := time.Since(start)
+	offsite.uploadMillis.Store(took.Milliseconds())
+	if err != nil {
+		offsite.uploadFailures.Add(1)
+		log.ErrLogger.Error().Err(err).Str("context", "checkpoint.upload").Str("run", run).Msg("upload failed")
+		http.Error(w, "upload: "+err.Error(), http.StatusBadGateway)
+		return
+	}
+	offsite.lastUpload.Store(time.Now().Unix())
+	log.ErrLogger.Info().Str("context", "checkpoint.upload").Str("key", key).Dur("took", took).Msg("uploaded")
+	w.Header().Set("content-type", "application/json")
+	json.NewEncoder(w).Encode(map[string]interface{}{"key": key, "tier": ret.Tier, "retain_until": ret.Until, "seconds": took.Seconds()})
+}
+
+// uploadCheckpoint streams dir to key: archived, compressed and encrypted
+// as the upload takes it, so nothing but the checkpoint is written here.
+func uploadCheckpoint(ctx context.Context, dir, key string, ret backup.Retention) error {
+	pr, pw := io.Pipe()
+	go func() {
+		pw.CloseWithError(backup.WriteArchive(pw, dir, offsite.recipient))
+	}()
+	err := offsite.store.Upload(ctx, key, pr, ret)
+	pr.CloseWithError(err) // stops the archive if the upload failed
+	return err
+}
+
+// journalEntry is one line of the security journal. Replayed, each merges
+// into the state as the change did: in any order, and as often.
+type journalEntry struct {
+	Time time.Time `json:"time"`
+	Node string    `json:"node"`
+	// Security is a change of the security state (revocations.apply).
+	Security map[uint32]*ContractState `json:"security,omitempty"`
+}
+
+const (
+	journalPending = "pending.jsonl"
+	journalBatch   = "batch-"
+)
+
+// securityJournal keeps the journal's lines on this node until they are
+// uploaded: pending.jsonl takes them as they come, and becomes a batch
+// file to upload every journalEvery. A batch is removed once in the bucket,
+// so a restart uploads what was left.
+type securityJournal struct {
+	dir   string
+	node  string
+	store journalUploader // nil: kept on this node only
+
+	mu sync.Mutex // appends to pending.jsonl, and its rotation
+
+	oldest       atomic.Int64 // unix nanos of the oldest line not uploaded; 0 if none
+	lastUpload   atomic.Int64 // unix seconds
+	failures     atomic.Int64
+	recordErrors atomic.Int64
+}
+
+// journalUploader is where journal batches go: the bucket.
+type journalUploader interface {
+	Put(ctx context.Context, key string, b []byte, ret backup.Retention) error
+	JournalKey(node string, t time.Time) string
+}
+
+func newSecurityJournal(dir, node string, st journalUploader) (*securityJournal, error) {
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return nil, fmt.Errorf("security journal: %v", err)
+	}
+	j := &securityJournal{dir: dir, node: node, store: st}
+	j.oldest.Store(j.oldestLeft())
+	return j, nil
+}
+
+// record appends e to the journal, and fsyncs it, before the change it
+// records takes effect for clients.
+func (j *securityJournal) record(e journalEntry) {
+	e.Time, e.Node = time.Now().UTC(), j.node
+	b, err := json.Marshal(e)
+	if err == nil {
+		j.mu.Lock()
+		err = appendSynced(filepath.Join(j.dir, journalPending), append(b, '\n'))
+		if err == nil {
+			j.oldest.CompareAndSwap(0, e.Time.UnixNano())
+		}
+		j.mu.Unlock()
+	}
+	if err != nil {
+		j.recordErrors.Add(1)
+		log.ErrLogger.Error().Err(err).Str("context", "securityJournal.record").Msg("a security change isn't in the journal")
+	}
+}
+
+func appendSynced(path string, b []byte) error {
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	if err != nil {
+		return err
+	}
+	if _, err := f.Write(b); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		return err
+	}
+	return f.Close()
+}
+
+func (j *securityJournal) run() {
+	for {
+		time.Sleep(journalEvery)
+		j.flush()
+	}
+}
+
+// flush turns the pending lines into a batch, and uploads the batches left,
+// oldest first, until one fails.
+func (j *securityJournal) flush() {
+	j.mu.Lock()
+	pending := filepath.Join(j.dir, journalPending)
+	if fi, err := os.Stat(pending); err == nil && fi.Size() > 0 {
+		batch := filepath.Join(j.dir, fmt.Sprintf("%s%020d.jsonl", journalBatch, time.Now().UnixNano()))
+		if err := os.Rename(pending, batch); err != nil {
+			log.ErrLogger.Error().Err(err).Str("context", "securityJournal.flush").Msg("unable to start a batch")
+		}
+		syncDir(j.dir)
+	}
+	j.mu.Unlock()
+	if j.store == nil {
+		return
+	}
+	for _, name := range j.batches() {
+		path := filepath.Join(j.dir, name)
+		b, err := os.ReadFile(path)
+		if err != nil {
+			j.failures.Add(1)
+			break
+		}
+		now := time.Now()
+		if err := j.store.Put(context.Background(), j.store.JournalKey(j.node, now), b, backup.JournalRetention(now)); err != nil {
+			j.failures.Add(1)
+			log.ErrLogger.Error().Err(err).Str("context", "securityJournal.flush").Msg("journal upload failed: kept here, tried again")
+			break
+		}
+		os.Remove(path)
+		j.lastUpload.Store(now.Unix())
+	}
+	j.mu.Lock()
+	j.oldest.Store(j.oldestLeft())
+	j.mu.Unlock()
+}
+
+// batches returns the batch files left, oldest first.
+func (j *securityJournal) batches() []string {
+	entries, _ := os.ReadDir(j.dir)
+	var names []string
+	for _, e := range entries {
+		if strings.HasPrefix(e.Name(), journalBatch) && strings.HasSuffix(e.Name(), ".jsonl") {
+			names = append(names, e.Name())
+		}
+	}
+	sort.Strings(names)
+	return names
+}
+
+// oldestLeft returns when the oldest line not uploaded was written, in unix
+// nanos, or 0.
+func (j *securityJournal) oldestLeft() int64 {
+	files := j.batches()
+	files = append(files, journalPending)
+	for _, name := range files {
+		f, err := os.Open(filepath.Join(j.dir, name))
+		if err != nil {
+			continue
+		}
+		line, _ := bufio.NewReader(f).ReadBytes('\n')
+		f.Close()
+		var e journalEntry
+		if json.Unmarshal(line, &e) == nil && !e.Time.IsZero() {
+			return e.Time.UnixNano()
+		}
+	}
+	return 0
+}
+
+func syncDir(dir string) {
+	if d, err := os.Open(dir); err == nil {
+		d.Sync()
+		d.Close()
+	}
+}
+
+// journalSecurity records a change of the security state this node makes.
+func journalSecurity(changes map[uint32]*ContractState) {
+	if j := journal.Load(); j != nil {
+		j.record(journalEntry{Security: changes})
+	}
+}
+
+// ReplayJournal merges the journal under dir (the backup command's
+// fetch-journal) into this node's security state: every change since the
+// run a restore is from. Lines merge as the changes did, so lines
+// older than the checkpoint, or replayed twice, change nothing. Run it
+// after the service is made, before the cluster starts.
+func ReplayJournal(dir string) error {
+	var entries []journalEntry
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil || info.IsDir() || !strings.HasSuffix(path, ".jsonl") {
+			return err
+		}
+		f, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		sc := bufio.NewScanner(f)
+		sc.Buffer(make([]byte, 64*1024), 16<<20)
+		for sc.Scan() {
+			var e journalEntry
+			if err := json.Unmarshal(sc.Bytes(), &e); err != nil {
+				return fmt.Errorf("%s: %v", path, err)
+			}
+			entries = append(entries, e)
+		}
+		return sc.Err()
+	})
+	if err != nil {
+		return fmt.Errorf("journal replay: %v", err)
+	}
+	sort.Slice(entries, func(i, j int) bool { return entries[i].Time.Before(entries[j].Time) })
+	security := 0
+	for _, e := range entries {
+		if e.Security == nil {
+			continue
+		}
+		changed, err := Globals.Service.revocations.replay(e.Security)
+		if err != nil {
+			return fmt.Errorf("journal replay: %v", err)
+		}
+		if len(changed) > 0 {
+			security++
+		}
+	}
+	log.ErrLogger.Info().Str("context", "ReplayJournal").Int("lines", len(entries)).Int("security_changes", security).Msg("replayed the security journal")
+	return nil
+}
+
+// writeOffsiteMetrics writes the uploads' and the journal's metrics.
+func writeOffsiteMetrics(m *metricsWriter) {
+	if offsite.store != nil {
+		m.one("unitdb_backup_upload_last_success_timestamp_seconds", "gauge", "When this node last uploaded its checkpoint of a run, in unix seconds; 0 before any.", float64(offsite.lastUpload.Load()))
+		m.one("unitdb_backup_upload_failures_total", "counter", "Checkpoint uploads that failed.", float64(offsite.uploadFailures.Load()))
+		m.one("unitdb_backup_upload_duration_seconds", "gauge", "How long the last checkpoint upload took.", float64(offsite.uploadMillis.Load())/1000)
+	}
+	if j := journal.Load(); j != nil {
+		lag := 0.0
+		if o := j.oldest.Load(); o != 0 {
+			lag = time.Since(time.Unix(0, o)).Seconds()
+		}
+		m.one("unitdb_security_journal_lag_seconds", "gauge", "Age of the oldest security change not in the bucket yet; 0 if none.", lag)
+		m.one("unitdb_security_journal_upload_failures_total", "counter", "Journal uploads that failed (kept, and tried again).", float64(j.failures.Load()))
+		m.one("unitdb_security_journal_record_errors_total", "counter", "Security changes that couldn't be written to the journal.", float64(j.recordErrors.Load()))
+	}
+}
diff --git a/server/internal/offsite_test.go b/server/internal/offsite_test.go
new file mode 100644
index 0000000..c5f6819
--- /dev/null
+++ b/server/internal/offsite_test.go
@@ -0,0 +1,98 @@
+package internal
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/unit-io/unitdb/server/internal/backup"
+)
+
+type fakeBucket struct {
+	mu      sync.Mutex
+	objects map[string]string
+	down    bool
+}
+
+func (f *fakeBucket) Put(_ context.Context, key string, b []byte, _ backup.Retention) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.down {
+		return errors.New("bucket unreachable")
+	}
+	f.objects[key] = string(b)
+	return nil
+}
+
+func (f *fakeBucket) JournalKey(node string, t time.Time) string {
+	return "c/journal/" + node + "-" + t.Format(time.RFC3339Nano)
+}
+
+// TestSecurityJournalSurvivesRestart checks that a security change waits on
+// the node until it is uploaded: the bucket is down, the node restarts, and
+// the next upload after takes it; the lag says how old it is meanwhile.
+func TestSecurityJournalSurvivesRestart(t *testing.T) {
+	dir := t.TempDir()
+	bucket := &fakeBucket{objects: map[string]string{}, down: true}
+	j, err := newSecurityJournal(dir, "n1", bucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	j.record(journalEntry{Security: map[uint32]*ContractState{7: {Revoked: map[uint64]int64{42: 0}}}})
+	j.flush() // the bucket is down: kept
+	if len(bucket.objects) != 0 || j.oldest.Load() == 0 || j.failures.Load() != 1 {
+		t.Fatalf("with the bucket down: %d objects, oldest %d, failures %d", len(bucket.objects), j.oldest.Load(), j.failures.Load())
+	}
+	j.record(journalEntry{Security: map[uint32]*ContractState{7: {NotBefore: 1700000000}}})
+
+	// The node restarts.
+	bucket.down = false
+	j2, err := newSecurityJournal(dir, "n1", bucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if j2.oldest.Load() == 0 {
+		t.Error("after a restart, the journal doesn't know it has lines to upload")
+	}
+	j2.flush()
+	var all string
+	for _, v := range bucket.objects {
+		all += v
+	}
+	if !strings.Contains(all, `"42":0`) || !strings.Contains(all, `"not_before":1700000000`) {
+		t.Errorf("uploaded %q", all)
+	}
+	if j2.oldest.Load() != 0 {
+		t.Error("lines left after every batch was uploaded")
+	}
+	left, _ := filepath.Glob(filepath.Join(dir, "*.jsonl"))
+	if len(left) != 0 {
+		t.Errorf("files left: %v", left)
+	}
+}
+
+func TestReplayJournalRejectsGarbage(t *testing.T) {
+	dir := t.TempDir()
+	os.WriteFile(filepath.Join(dir, "a.jsonl"), []byte("not json\n"), 0600)
+	if err := ReplayJournal(dir); err == nil {
+		t.Error("a journal that isn't JSON replayed")
+	}
+}
+
+// TestParseRun checks that only a run id makes a run, and that the run used
+// in paths is formatted anew from its time.
+func TestParseRun(t *testing.T) {
+	if run, at, ok := parseRun("20261004T213000Z"); !ok || run != "20261004T213000Z" || at.Hour() != 21 {
+		t.Errorf("a run id: %q %v %v", run, at, ok)
+	}
+	for _, bad := range []string{"", "../../etc", "20261004T213000Z/..", "20261304T213000Z", "2026-10-04"} {
+		if _, _, ok := parseRun(bad); ok {
+			t.Errorf("%q taken for a run id", bad)
+		}
+	}
+}
diff --git a/server/internal/restore.go b/server/internal/restore.go
new file mode 100644
index 0000000..068923d
--- /dev/null
+++ b/server/internal/restore.go
@@ -0,0 +1,137 @@
+package internal
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/unit-io/unitdb/server/internal/pkg/log"
+	"github.com/unit-io/unitdb/server/internal/store"
+)
+
+// A node started at a checkpoint (a db_path holding checkpoint.json) inside
+// a cluster that runs never gets what was written since the checkpoint,
+// unless it reconciles: it doesn't catch up, and as a topic's owner it
+// answers relays without it (docs/backup-restore.md, gap 1). So such a node
+// refuses to start, and says to start empty instead, which rebuilds it from
+// the others, or with -restored, which reconciles it with them
+// (cluster_reconcile.go) before it takes clients.
+//
+// A whole-cluster restore starts every node at a checkpoint of the same
+// backup run, one after another, so the nodes tell each other the
+// checkpoint they started at (Cluster.StartedFrom): a node at a checkpoint
+// starts if every peer that answers started at one of the same run, even
+// without -restored. With it, the nodes also settle on the union of what
+// their checkpoints hold (the seconds between two nodes' checkpoints). A
+// single server starts at a checkpoint as before: that's how one is
+// restored.
+
+// sameRun is how far apart two nodes' checkpoints taken without a run id
+// may be and still be of one backup run: a run checkpoints its nodes one
+// after another, in minutes. Checkpoints with a run id compare it instead.
+const sameRun = time.Hour
+
+// startedFrom is the checkpoint this node's store was started at; nil if it
+// wasn't. Set before the cluster starts, read-only after.
+var startedFrom *store.CheckpointInfo
+
+// StartedFromReq asks a node which checkpoint its store was started at.
+type StartedFromReq struct {
+	// Node is the name of the node asking.
+	Node string
+}
+
+// StartedFromResp is a node's answer: the checkpoint its store was started
+// at, nil if none.
+type StartedFromResp struct {
+	Checkpoint *store.CheckpointInfo
+}
+
+// StartedFrom answers which checkpoint this node's store was started at. A
+// node built before it answers that it lacks the method, which a node at a
+// checkpoint takes as "a running cluster".
+func (c *Cluster) StartedFrom(req *StartedFromReq, resp *StartedFromResp) error {
+	resp.Checkpoint = startedFrom
+	return nil
+}
+
+// CheckStart refuses a start that would leave this node's data silently
+// behind the cluster's: at a checkpoint, while a peer that wasn't restored
+// from the same backup run answers. It reads the store's checkpoint.json,
+// if any, so it runs before the store opens, and after ClusterInit.
+// restored is the -restored flag: the node starts at its checkpoint
+// whatever its peers run, and reconciles with them.
+func CheckStart(dbPath string, restored bool) error {
+	const startEmpty = "start this node with an empty store instead (a new, empty claim): it copies its topics from the other nodes, and takes no clients until it has (docs/backup-restore.md, case A); or start it with -restored, which reconciles its checkpoint with them"
+	info, err := store.ReadCheckpointInfo(dbPath)
+	if err != nil {
+		return fmt.Errorf("db_path %s looks like a checkpoint, but its %s is unreadable: %v", dbPath, store.CheckpointInfoFile, err)
+	}
+	if restored && info == nil {
+		return fmt.Errorf("-restored: db_path %s isn't a checkpoint (it has no %s): -restored is for a node started at its checkpoint of a backup run", dbPath, store.CheckpointInfoFile)
+	}
+	if info == nil {
+		return nil
+	}
+	startedFrom = info
+	if restored {
+		restoreRequested, restoredPath = true, dbPath
+		log.ErrLogger.Info().Str("context", "CheckStart").Time("checkpoint", info.Time).Str("run", info.Run).Msg("restored: reconciling with the other nodes before taking clients")
+		return nil
+	}
+	c := Globals.Cluster
+	if c == nil {
+		log.ErrLogger.Info().Str("context", "CheckStart").Time("checkpoint", info.Time).Msg("a single server, started at a checkpoint")
+		return nil
+	}
+	for _, n := range c.nodes {
+		peer, answered, err := n.startedFrom()
+		if !answered {
+			continue
+		}
+		at := info.Time.Format(time.RFC3339)
+		switch {
+		case err != nil:
+			return fmt.Errorf("db_path %s is a checkpoint of %s, and node %s answers but not which checkpoint it started at (%v), so it's taken to run with its own data: started here, this node would never get what was written since; %s", dbPath, at, n.name, err, startEmpty)
+		case peer == nil:
+			return fmt.Errorf("db_path %s is a checkpoint of %s, and node %s runs with its own data: started here, this node would never get what was written since; %s", dbPath, at, n.name, startEmpty)
+		case !sameBackupRun(peer, info):
+			return fmt.Errorf("db_path %s is a checkpoint of %s, and node %s was started at one of %s, not the same backup run: start every node of a whole-cluster restore at one run's checkpoints, or, to restore one node, %s", dbPath, at, n.name, peer.Time.Format(time.RFC3339), startEmpty)
+		}
+	}
+	log.ErrLogger.Info().Str("context", "CheckStart").Time("checkpoint", info.Time).Msg("started at a checkpoint: no peer answers that wasn't restored from the same run")
+	return nil
+}
+
+// startedFrom asks the node which checkpoint it was started at. answered is
+// false if it can't be reached: a node that is down doesn't stop a start.
+func (n *ClusterNode) startedFrom() (cp *store.CheckpointInfo, answered bool, err error) {
+	endpoint, _, err := n.dial()
+	if err != nil {
+		return nil, false, nil
+	}
+	defer endpoint.Close()
+	var resp StartedFromResp
+	call := endpoint.Go("Cluster.StartedFrom", &StartedFromReq{Node: Globals.Cluster.thisNodeName}, &resp, nil)
+	select {
+	case <-call.Done:
+		return resp.Checkpoint, true, call.Error
+	case <-time.After(2 * time.Second):
+		return nil, true, fmt.Errorf("no answer in 2s")
+	}
+}
+
+// sameBackupRun reports whether two checkpoints are of one backup run: the
+// same run id, or, for checkpoints taken without one, within sameRun.
+func sameBackupRun(a, b *store.CheckpointInfo) bool {
+	if a.Run != "" && b.Run != "" {
+		return a.Run == b.Run
+	}
+	return absDuration(a.Time.Sub(b.Time)) <= sameRun
+}
+
+func absDuration(d time.Duration) time.Duration {
+	if d < 0 {
+		return -d
+	}
+	return d
+}
diff --git a/server/internal/revocation.go b/server/internal/revocation.go
index 24c4a9c..136fcd4 100644
--- a/server/internal/revocation.go
+++ b/server/internal/revocation.go
@@ -213,8 +213,12 @@ func (r *revocations) refuses(contract uint32, uuid uint64, issuedAt uint32) str
 
 // apply merges changes into the state and saves it, and sends what changed
 // to every other node but from, if set: the node changes came from. It
-// returns what changed, as it is now.
+// returns what changed, as it is now. A change this node makes (from unset)
+// goes in the security journal first (offsite.go), for a restore to replay.
 func (r *revocations) apply(changes map[uint32]*ContractState, from string) (map[uint32]*ContractState, error) {
+	if from == "" {
+		journalSecurity(changes)
+	}
 	r.mu.Lock()
 	changed := r.mergeLocked(changes, time.Now().Unix())
 	var err error
@@ -230,6 +234,19 @@ func (r *revocations) apply(changes map[uint32]*ContractState, from string) (map
 	return changed, err
 }
 
+// replay merges changes from the security journal into the state and saves
+// it, without sending them: every node of a restore replays the journal.
+// It returns what changed.
+func (r *revocations) replay(changes map[uint32]*ContractState) (map[uint32]*ContractState, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	changed := r.mergeLocked(changes, time.Now().Unix())
+	if len(changed) == 0 {
+		return changed, nil
+	}
+	return changed, r.saveLocked()
+}
+
 // mergeLocked merges changes into the state, and returns the contracts that
 // changed, as they are now. The caller holds r.mu, or is the only one with r.
 func (r *revocations) mergeLocked(changes map[uint32]*ContractState, now int64) map[uint32]*ContractState {
diff --git a/server/internal/store/canary.go b/server/internal/store/canary.go
new file mode 100644
index 0000000..4cb5c5b
--- /dev/null
+++ b/server/internal/store/canary.go
@@ -0,0 +1,56 @@
+package store
+
+import (
+	"errors"
+	"hash/fnv"
+	"strconv"
+)
+
+// The canary (docs/backup-restore.md): a node writes a backup
+// run's id into the store just before its checkpoint of the run, both as a
+// message (the DB) and as a record (memdb), so that the weekly restore test
+// can tell a checkpoint that opens with what it held from one that only
+// opens. Both are written as records are, sealed at rest when sealing is
+// on, so reading them back also proves the escrowed keyring opens them.
+
+const sysCanary = "canary"
+
+// canaryKey is the canary record's memdb key: a hash of a string no other
+// record's key comes from.
+var canaryKey = func() uint64 {
+	h := fnv.New64a()
+	h.Write([]byte("\x00unitdb-backup-canary"))
+	return h.Sum64()
+}()
+
+// canaryTTL keeps the canary message as long as the oldest backup.
+const canaryTTL = 400 * 24 * 3600
+
+// WriteCanary writes run's id as the canary message and record.
+func WriteCanary(run string) error {
+	if err := adp.Put(sysContract, sysTopic(sysCanary, "run"), []byte(run), strconv.Itoa(canaryTTL)); err != nil {
+		return err
+	}
+	return adp.PutMessage(canaryKey, []byte(run))
+}
+
+// ReadCanary returns the run ids of the newest canary message and of the
+// canary record.
+func ReadCanary() (message, record string, err error) {
+	// Run ids sort as their times do: the newest is the largest, whatever
+	// order the store returns them in.
+	msgs, err := adp.Get(sysContract, sysTopic(sysCanary, "run"), "")
+	if err != nil {
+		return "", "", err
+	}
+	for _, m := range msgs {
+		if s := string(m); s > message {
+			message = s
+		}
+	}
+	b, err := adp.GetMessage(canaryKey)
+	if err != nil && message == "" {
+		return "", "", errors.New("no canary")
+	}
+	return message, string(b), nil
+}
diff --git a/server/internal/store/canary_test.go b/server/internal/store/canary_test.go
new file mode 100644
index 0000000..291d5ff
--- /dev/null
+++ b/server/internal/store/canary_test.go
@@ -0,0 +1,20 @@
+package store_test
+
+import (
+	"testing"
+
+	"github.com/unit-io/unitdb/server/internal/store"
+)
+
+func TestCanary(t *testing.T) {
+	newStore(t, 0, ring(0), true)
+	for _, run := range []string{"20261004T213000Z", "20261005T213000Z"} {
+		if err := store.WriteCanary(run); err != nil {
+			t.Fatal(err)
+		}
+	}
+	msg, rec, err := store.ReadCanary()
+	if err != nil || msg != "20261005T213000Z" || rec != "20261005T213000Z" {
+		t.Errorf("canary: message %q, record %q, %v; want the newest run", msg, rec, err)
+	}
+}
diff --git a/server/internal/store/checkpoint_test.go b/server/internal/store/checkpoint_test.go
index 99fbc55..a0468ce 100644
--- a/server/internal/store/checkpoint_test.go
+++ b/server/internal/store/checkpoint_test.go
@@ -20,10 +20,13 @@ func TestCheckpointAndStats(t *testing.T) {
 		t.Errorf("stats %+v", s)
 	}
 	dst := t.TempDir()
-	if err := store.Checkpoint(dst); err != nil {
+	if _, err := store.Checkpoint(dst, store.CheckpointInfo{Node: "n1"}); err != nil {
 		t.Fatal(err)
 	}
-	if err := store.Checkpoint(dst); err == nil {
+	if info, err := store.ReadCheckpointInfo(dst); err != nil || info == nil || info.Node != "n1" {
+		t.Errorf("the checkpoint's description: %+v, %v", info, err)
+	}
+	if _, err := store.Checkpoint(dst, store.CheckpointInfo{}); err == nil {
 		t.Error("a checkpoint into a directory that isn't empty was taken")
 	}
 	if err := store.Probe(); err != nil {
diff --git a/server/internal/store/security.go b/server/internal/store/security.go
index a561499..c83d64b 100644
--- a/server/internal/store/security.go
+++ b/server/internal/store/security.go
@@ -113,12 +113,37 @@ func Probe() error {
 // Stats is the size of the store.
 type Stats = adapter.Stats
 
+// CheckpointInfo describes a checkpoint (CheckpointInfoFile in it).
+type CheckpointInfo = adapter.CheckpointInfo
+
+// CheckpointInfoFile is the file in a checkpoint that describes it.
+const CheckpointInfoFile = adapter.CheckpointInfoFile
+
+// ManifestFile is the file in a checkpoint of a backup run that lists the
+// run's checkpoints.
+const ManifestFile = adapter.ManifestFile
+
 // Checkpoint writes a copy of the store into dst, a directory that doesn't
 // exist or is empty, that opens as the store was at one moment (db_path set
 // to dst). Writes wait while it runs. Records sealed at rest stay sealed: the
-// copy opens with the same keyring only. Errors start "store checkpoint: ".
-func Checkpoint(dst string) error {
-	return adp.Checkpoint(dst)
+// copy opens with the same keyring only. Last it writes info, with the
+// copy's stats and the engine's version, into the copy as
+// CheckpointInfoFile, and returns what it wrote. Errors start
+// "store checkpoint: ".
+func Checkpoint(dst string, info CheckpointInfo) (CheckpointInfo, error) {
+	return adp.Checkpoint(dst, info)
+}
+
+// WriteManifest writes a backup run's manifest into dir, this node's
+// checkpoint of the run.
+func WriteManifest(dir string, manifest []byte) error {
+	return adapter.WriteManifest(dir, manifest)
+}
+
+// ReadCheckpointInfo returns the description of the checkpoint in dir, or
+// nil if dir isn't one.
+func ReadCheckpointInfo(dir string) (*CheckpointInfo, error) {
+	return adapter.ReadCheckpointInfo(dir)
 }
 
 // StoreStats returns the size of the store, cheaply: for metrics.
diff --git a/server/main.go b/server/main.go
index 7ad9923..8a19659 100644
--- a/server/main.go
+++ b/server/main.go
@@ -18,6 +18,7 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"flag"
 	"os"
 	"path/filepath"
@@ -44,6 +45,8 @@ func main() {
 	var clusterSelf = flag.String("cluster_self", "", "Override the name of the current cluster node")
 	var dbPath = flag.String("db_path", "/tmp/unitdb", "Override the db path.")
 	var varzPath = flag.String("varz", "/varz", "Expose runtime stats at the given endpoint, e.g. /varz. Disabled if not set")
+	var journalDir = flag.String("journal", "", "With -restored: a directory of the security journal since the backup run (the backup command's fetch-journal), replayed before the node takes clients.")
+	var restored = flag.Bool("restored", false, "Start a node at its checkpoint of a backup run (-db_path), and reconcile it with the other nodes before it takes clients (docs/backup-restore.md).")
 	flag.Parse()
 
 	// Default level for is fatal, unless debug flag is present
@@ -90,6 +93,19 @@ func main() {
 	// Cluster won't be started here yet.
 	internal.ClusterInit(cfg.Cluster, clusterSelf)
 
+	// Before the store opens: a node at a checkpoint doesn't start into a
+	// cluster that runs without it.
+	if err := internal.CheckStart(string(cfg.DBPath), *restored); err != nil {
+		log.Fatal("main", "Refusing to start", err)
+	}
+	if *journalDir != "" && !*restored {
+		log.Fatal("main", "Refusing to start", errors.New("-journal is replayed by a node started with -restored"))
+	}
+	// Before the service: the security changes it makes are journalled.
+	if err := internal.InitOffsite(); err != nil {
+		log.Fatal("main", "Failed to start the backups' uploads", err)
+	}
+
 	svc, err := internal.NewService(cfg)
 	if err != nil {
 		log.Fatal("main", "Failed to start the service", err)
@@ -100,6 +116,14 @@ func main() {
 	// connections of the service.
 	internal.Globals.Service = svc
 
+	// Restored: what was revoked since the backup run is revoked again,
+	// before the cluster and the clients see this node.
+	if *journalDir != "" {
+		if err := internal.ReplayJournal(*journalDir); err != nil {
+			log.Fatal("main", "Refusing to start", err)
+		}
+	}
+
 	// Start accepting cluster traffic.
 	if internal.Globals.Cluster != nil {
 		internal.Globals.Cluster.Start()

From ee7b158c9a59a67ff1724bdd8091b43ef1f7b13b Mon Sep 17 00:00:00 2001
From: unit-adm <314923187+mumtaz6@users.noreply.github.com>
Date: Sun, 4 Oct 2026 12:12:50 +0530
Subject: [PATCH 3/3] CI: give the tests 25 minutes

The e2e tests took 599 s of go test's default 10 minutes with the
backup and restore tests in: give them room, within the job's 30.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5acc3ff..d83433f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
           go-version: ${{ matrix.go != 'go.mod' && matrix.go || '' }}
           go-version-file: ${{ matrix.go == 'go.mod' && 'go.mod' || '' }}
       - run: go vet ./...
-      - run: go test ./...
+      - run: go test -timeout 25m ./...
 
   staticcheck:
     runs-on: ubuntu-latest