diff --git a/server/e2e/backup_test.go b/server/e2e/backup_test.go new file mode 100644 index 0000000..9279060 --- /dev/null +++ b/server/e2e/backup_test.go @@ -0,0 +1,188 @@ +package e2e + +// Backups: a checkpoint taken through POST /_checkpoint while writes go on +// opens, on a second server with the same key, as the store was: the +// restore drill. + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +const checkpointToken = "e2e-checkpoint-token" + +func checkpointPost(t *testing.T, s *server, token string) (int, string) { + t.Helper() + req, _ := http.NewRequest("POST", "http://"+s.monitorAddr+"/_checkpoint", nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := (&http.Client{Timeout: 60 * time.Second}).Do(req) + if err != nil { + t.Fatalf("checkpoint: %v", err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, strings.TrimSpace(string(b)) +} + +// storedOn returns the messages stored on topic, relayed from s: at least n +// of them, and any more that come before the relay goes quiet. A topic can +// hold more than n (writes taken during a checkpoint), and the relay +// doesn't send them oldest first. +func storedOn(t *testing.T, s *server, cid, topic string, n int) map[int]string { + t.Helper() + sub, err := dial(context.Background(), s.tcpAddr) + if err != nil { + t.Fatal(err) + } + defer sub.close() + if _, err := sub.connect(cid, true, nextSess()); err != nil { + t.Fatal(err) + } + rid, err := sub.relay(topic, "24h") + if err != nil { + t.Fatal(err) + } + if !sub.waitAck(rid, 5*time.Second) { + t.Fatalf("no relay ack") + } + got := map[int]string{} + deadline := time.Now().Add(30 * time.Second) + for { + wait := time.Until(deadline) + if len(got) >= n { + wait = 500 * time.Millisecond + } + msg, ok := sub.waitPub(wait) + if !ok { + break + } + for _, m := range msg.Messages { + if seq, body, ok := decodePayload(m.Payload); ok { + got[seq] = string(body) + } + } + } + return got +} + +func TestBackupCheckpointRestores(t *testing.T) { + dir := t.TempDir() + env := []string{"CHECKPOINT_DIR=" + dir, "CHECKPOINT_TOKEN=" + checkpointToken, "CHECKPOINT_MIN_MINUTES=0", "CHECKPOINT_KEEP=2"} + s := startServerWith(t, serverOpts{env: env}) + ctx := context.Background() + cid := newClientID(0x0e2eb4c7) + topic := "groups.backup.x.message" + + pub, err := dial(ctx, s.tcpAddr) + if err != nil { + t.Fatal(err) + } + defer pub.close() + if _, err := pub.connect(cid, true, nextSess()); err != nil { + t.Fatal(err) + } + n := scaled(50) + for i := 0; i < n; i++ { + id, err := pub.publish(1, topic, encodePayload(i, fmt.Sprintf("m%d", i)), "24h") + if err != nil { + t.Fatal(err) + } + if !pub.waitAck(id, 5*time.Second) { + t.Fatalf("publish %d: no RECEIPT", i) + } + } + // A subscription made and dropped before the checkpoint: a delete. + if _, err := pub.subscribe(1, topic); err == nil { + pub.unsubscribe(topic) + } + + // The token is needed, and writes go on while it runs. + if code, _ := checkpointPost(t, s, ""); code != http.StatusUnauthorized { + t.Errorf("no token: %d, want 401", code) + } + if code, _ := checkpointPost(t, s, "wrong"); code != http.StatusUnauthorized { + t.Errorf("a wrong token: %d, want 401", code) + } + var wg sync.WaitGroup + stop := make(chan struct{}) + wg.Add(1) + go func() { + defer wg.Done() + for i := n; ; i++ { + select { + case <-stop: + return + default: + } + if _, err := pub.publish(1, topic, encodePayload(i, fmt.Sprintf("m%d", i)), "24h"); err != nil { + return + } + time.Sleep(20 * time.Millisecond) + } + }() + code, body := checkpointPost(t, s, checkpointToken) + close(stop) + wg.Wait() + if code != http.StatusOK { + t.Fatalf("checkpoint: %d %s\nlogs:\n%s", code, body, s.logs.String()) + } + var ck struct { + Dir string `json:"dir"` + Seconds float64 `json:"seconds"` + } + if err := json.Unmarshal([]byte(body), &ck); err != nil || !strings.HasPrefix(ck.Dir, dir) { + t.Fatalf("checkpoint answer %q", body) + } + t.Logf("checkpoint in %.2fs at %s", ck.Seconds, ck.Dir) + + // The metrics say so. + if _, metrics := monitorGet(t, s, "/_metrics"); !strings.Contains(metrics, "unitdb_checkpoint_last_success_timestamp_seconds ") || + strings.Contains(metrics, "unitdb_checkpoint_last_success_timestamp_seconds 0\n") || + !strings.Contains(metrics, "unitdb_store_messages ") { + t.Errorf("metrics after a checkpoint:\n%s", metrics) + } + + // The drill: a second server on the checkpoint, with the same key. + restored := startServerWith(t, serverOpts{args: []string{"-db_path", ck.Dir}}) + got := storedOn(t, restored, cid, topic, n) + for i := 0; i < n; i++ { + if want := fmt.Sprintf("m%d", i); got[i] != want { + t.Fatalf("restored seq %d: %q, want %q (got %d messages)\nlogs:\n%s", i, got[i], want, len(got), restored.logs.String()) + } + } + t.Logf("restored %d messages, %d written during the checkpoint", len(got), len(got)-n) + + // Old checkpoints go: two are kept. + for i := 0; i < 2; i++ { + time.Sleep(1100 * time.Millisecond) // a new name each second + if code, body := checkpointPost(t, s, checkpointToken); code != http.StatusOK { + t.Fatalf("checkpoint %d: %d %s", i+2, code, body) + } + } + entries, _ := os.ReadDir(dir) + var kept []string + for _, e := range entries { + kept = append(kept, filepath.Base(e.Name())) + } + if len(kept) != 2 { + t.Errorf("kept %v, want the newest two", kept) + } +} + +func TestBackupCheckpointOff(t *testing.T) { + s := startServer(t) + if code, _ := checkpointPost(t, s, checkpointToken); code != http.StatusNotFound { + t.Errorf("checkpoints off: %d, want 404", code) + } +} diff --git a/server/e2e/helpers_test.go b/server/e2e/helpers_test.go index 8947b85..2455284 100644 --- a/server/e2e/helpers_test.go +++ b/server/e2e/helpers_test.go @@ -91,3 +91,14 @@ func collectUnique(c *client, want int, overall, quiet time.Duration) (map[int]s } return got, dups, nil } + +// scaled scales a load size down under -race, where the server and the test +// both run many times slower, so the suite stays within its time budget. +func scaled(n int) int { + if raceEnabled { + if n /= 4; n < 1 { + n = 1 + } + } + return n +} diff --git a/server/internal/checkpoint.go b/server/internal/checkpoint.go new file mode 100644 index 0000000..4c044ab --- /dev/null +++ b/server/internal/checkpoint.go @@ -0,0 +1,144 @@ +package internal + +import ( + "crypto/subtle" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/unit-io/unitdb/server/internal/pkg/log" + "github.com/unit-io/unitdb/server/internal/store" +) + +// Checkpoints: a copy of the store that opens as it was at one moment +// (store.Checkpoint), taken on request at POST /_checkpoint on the monitor +// port, into CHECKPOINT_DIR, and kept there for a volume snapshot or a copy +// elsewhere. +// Restoring is starting a server with -db_path at the checkpoint, and the +// same keyring. +// +// The monitor port has no other authentication, and a checkpoint holds +// writes back for up to about 1.5 s, so: +// - it's off unless CHECKPOINT_DIR and CHECKPOINT_TOKEN are both set; +// - a request needs "Authorization: Bearer "; +// - the server names the checkpoint's directory, never the request; +// - one runs at a time, and one per CHECKPOINT_MIN_MINUTES (10); +// - the newest CHECKPOINT_KEEP (3) are kept, older ones removed. + +const checkpointPrefix = "ckpt-" + +type checkpointer struct { + dir string + token string + keep int + minEvery time.Duration + + mu sync.Mutex // held while one runs + last time.Time // when the last one began, under mu + + lastOK atomic.Int64 // unix seconds of the last success; 0 before any + failures atomic.Int64 + seconds atomic.Int64 // milliseconds the last one took +} + +// newCheckpointerFromEnv returns the checkpointer the environment asks for, +// or nil when checkpoints are off. +func newCheckpointerFromEnv() *checkpointer { + dir, token := os.Getenv("CHECKPOINT_DIR"), os.Getenv("CHECKPOINT_TOKEN") + if dir == "" || token == "" { + return nil + } + keep, _ := strconv.Atoi(os.Getenv("CHECKPOINT_KEEP")) + if keep < 1 { + keep = 3 + } + mins, _ := strconv.Atoi(os.Getenv("CHECKPOINT_MIN_MINUTES")) + if mins < 0 || os.Getenv("CHECKPOINT_MIN_MINUTES") == "" { + mins = 10 + } + return &checkpointer{dir: dir, token: token, keep: keep, minEvery: time.Duration(mins) * time.Minute} +} + +func (c *checkpointer) authorized(r *http.Request) bool { + got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") + return subtle.ConstantTimeCompare([]byte(got), []byte(c.token)) == 1 +} + +func (c *checkpointer) handle(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "POST only", http.StatusMethodNotAllowed) + return + } + if !c.authorized(r) { + http.Error(w, "a checkpoint needs its token", http.StatusUnauthorized) + return + } + if !c.mu.TryLock() { + http.Error(w, "a checkpoint is running", http.StatusConflict) + return + } + defer c.mu.Unlock() + now := time.Now().UTC() + if !c.last.IsZero() && now.Sub(c.last) < c.minEvery { + http.Error(w, fmt.Sprintf("one checkpoint per %s", c.minEvery), http.StatusTooManyRequests) + return + } + c.last = now + dst := filepath.Join(c.dir, checkpointPrefix+now.Format("20060102T150405Z")) + start := time.Now() + err := store.Checkpoint(dst) + took := time.Since(start) + c.seconds.Store(took.Milliseconds()) + if err != nil { + c.failures.Add(1) + os.RemoveAll(dst) + log.Error("checkpoint", err.Error()) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + c.lastOK.Store(now.Unix()) + log.Info("checkpoint", fmt.Sprintf("checkpoint %s in %s", dst, took.Round(time.Millisecond))) + c.prune() + w.Header().Set("content-type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "dir": dst, + "seconds": took.Seconds(), + }) +} + +// prune removes all but the newest keep checkpoints. +func (c *checkpointer) prune() { + entries, err := os.ReadDir(c.dir) + if err != nil { + return + } + var names []string + for _, e := range entries { + if e.IsDir() && strings.HasPrefix(e.Name(), checkpointPrefix) { + names = append(names, e.Name()) + } + } + sort.Strings(names) // the names sort by time + for len(names) > c.keep { + if err := os.RemoveAll(filepath.Join(c.dir, names[0])); err != nil { + log.Error("checkpoint", "removing an old checkpoint: "+err.Error()) + } + names = names[1:] + } +} + +func (c *checkpointer) writeMetrics(m *metricsWriter) { + m.one("unitdb_checkpoint_last_success_timestamp_seconds", "gauge", + "When the last checkpoint succeeded, in unix seconds; 0 before any.", float64(c.lastOK.Load())) + m.one("unitdb_checkpoint_failures_total", "counter", "Checkpoints that failed.", float64(c.failures.Load())) + m.one("unitdb_checkpoint_duration_seconds", "gauge", + "How long the last checkpoint took, writes held back meanwhile.", float64(c.seconds.Load())/1000) +} diff --git a/server/internal/db/adapter.go b/server/internal/db/adapter.go index c7982e4..1bd26dd 100644 --- a/server/internal/db/adapter.go +++ b/server/internal/db/adapter.go @@ -88,4 +88,25 @@ type Adapter interface { // Flush waits for the messages put before it to be written to the // store's log, from which they are recovered after a crash. Flush() error + + // Checkpoint writes a copy of the store into dst, a directory that + // doesn't exist or is empty. The copy opens as the store was at one + // moment, as after a clean shutdown. Writes wait while it runs. + Checkpoint(dst string) error + + // Stats returns the size of the store. + Stats() Stats +} + +// Stats is the size of a store. +type Stats struct { + // Messages is the number of messages in the store. + Messages uint64 + // DiskBytes is the size of the store's files. + DiskBytes int64 + // MemEntries is the number of records in memory (sessions, logs and the + // like), and MemSize the configured size of that memory (mem_size); 0 + // if not configured. + MemEntries int64 + MemSize int64 } diff --git a/server/internal/db/unitdb/adapter.go b/server/internal/db/unitdb/adapter.go index 08843b3..71b984f 100644 --- a/server/internal/db/unitdb/adapter.go +++ b/server/internal/db/unitdb/adapter.go @@ -21,6 +21,9 @@ import ( "errors" "io" "os" + "sync" + "sync/atomic" + "time" "github.com/unit-io/unitdb" "github.com/unit-io/unitdb/memdb" @@ -56,6 +59,13 @@ type adapter struct { mem *memdb.DB // The underlying memdb to store messages. config *configType version int + // path is the directory the store is in. + path string + // wmu is held for reading by every write, and for writing by Checkpoint, + // so that a checkpoint copies the store between writes, and by Close. + wmu sync.RWMutex + // lastWrite is when the DB was last written to, in unix nanoseconds. + lastWrite atomic.Int64 // close closer io.Closer @@ -96,12 +106,15 @@ func (a *adapter) Open(path, jsonconfig string, reset bool) error { } a.config = &config + a.path = path return nil } // Close closes the underlying database connection func (a *adapter) Close() error { + a.wmu.Lock() + defer a.wmu.Unlock() var err error if a.db != nil { err = a.db.Close() @@ -128,6 +141,9 @@ func (a *adapter) GetName() string { // Put appends the messages to the store. func (a *adapter) Put(contract uint32, topic string, payload []byte, ttl string) error { + a.wmu.RLock() + defer a.wmu.RUnlock() + defer a.lastWrite.Store(time.Now().UnixNano()) entry := unitdb.NewEntry([]byte(topic), payload).WithContract(contract) if ttl != "" { entry.WithTTL(ttl) @@ -137,6 +153,9 @@ func (a *adapter) Put(contract uint32, topic string, payload []byte, ttl string) // PutWithID appends the messages to the store using a pre generated messageId. func (a *adapter) PutWithID(contract uint32, messageId []byte, topic string, payload []byte, ttl string) error { + a.wmu.RLock() + defer a.wmu.RUnlock() + defer a.lastWrite.Store(time.Now().UnixNano()) entry := unitdb.NewEntry([]byte(topic), payload).WithContract(contract).WithID(messageId) if ttl != "" { entry.WithTTL(ttl) @@ -178,6 +197,9 @@ func (a *adapter) NewID() ([]byte, error) { // Put appends the messages to the store. func (a *adapter) Delete(contract uint32, messageId []byte, topic string) error { + a.wmu.RLock() + defer a.wmu.RUnlock() + defer a.lastWrite.Store(time.Now().UnixNano()) entry := unitdb.NewEntry([]byte(topic), nil) entry.WithContract(contract) return a.db.DeleteEntry(entry.WithID(messageId)) @@ -189,6 +211,8 @@ func (a *adapter) Delete(contract uint32, messageId []byte, topic string) error // a get returns the latest: the older versions are deleted first, so that a // later delete removes the key. func (a *adapter) PutMessage(key uint64, payload []byte) error { + a.wmu.RLock() + defer a.wmu.RUnlock() if err := a.deleteVersions(key); err != nil { return err } @@ -229,6 +253,8 @@ func (a *adapter) Flush() error { // DeleteMessage deletes message from memdb store. func (a *adapter) DeleteMessage(key uint64) error { + a.wmu.RLock() + defer a.wmu.RUnlock() return a.deleteVersions(key) } diff --git a/server/internal/db/unitdb/checkpoint.go b/server/internal/db/unitdb/checkpoint.go new file mode 100644 index 0000000..8fc707b --- /dev/null +++ b/server/internal/db/unitdb/checkpoint.go @@ -0,0 +1,169 @@ +package adapter + +import ( + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/unit-io/unitdb/memdb" + + dbapi "github.com/unit-io/unitdb/server/internal/db" +) + +// A copy of a running store's files is not a backup: they are copied at +// different moments, and a copy taken while the DB syncs can pair a new index +// with old data. memdb writes its log in the background, so its files miss +// the latest records, and can hold one half written. A volume snapshot is +// like a power cut, which unitdb's recovery isn't tested for: the WAL isn't +// fsynced. +// +// Checkpoint copies the store between writes instead. It holds every write +// back, and waits until the DB can write out the latest: the DB keeps a new +// entry in memory until its time block has passed and its log has been +// committed, about 1.1s with unitdb's defaults (a 1s block, a 100ms commit +// interval), which the adapter uses. Then it syncs the DB and copies its +// files, and copies memdb's records, not its files, into a fresh memdb in +// dst, and fsyncs the copy. Reads go on throughout; writes wait, up to +// settleTime. The copy opens with db_path set to dst, as after a clean +// shutdown. Records sealed at rest stay sealed: it opens with the same +// keyring only. + +// settleTime is how long after its last write the DB has written everything +// out, with unitdb's default block duration and commit interval. +const settleTime = 1500 * time.Millisecond + +// Checkpoint writes a copy of the store into dst, which must not exist or be +// empty. +func (a *adapter) Checkpoint(dst string) error { + fail := func(what string, err error) error { + return fmt.Errorf("store checkpoint: %s: %v", what, err) + } + if a.db == nil { + return fail("the store", fmt.Errorf("is not open")) + } + if entries, err := os.ReadDir(dst); err == nil && len(entries) > 0 { + return fail(dst, fmt.Errorf("is not empty")) + } + if err := os.MkdirAll(dst, 0700); err != nil { + return fail("make "+dst, err) + } + + a.wmu.Lock() + defer a.wmu.Unlock() + + if wait := settleTime - time.Since(time.Unix(0, a.lastWrite.Load())); wait > 0 { + time.Sleep(wait) + } + // Twice: an entry whose log was committed during the first sync. + for i := 0; i < 2; i++ { + if err := a.db.Sync(); err != nil { + return fail("sync", err) + } + } + src := filepath.Join(a.path, defaultDatabase) + if err := copyTree(src, filepath.Join(dst, defaultDatabase)); err != nil { + return fail("copy the DB", err) + } + + var opts []memdb.Options + opts = append(opts, memdb.WithLogFilePath(dst)) + if a.config != nil && a.config.Size > 0 { + opts = append(opts, memdb.WithBufferSize(a.config.Size)) + } + mem, err := memdb.Open(opts...) + if err != nil { + return fail("open the copy's memdb", err) + } + for _, key := range a.Keys() { + b, err := a.mem.Get(key) + if err != nil { + continue // deleted since Keys listed it + } + if _, err := mem.Put(key, b); err != nil { + mem.Close() + return fail("copy memdb", err) + } + } + if err := mem.Close(); err != nil { + return fail("close the copy's memdb", err) + } + if err := syncTree(dst); err != nil { + return fail("fsync the copy", err) + } + return nil +} + +// Stats returns the size of the store. +func (a *adapter) Stats() dbapi.Stats { + var s dbapi.Stats + if a.db == nil { + return s + } + s.Messages = a.db.Count() + s.DiskBytes, _ = a.db.FileSize() + if a.mem != nil { + s.MemEntries = a.mem.Size() + } + if a.config != nil { + s.MemSize = a.config.Size + } + return s +} + +// copyTree copies the files under src to dst, but for unitdb's lock file, +// which the copy's DB makes its own. +func copyTree(src, dst string) error { + return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + switch { + case info.IsDir(): + return os.MkdirAll(target, 0700) + case filepath.Ext(path) == ".lock": + return nil + case !info.Mode().IsRegular(): + return nil + } + return copyFile(path, target, info.Mode().Perm()) + }) +} + +func copyFile(src, dst string, perm os.FileMode) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + return err + } + return out.Close() +} + +// syncTree fsyncs every file and directory under dir. +func syncTree(dir string) error { + return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return f.Sync() + }) +} diff --git a/server/internal/db/unitdb/checkpoint_test.go b/server/internal/db/unitdb/checkpoint_test.go new file mode 100644 index 0000000..7e24cea --- /dev/null +++ b/server/internal/db/unitdb/checkpoint_test.go @@ -0,0 +1,112 @@ +package adapter + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" +) + +const testConfig = `{"mem_size": 16777216}` + +func openAdapter(t *testing.T, dir string) *adapter { + t.Helper() + a := &adapter{} + if err := a.Open(dir, testConfig, false); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + return a +} + +// TestCheckpoint checks that a checkpoint taken while writes go on opens as +// a store with every message and record written before it, and that it +// refuses a directory that isn't empty. +func TestCheckpoint(t *testing.T) { + a := openAdapter(t, t.TempDir()) + const contract, topic = 7, "groups.checkpoint" + for i := 0; i < 50; i++ { + if err := a.Put(contract, topic, []byte(fmt.Sprintf("before-%d", i)), ""); err != nil { + t.Fatal(err) + } + if err := a.PutMessage(uint64(1000+i), []byte(fmt.Sprintf("record-%d", i))); err != nil { + t.Fatal(err) + } + } + // A record written twice keeps its last version. + if err := a.PutMessage(1000, []byte("record-0-again")); err != nil { + t.Fatal(err) + } + + // Writes go on during the checkpoint. + stop := make(chan struct{}) + var during atomic.Int64 + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + a.Put(contract, topic, []byte(fmt.Sprintf("during-%d", i)), "") + a.PutMessage(uint64(5000+i%100), []byte("during")) + during.Add(1) + } + }() + dst := filepath.Join(t.TempDir(), "checkpoint") + err := a.Checkpoint(dst) + close(stop) + wg.Wait() + if err != nil { + t.Fatal(err) + } + if during.Load() == 0 { + t.Log("no write ran during the checkpoint") + } + + b := openAdapter(t, dst) + msgs, err := b.Get(contract, topic, "") + if err != nil { + t.Fatal(err) + } + before := 0 + for _, m := range msgs { + if bytes.HasPrefix(m, []byte("before-")) { + before++ + } + } + if before != 50 { + t.Errorf("the copy has %d of the 50 messages written before the checkpoint (%d in all)", before, len(msgs)) + } + for i := 0; i < 50; i++ { + want := fmt.Sprintf("record-%d", i) + if i == 0 { + want = "record-0-again" + } + got, err := b.GetMessage(uint64(1000 + i)) + if err != nil || string(got) != want { + t.Errorf("record %d in the copy: %q (%v), want %q", i, got, err, want) + } + } + + if s := b.Stats(); s.Messages == 0 || s.DiskBytes == 0 || s.MemEntries == 0 || s.MemSize != 16777216 { + t.Errorf("stats of the copy: %+v", s) + } + + // A directory that isn't empty is refused. + if err := a.Checkpoint(dst); err == nil || !strings.HasPrefix(err.Error(), "store checkpoint: ") { + t.Errorf("a checkpoint into a directory that isn't empty: %v", err) + } + if _, err := os.Stat(filepath.Join(dst, defaultDatabase, "unitdb.lock")); err != nil { + // The copy's own DB made one when it opened; the checkpoint didn't + // copy the source's. + t.Logf("no lock file in the copy: %v", err) + } +} diff --git a/server/internal/health.go b/server/internal/health.go index b20ec88..381ed40 100644 --- a/server/internal/health.go +++ b/server/internal/health.go @@ -61,6 +61,8 @@ type healthMonitor struct { srv *http.Server // metrics writes /_metrics, when set. metrics func(io.Writer) + // checkpoints takes POST /_checkpoint, when set. + checkpoints *checkpointer } func newHealthMonitor(started time.Time) *healthMonitor { @@ -174,6 +176,9 @@ func (h *healthMonitor) handler() http.Handler { } w.Write([]byte("ready")) }) + if h.checkpoints != nil { + mux.HandleFunc("/_checkpoint", h.checkpoints.handle) + } mux.HandleFunc("/_metrics", func(w http.ResponseWriter, r *http.Request) { if h.metrics == nil { http.NotFound(w, r) @@ -244,6 +249,9 @@ func (h *healthMonitor) close() { // addServiceChecks adds the server's checks: the store, and in a cluster, // this node's place in it. func (h *healthMonitor) addServiceChecks() { + // Checkpoints, when the environment asks for them (checkpoint.go). + h.checkpoints = newCheckpointerFromEnv() + h.add("store", func() (string, error) { if !store.IsOpen() { return "", errors.New("the store is not open") diff --git a/server/internal/metrics.go b/server/internal/metrics.go index 7692f44..b043d41 100644 --- a/server/internal/metrics.go +++ b/server/internal/metrics.go @@ -7,6 +7,8 @@ import ( "sort" "strings" "time" + + "github.com/unit-io/unitdb/server/internal/store" ) // Metrics, in the Prometheus text format, at /_metrics on the monitor port: @@ -130,7 +132,27 @@ func (s *_Service) writeMetrics(w io.Writer) { m.one("unitdb_draining", "gauge", "Whether the server is shutting down.", draining) } + // The store: cheap to read per scrape. + if st := s.storeStats(); st != nil { + m.one("unitdb_store_messages", "gauge", "Messages in the store.", float64(st.Messages)) + m.one("unitdb_store_disk_bytes", "gauge", "Bytes of the store's files.", float64(st.DiskBytes)) + m.one("unitdb_store_mem_entries", "gauge", "Records in the memory store (memdb): entries, not bytes.", float64(st.MemEntries)) + m.one("unitdb_store_mem_size", "gauge", "The memory store's configured mem_size; 0 if unset.", float64(st.MemSize)) + } + if s.health != nil && s.health.checkpoints != nil { + s.health.checkpoints.writeMetrics(m) + } + if c := Globals.Cluster; c != nil { c.writeMetrics(m) } } + +// storeStats is the store's size, or nil when it isn't open. +func (s *_Service) storeStats() *store.Stats { + if !store.IsOpen() { + return nil + } + st := store.StoreStats() + return &st +} diff --git a/server/internal/store/checkpoint_test.go b/server/internal/store/checkpoint_test.go new file mode 100644 index 0000000..99fbc55 --- /dev/null +++ b/server/internal/store/checkpoint_test.go @@ -0,0 +1,35 @@ +package store_test + +import ( + "testing" + + "github.com/unit-io/unitdb/server/internal/store" +) + +// TestCheckpointAndStats checks the store's checkpoint and stats, which the +// adapter's tests cover in depth. +func TestCheckpointAndStats(t *testing.T) { + newStore(t, 0, ring(0), true) + if err := store.Message.Put(contract, "groups.stats", []byte("x"), ""); err != nil { + t.Fatal(err) + } + if err := store.Probe(); err != nil { + t.Fatal(err) + } + if s := store.StoreStats(); s.DiskBytes == 0 || s.MemEntries == 0 { + t.Errorf("stats %+v", s) + } + dst := t.TempDir() + if err := store.Checkpoint(dst); err != nil { + t.Fatal(err) + } + if err := store.Checkpoint(dst); err == nil { + t.Error("a checkpoint into a directory that isn't empty was taken") + } + if err := store.Probe(); err != nil { + t.Errorf("the store after a checkpoint: %v", err) + } + if got, err := store.Message.Get(contract, "groups.stats", ""); err != nil || len(got) == 0 { + t.Errorf("the store after a checkpoint: %d messages (%v)", len(got), err) + } +} diff --git a/server/internal/store/security.go b/server/internal/store/security.go index c32f95b..a561499 100644 --- a/server/internal/store/security.go +++ b/server/internal/store/security.go @@ -22,6 +22,8 @@ import ( "errors" "fmt" "hash/fnv" + + adapter "github.com/unit-io/unitdb/server/internal/db" ) // securityTopic is the topic the cluster's security state is kept under, in @@ -107,3 +109,19 @@ func Probe() error { } return nil } + +// Stats is the size of the store. +type Stats = adapter.Stats + +// Checkpoint writes a copy of the store into dst, a directory that doesn't +// exist or is empty, that opens as the store was at one moment (db_path set +// to dst). Writes wait while it runs. Records sealed at rest stay sealed: the +// copy opens with the same keyring only. Errors start "store checkpoint: ". +func Checkpoint(dst string) error { + return adp.Checkpoint(dst) +} + +// StoreStats returns the size of the store, cheaply: for metrics. +func StoreStats() Stats { + return adp.Stats() +}