Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions ci/journey.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3303,6 +3303,76 @@ EOSQL
q "$HOST" "DROP TABLE IF EXISTS public.tc113_unlogged CASCADE;" >/dev/null 2>&1
}

# ───────────────────────────────────────────────────────────────────────────
# Story — partition bounds outside the ordinary finite-ISO shape. PostgreSQL
# spans 4713 BC to 5874897 AD and renders " BC" below year 1, and a partition
# edge may be open (MINVALUE/MAXVALUE, infinity) or absent entirely (DEFAULT).
# A table carrying any of these must not abort the run: the archiver reads every
# table in one pass, so one unreadable bound would strand every table after it.
# ───────────────────────────────────────────────────────────────────────────
story_exotic_partition_bounds() {
step "exotic partition bounds do not abort the archiver"
# The MINVALUE partition ends up below the watermark once the BC one
# archives, so it takes the cleanupAlreadyArchived path, whose peer detach
# needs peer DSNs the host-side archiver cannot resolve. Same reason TC-024
# skips here; bound parsing is topology-independent and the vanilla cells
# cover it.
if [ "$MESH" = 1 ]; then
note "exotic bounds: skipping in mesh mode (cleanupAlreadyArchived peer detach requires resolvable peer DSNs)"
return
fi
local dsn="host=${DB_IP} port=5432 dbname=coldfront user=coldfront password=coldfront sslmode=disable"
q "$HOST" "CREATE SCHEMA IF NOT EXISTS xb;" >/dev/null
# Ascending, non-overlapping, all inside PostgreSQL's range (4713 BC to
# 294276 AD for timestamptz). No default here: a default constrains
# maintenance table-wide, which is exercised on its own table below.
qf "$HOST" <<'EOSQL' >/dev/null
CREATE TABLE IF NOT EXISTS xb.events (
id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz NOT NULL, val text,
PRIMARY KEY (id, ts)
) PARTITION BY RANGE (ts);
CREATE TABLE IF NOT EXISTS xb.events_open_lo PARTITION OF xb.events FOR VALUES FROM (MINVALUE) TO ('0100-01-01 BC');
CREATE TABLE IF NOT EXISTS xb.events_bc PARTITION OF xb.events FOR VALUES FROM ('0100-01-01 BC') TO ('0043-01-01 BC');
CREATE TABLE IF NOT EXISTS xb.events_wide PARTITION OF xb.events FOR VALUES FROM ('10000-06-01') TO ('20000-01-01');
INSERT INTO xb.events (ts, val) VALUES ('0044-03-15 00:00:00+00 BC', 'ides');
EOSQL
"$PARTITIONER" register --dsn "$dsn" --schema xb --table events \
--period monthly --hot-period "30 days" --retention "5 years" >/dev/null 2>&1
if archive_only "schema_name='xb' AND table_name='events'" /tmp/journey-archiver.yaml /tmp/journey-xb.log; then
pass "exotic bounds: archiver completed instead of aborting"
else
fail "exotic bounds: archiver aborted — see /tmp/journey-xb.log"; tail -8 /tmp/journey-xb.log
fi
# The BC partition is past the hot window, so it tiered: its bound is read as
# an astronomical year and written back with the era suffix PostgreSQL wants.
assert_eq "exotic bounds: the BC partition archived to the cold tier" "1" \
"$(q "$HOST" "SELECT count(*) FROM coldfront.tiered_views WHERE schema_name='xb' AND relname='events';")"
# MINVALUE and the year-20000 bound must not have stopped the pass.
assert_contains "exotic bounds: the BC partition reached cutover" "archived events_bc" \
"$(cat /tmp/journey-xb.log)"

# A DEFAULT partition is refused outright at registration: its rows could
# never tier or expire, and PostgreSQL blocks concurrent detach of every
# partition of the table while one exists, which is how partitions expire.
qf "$HOST" <<'EOSQL' >/dev/null
CREATE TABLE IF NOT EXISTS public.xb_withdef (
id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz NOT NULL,
PRIMARY KEY (id, ts)
) PARTITION BY RANGE (ts);
CREATE TABLE IF NOT EXISTS public.xb_withdef_default PARTITION OF public.xb_withdef DEFAULT;
EOSQL
assert_register_rejected "default partition: register names it and the remedy" \
xb_withdef xb_withdef_default
assert_eq "default partition: nothing registered" "0" \
"$(q "$HOST" "SELECT count(*) FROM coldfront.partition_config WHERE table_name='xb_withdef';")"
q "$HOST" "DROP TABLE IF EXISTS public.xb_withdef CASCADE;" >/dev/null 2>&1

q "$HOST" "DELETE FROM coldfront.partition_config WHERE schema_name='xb';" >/dev/null 2>&1
q "$HOST" "DELETE FROM coldfront.archive_watermark WHERE schema_name='xb';" >/dev/null 2>&1
q "$HOST" "DELETE FROM coldfront.tiered_views WHERE schema_name='xb';" >/dev/null 2>&1
q "$HOST" "DROP SCHEMA xb CASCADE;" >/dev/null 2>&1
}

# ───────────────────────────────────────────────────────────────────────────
# Story — a name differing only by case is refused at register. PostgreSQL keeps
# public."Events" and public.events apart; DuckDB matches identifiers
Expand Down Expand Up @@ -3763,6 +3833,7 @@ if [ "$MODE" = "tiered" ]; then
story_partition_col_timestamp # TC-066: timestamp (no tz) partition column
story_partition_col_date # TC-067: date partition column
story_partition_col_text_rejected # TC-070: text partition column rejected at archive
story_exotic_partition_bounds # BC, wide years, open-ended and DEFAULT bounds
story_schema_collision # TC-138: same table name in two schemas — distinct Iceberg namespaces
story_seaweedfs_restart # TC-062: SeaweedFS restart; cold data persists
story_pg_restart # TC-063: PG restart; DuckDB secret reloads; cold reads work
Expand Down
6 changes: 3 additions & 3 deletions cmd/archiver/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -905,7 +905,7 @@ func dropColdBeforeRetention(ctx context.Context, conn *pgx.Conn, iceTable, part
`DELETE FROM %s WHERE %s < '%s'::timestamptz`,
iceTable,
pgx.Identifier{partCol}.Sanitize(),
cutoff.UTC().Format("2006-01-02 15:04:05+00"))
sqlutil.Timestamp(cutoff))
q, err := dollarQuote(inner)
if err != nil {
return err
Expand Down Expand Up @@ -1134,9 +1134,9 @@ func wipeIcebergRange(ctx context.Context, conn *pgx.Conn, iceTable, partCol str
`DELETE FROM %s WHERE %s >= '%s'::timestamptz AND %s < '%s'::timestamptz%s`,
iceTable,
pgx.Identifier{partCol}.Sanitize(),
lower.UTC().Format("2006-01-02 15:04:05+00"),
sqlutil.Timestamp(lower),
pgx.Identifier{partCol}.Sanitize(),
upper.UTC().Format("2006-01-02 15:04:05+00"),
sqlutil.Timestamp(upper),
listPred)
q, err := dollarQuote(inner)
if err != nil {
Expand Down
33 changes: 33 additions & 0 deletions docs/usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,39 @@ The data lifecycle is **hot PG → `hot_period` → cold Iceberg →
→ dropped** (partition-only). Setting `hot_period` makes a table tiered;
omitting it makes it partition-only.

### What registration refuses

`register`, `import` and `set` share one validation gate, so a table that
one command rejects cannot be added by another. Registration fails when:

- any relation in the partition tree is `UNLOGGED`, because that data is
truncated after a crash and replicates nowhere, so archiving it would
preserve rows PostgreSQL never promised to keep.
- the table has a `DEFAULT` partition. Rows it catches carry no time
bounds, so they can never be tiered or expired, and PostgreSQL refuses
`DETACH PARTITION ... CONCURRENTLY` for every partition of a table that
has one, which is how partitions are expired. Its mere existence is
enough; it does not have to hold any rows. Move any rows it holds into
real partitions and detach it.
- the name differs only by case from an already-registered table.
PostgreSQL keeps `public."Events"` and `public.events` apart, but DuckDB
matches identifiers case-insensitively even when they are quoted, so both
names would resolve to one Iceberg table and overwrite each other.
- the name starts with an underscore, which is reserved for the tiered hot
table.
- the name leaves no room for the generated partition suffix: 53
characters for monthly and 50 for daily, within PostgreSQL's 63-byte
identifier limit.

Registration validates the table as it is at that moment, not
continuously. Adding a `DEFAULT` partition to an already-registered table
is therefore not caught then; the next archiver run fails while reading
that table's partition bounds.

Other unusual bounds are supported rather than refused. Partitions open at
either end (`MINVALUE`, `MAXVALUE`, `infinity`) are handled, as is
PostgreSQL's full timestamp range, from 4713 BC through year 294276.

`hot_period` and `retention_period` are native PostgreSQL `interval`s -
use any interval syntax (`1 month`, `90 days`, `1 year 2 mons`, `5
years`). Expiry boundaries are computed with calendar-accurate interval
Expand Down
32 changes: 32 additions & 0 deletions internal/partcfg/commands.go
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ type validateDB interface {
// - the name is representable by the partition naming scheme: no reserved
// leading underscore, and short enough for the generated leaf names.
// - every relation in the partition tree is WAL-logged (see requireLogged).
// - the table has no DEFAULT partition (see requireNoDefaultPartition).
func validateRow(ctx context.Context, db validateDB, row configRow) error {
// After the archiver's first-run swap the source is a VIEW over "_"+name, so
// validate the PK / partition key against the real partitioned table. register
Expand All @@ -209,6 +210,9 @@ func validateRow(ctx context.Context, db validateDB, row configRow) error {
if err := requireLogged(ctx, db, row.schema, base); err != nil {
return err
}
if err := requireNoDefaultPartition(ctx, db, row.schema, base); err != nil {
return err
}
if err := validatePKSuperset(ctx, db, row.schema, base, row.column, row.subValues != ""); err != nil {
return err
}
Expand Down Expand Up @@ -239,6 +243,34 @@ func requireNoCaseCollision(ctx context.Context, db partition.RowQuerier, schema
return nil
}

// requireNoDefaultPartition rejects a table that has a DEFAULT partition. Such a
// partition has no bounds, so nothing it catches can be tiered or expired, and
// PostgreSQL refuses DETACH CONCURRENTLY for every partition of a table that has
// one, which is how partitions are expired. Its presence also blocks creating a
// range partition over rows it already holds.
func requireNoDefaultPartition(ctx context.Context, db partition.RowQuerier, schema, table string) error {
var name string
err := db.QueryRow(ctx, `
SELECT coalesce(max(format('%I.%I', n.nspname, c.relname)), '')
FROM pg_inherits i
JOIN pg_class c ON c.oid = i.inhrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
JOIN pg_class p ON p.oid = i.inhparent
JOIN pg_namespace pn ON pn.oid = p.relnamespace
WHERE pn.nspname = $1::text AND p.relname = $2::text
AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'`, schema, table).Scan(&name)
if err != nil {
return fmt.Errorf("check for a default partition on %s.%s: %w", schema, table, err)
}
if name != "" {
return fmt.Errorf("%s.%s has a default partition (%s): its rows can never be tiered or "+
"expired, and PostgreSQL blocks concurrent detach of every partition while it exists; "+
"move any rows it holds and detach it",
schema, table, name)
}
return nil
}

// requireLogged rejects a table whose partition tree contains an UNLOGGED
// relation: that data is truncated after a crash and replicates nowhere. The
// whole tree is checked, since a permanent parent may hold UNLOGGED partitions
Expand Down
34 changes: 34 additions & 0 deletions internal/partcfg/commands_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -263,3 +263,37 @@ func TestRequireNoCaseCollision_PropagatesQueryError(t *testing.T) {
t.Fatal("query failure must not be reported as no collision")
}
}

func TestRequireNoDefaultPartition_Rejects(t *testing.T) {
// A DEFAULT partition has no bounds, so its rows never tier and never
// expire, and PostgreSQL refuses DETACH CONCURRENTLY for every partition of
// a table that has one, which is how partitions are expired. The error names
// it and the remedy.
db := &persistDB{scalar: "public.events_default"}
err := requireNoDefaultPartition(context.Background(), db, "public", "events")
if err == nil {
t.Fatal("expected rejection")
}
for _, want := range []string{"events_default", "default partition", "detach"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("error %q does not mention %q", err.Error(), want)
}
}
}

func TestRequireNoDefaultPartition_AcceptsPlainTable(t *testing.T) {
db := &persistDB{scalar: ""}
if err := requireNoDefaultPartition(context.Background(), db, "public", "events"); err != nil {
t.Fatalf("a table with no default must pass: %v", err)
}
if len(db.gotArgs) != 2 || db.gotArgs[0] != "public" || db.gotArgs[1] != "events" {
t.Errorf("schema/table not passed as args: %v", db.gotArgs)
}
}

func TestRequireNoDefaultPartition_PropagatesQueryError(t *testing.T) {
db := &persistDB{err: errors.New("boom")}
if err := requireNoDefaultPartition(context.Background(), db, "public", "events"); err == nil {
t.Fatal("query failure must not be reported as no default")
}
}
38 changes: 35 additions & 3 deletions internal/partition/boundary.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"strconv"
"strings"
"time"

"github.com/pgedge/coldfront/internal/sqlutil"
)

// Partition key modes and id schemes. The valid set lives here (like the Period
Expand Down Expand Up @@ -41,7 +43,7 @@ type TimeBoundary struct{}

// Literal renders t as a quoted timestamptz bound.
func (TimeBoundary) Literal(t time.Time) string {
return "'" + t.Format("2006-01-02 15:04:05+00") + "'"
return "'" + sqlutil.Timestamp(t) + "'"
}

// Parse reads a (possibly quoted) timestamp bound back to its time.
Expand Down Expand Up @@ -101,22 +103,52 @@ func BoundaryFor(mode, scheme string) (Boundary, error) {
// the quoting and the type conversion.
var boundRe = regexp.MustCompile(`FOR VALUES FROM \((.+?)\) TO \((.+?)\)`)

// negInfinity and posInfinity stand for an unbounded partition edge, outside
// PostgreSQL's own range (4713 BC to 5874897 AD) so no real bound equals one.
// The ordinary comparisons then hold: an edge open above is never past the hot
// window, never expires, and still covers the current period.
var (
negInfinity = time.Date(-5000, 1, 1, 0, 0, 0, 0, time.UTC)
posInfinity = time.Date(6000000, 1, 1, 0, 0, 0, 0, time.UTC)
)

// openBound maps an unbounded edge token to its sentinel. MINVALUE/MAXVALUE are
// partition-bound keywords; infinity/-infinity are timestamp values.
func openBound(lit string) (time.Time, bool) {
switch strings.ToLower(unquote(lit)) {
case "minvalue", "-infinity":
return negInfinity, true
case "maxvalue", "infinity":
return posInfinity, true
}
return time.Time{}, false
}

// parseBoundPair extracts the lower and upper bound values from a
// pg_get_expr(relpartbound) string and converts both to time via b.
func parseBoundPair(expr string, b Boundary) (lower, upper time.Time, err error) {
m := boundRe.FindStringSubmatch(expr)
if m == nil {
return time.Time{}, time.Time{}, fmt.Errorf("cannot parse partition bound: %q", expr)
}
if lower, err = b.Parse(m[1]); err != nil {
if lower, err = parseEdge(m[1], b); err != nil {
return time.Time{}, time.Time{}, fmt.Errorf("parse lower bound: %w", err)
}
if upper, err = b.Parse(m[2]); err != nil {
if upper, err = parseEdge(m[2], b); err != nil {
return time.Time{}, time.Time{}, fmt.Errorf("parse upper bound: %w", err)
}
return lower, upper, nil
}

// parseEdge resolves one bound value: a sentinel when the edge is open, else the
// Boundary's own conversion.
func parseEdge(lit string, b Boundary) (time.Time, error) {
if t, ok := openBound(lit); ok {
return t, nil
}
return b.Parse(lit)
}

// unquote strips one layer of surrounding single quotes, if present.
func unquote(s string) string {
s = strings.TrimSpace(s)
Expand Down
65 changes: 65 additions & 0 deletions internal/partition/boundary_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,3 +132,68 @@ func TestParseBoundPair_UnquotedBigint(t *testing.T) {
t.Fatalf("hi = %v, want %v", hi, jun)
}
}

// An open bound (MINVALUE/MAXVALUE, or the infinity literals) carries no time
// and resolves to a sentinel outside PostgreSQL's domain.
func TestParseBoundPair_OpenBounds(t *testing.T) {
finite := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
tests := []struct {
name string
expr string
wantLow, wantHigh time.Time
}{
{"minvalue lower", "FOR VALUES FROM (MINVALUE) TO ('2026-01-01 00:00:00+00')", negInfinity, finite},
{"maxvalue upper", "FOR VALUES FROM ('2026-01-01 00:00:00+00') TO (MAXVALUE)", finite, posInfinity},
{"both open", "FOR VALUES FROM (MINVALUE) TO (MAXVALUE)", negInfinity, posInfinity},
{"-infinity lower", "FOR VALUES FROM ('-infinity') TO ('2026-01-01 00:00:00+00')", negInfinity, finite},
{"infinity upper", "FOR VALUES FROM ('2026-01-01 00:00:00+00') TO ('infinity')", finite, posInfinity},
}
for _, tc := range tests {
low, high, err := parseBoundPair(tc.expr, TimeBoundary{})
if err != nil {
t.Errorf("%s: %v", tc.name, err)
continue
}
if !low.Equal(tc.wantLow) || !high.Equal(tc.wantHigh) {
t.Errorf("%s: got (%v, %v), want (%v, %v)", tc.name, low, high, tc.wantLow, tc.wantHigh)
}
}
}

// The sentinels must sit outside PostgreSQL's representable range, or a real
// bound could collide with one.
func TestOpenBoundSentinelsOutsidePostgresDomain(t *testing.T) {
pgMin := time.Date(-4712, 1, 1, 0, 0, 0, 0, time.UTC) // 4713 BC
pgMax := time.Date(5874897, 12, 31, 0, 0, 0, 0, time.UTC) // date ceiling
if !negInfinity.Before(pgMin) {
t.Errorf("negInfinity %v must precede %v", negInfinity, pgMin)
}
if !posInfinity.After(pgMax) {
t.Errorf("posInfinity %v must follow %v", posInfinity, pgMax)
}
}

// A bound is parsed from PostgreSQL and later written back to it, so the two
// directions must agree. Go prints a non-positive astronomical year as a
// negative number, which PostgreSQL rejects with "time zone displacement out of
// range"; it wants the era suffix instead.
func TestTimeBoundaryLiteral_RoundTripsPostgresRendering(t *testing.T) {
for _, in := range []string{
"2026-06-01 00:00:00+00",
"10000-06-01 00:00:00+00",
"294276-12-31 00:00:00+00",
"0001-01-01 00:00:00+00 BC",
"0043-03-15 00:00:00+00 BC",
"4713-01-01 00:00:00+00 BC",
} {
parsed, err := (TimeBoundary{}).Parse("'" + in + "'")
if err != nil {
t.Errorf("Parse(%q): %v", in, err)
continue
}
got := (TimeBoundary{}).Literal(parsed)
if got != "'"+in+"'" {
t.Errorf("round trip of %q gave %s", in, got)
}
}
}
Loading