From f09b6f693312fb5f01ff832a9b22d1e1f0b06fd7 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 11:29:48 +0200 Subject: [PATCH 01/30] testserver: add storage for postgres synced tables --- libs/testserver/fake_workspace.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 5430c68cbcc..db3ea7afcd4 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -168,10 +168,11 @@ type FakeWorkspace struct { DatabaseCatalogs map[string]database.DatabaseCatalog SyncedDatabaseTables map[string]database.SyncedDatabaseTable - PostgresProjects map[string]postgres.Project - PostgresBranches map[string]postgres.Branch - PostgresEndpoints map[string]postgres.Endpoint - PostgresOperations map[string]postgres.Operation + PostgresProjects map[string]postgres.Project + PostgresBranches map[string]postgres.Branch + PostgresEndpoints map[string]postgres.Endpoint + PostgresOperations map[string]postgres.Operation + PostgresSyncedTables map[string]postgres.SyncedTable // clusterVenvs caches Python venvs per existing cluster ID, // matching cloud behavior where libraries are cached on running clusters. @@ -300,6 +301,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { PostgresBranches: map[string]postgres.Branch{}, PostgresEndpoints: map[string]postgres.Endpoint{}, PostgresOperations: map[string]postgres.Operation{}, + PostgresSyncedTables: map[string]postgres.SyncedTable{}, clusterVenvs: map[string]*clusterEnv{}, Alerts: map[string]sql.AlertV2{}, Experiments: map[string]ml.GetExperimentResponse{}, From 9688b995045c8e06fe6bb81a849b6f24553fcc2c Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 11:33:43 +0200 Subject: [PATCH 02/30] testserver: add postgres synced table CRUD --- libs/testserver/postgres.go | 68 +++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index f3a488b5704..3da293997ae 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -608,9 +608,12 @@ func (s *FakeWorkspace) createOperationLocked(resourceName string, response any) // Determine resource type from name for metadata @type resourceType := "Project" - if strings.Contains(resourceName, "/endpoints/") { + switch { + case strings.HasPrefix(resourceName, "synced_tables/"): + resourceType = "SyncedTable" + case strings.Contains(resourceName, "/endpoints/"): resourceType = "Endpoint" - } else if strings.Contains(resourceName, "/branches/") { + case strings.Contains(resourceName, "/branches/"): resourceType = "Branch" } @@ -631,6 +634,67 @@ func (s *FakeWorkspace) createOperationLocked(resourceName string, response any) return op } +// PostgresSyncedTableCreate creates a new postgres synced table. +func (s *FakeWorkspace) PostgresSyncedTableCreate(req Request, syncedTableID string) Response { + defer s.LockUnlock()() + + if syncedTableID == "" { + return postgresErrorResponse(400, "INVALID_PARAMETER_VALUE", `Field 'synced_table_id' is required, expected non-default value (not "")!`) + } + + var table postgres.SyncedTable + if len(req.Body) > 0 { + if err := json.Unmarshal(req.Body, &table); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("cannot unmarshal request body: %v", err), + } + } + } + + name := "synced_tables/" + syncedTableID + + if _, exists := s.PostgresSyncedTables[name]; exists { + return postgresErrorResponse(409, "ALREADY_EXISTS", "synced table with such id already exists") + } + table.Name = name + table.Uid = nextUUID() + table.CreateTime = nowTime() + + // GET on the real API returns only status; clear spec to match. + table.Spec = nil + table.Status = &postgres.SyncedTableSyncedTableStatus{ + DetailedState: postgres.SyncedTableStateSyncedTableOnline, + UnityCatalogProvisioningState: postgres.ProvisioningInfoStateActive, + } + + s.PostgresSyncedTables[name] = table + + return Response{Body: s.createOperationLocked(name, table)} +} + +// PostgresSyncedTableGet retrieves a postgres synced table by name. +func (s *FakeWorkspace) PostgresSyncedTableGet(name string) Response { + defer s.LockUnlock()() + + table, exists := s.PostgresSyncedTables[name] + if !exists { + return postgresNotFoundResponse("synced table") + } + return Response{Body: table} +} + +// PostgresSyncedTableDelete deletes a postgres synced table. +func (s *FakeWorkspace) PostgresSyncedTableDelete(name string) Response { + defer s.LockUnlock()() + + if _, exists := s.PostgresSyncedTables[name]; !exists { + return postgresNotFoundResponse("synced table") + } + delete(s.PostgresSyncedTables, name) + return Response{Body: s.createOperationLocked(name, nil)} +} + // createDefaultBranchLocked creates a default branch for a project (caller must hold lock). // The default branch is named "production" to match cloud API behavior. func (s *FakeWorkspace) createDefaultBranchLocked(projectName string) { From f0036ce8df32c3a5aa395a5f5fc6c80e9e400d16 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 11:43:32 +0200 Subject: [PATCH 03/30] testserver: route postgres synced table endpoints --- libs/testserver/handlers.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index d98011fc7ba..79ce7671907 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -936,6 +936,24 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.PostgresEndpointDelete(name) }) + // Postgres Synced Tables: + server.Handle("POST", "/api/2.0/postgres/synced_tables", func(req Request) any { + syncedTableID := req.URL.Query().Get("synced_table_id") + return req.Workspace.PostgresSyncedTableCreate(req, syncedTableID) + }) + + server.Handle("GET", "/api/2.0/postgres/synced_tables/{id}", func(req Request) any { + return req.Workspace.PostgresSyncedTableGet("synced_tables/" + req.Vars["id"]) + }) + + server.Handle("DELETE", "/api/2.0/postgres/synced_tables/{id}", func(req Request) any { + return req.Workspace.PostgresSyncedTableDelete("synced_tables/" + req.Vars["id"]) + }) + + server.Handle("GET", "/api/2.0/postgres/operations/{operation_id}", func(req Request) any { + return req.Workspace.PostgresOperationGet("operations/" + req.Vars["operation_id"]) + }) + // Catch-all handler for invalid postgres resource names. // This handles cases like GET /api/2.0/postgres/1234 where "1234" is not a valid resource name. server.Handle("GET", "/api/2.0/postgres/{name}", func(req Request) any { From 721a5980fe8bbc9499a058b8a17c08ed6f3f8270 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 11:46:27 +0200 Subject: [PATCH 04/30] bundle: add PostgresSyncedTable config resource type --- .../config/resources/postgres_synced_table.go | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 bundle/config/resources/postgres_synced_table.go diff --git a/bundle/config/resources/postgres_synced_table.go b/bundle/config/resources/postgres_synced_table.go new file mode 100644 index 00000000000..fa4016611ca --- /dev/null +++ b/bundle/config/resources/postgres_synced_table.go @@ -0,0 +1,68 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/postgres" +) + +type PostgresSyncedTableConfig struct { + postgres.SyncedTableSyncedTableSpec + + // SyncedTableId is the user-specified three-part UC name (catalog.schema.table). + // Becomes the trailing component of the server-assigned Name: + // "synced_tables/{synced_table_id}". + SyncedTableId string `json:"synced_table_id"` +} + +func (c *PostgresSyncedTableConfig) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c *PostgresSyncedTableConfig) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +type PostgresSyncedTable struct { + BaseResource + PostgresSyncedTableConfig +} + +func (s *PostgresSyncedTable) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.Postgres.GetSyncedTable(ctx, postgres.GetSyncedTableRequest{Name: name}) + if err != nil { + log.Debugf(ctx, "postgres synced table %s does not exist", name) + return false, err + } + return true, nil +} + +func (s *PostgresSyncedTable) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "postgres_synced_table", + PluralName: "postgres_synced_tables", + SingularTitle: "Postgres synced table", + PluralTitle: "Postgres synced tables", + } +} + +func (s *PostgresSyncedTable) GetName() string { + // Synced tables don't expose a display name distinct from their three-part id. + return s.SyncedTableId +} + +func (s *PostgresSyncedTable) GetURL() string { + return s.URL +} + +func (s *PostgresSyncedTable) InitializeURL(baseURL url.URL) { + if s.SyncedTableId == "" { + return + } + baseURL.Path = "explore/data/" + s.SyncedTableId + s.URL = baseURL.String() +} From 3c1f4c84804b1ccc61ec61a9eca223c6e9d20006 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 11:48:45 +0200 Subject: [PATCH 05/30] dresources: add TrimSyncedTablesPrefix helper --- bundle/direct/dresources/util.go | 7 +++++++ bundle/direct/dresources/util_test.go | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/bundle/direct/dresources/util.go b/bundle/direct/dresources/util.go index 3bd0ab4ec73..60536c9b389 100644 --- a/bundle/direct/dresources/util.go +++ b/bundle/direct/dresources/util.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "regexp" + "strings" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" @@ -73,3 +74,9 @@ func truncateAtIndex(path string) string { } return p.Prefix(1).String() } + +// TrimSyncedTablesPrefix extracts the user-facing synced table id from the API name. +// E.g. "synced_tables/main.public.trips" -> "main.public.trips". +func TrimSyncedTablesPrefix(name string) string { + return strings.TrimPrefix(name, "synced_tables/") +} diff --git a/bundle/direct/dresources/util_test.go b/bundle/direct/dresources/util_test.go index bbf04717099..4b601550409 100644 --- a/bundle/direct/dresources/util_test.go +++ b/bundle/direct/dresources/util_test.go @@ -120,3 +120,22 @@ func TestParsePostgresName(t *testing.T) { }) } } + +func TestTrimSyncedTablesPrefix(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"happy path", "synced_tables/main.public.trips_synced", "main.public.trips_synced"}, + {"missing prefix is returned unchanged", "main.public.trips_synced", "main.public.trips_synced"}, + {"empty string", "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := TrimSyncedTablesPrefix(tt.in); got != tt.want { + t.Errorf("TrimSyncedTablesPrefix(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} From 9037247a46acec443565eec043517d6f236ae143 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 11:50:19 +0200 Subject: [PATCH 06/30] dresources: implement postgres synced table handler Co-authored-by: Isaac --- .../dresources/postgres_synced_table.go | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 bundle/direct/dresources/postgres_synced_table.go diff --git a/bundle/direct/dresources/postgres_synced_table.go b/bundle/direct/dresources/postgres_synced_table.go new file mode 100644 index 00000000000..3006f47282b --- /dev/null +++ b/bundle/direct/dresources/postgres_synced_table.go @@ -0,0 +1,87 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/postgres" +) + +type ResourcePostgresSyncedTable struct { + client *databricks.WorkspaceClient +} + +type PostgresSyncedTableState = resources.PostgresSyncedTableConfig + +func (*ResourcePostgresSyncedTable) New(client *databricks.WorkspaceClient) *ResourcePostgresSyncedTable { + return &ResourcePostgresSyncedTable{client: client} +} + +func (*ResourcePostgresSyncedTable) PrepareState(input *resources.PostgresSyncedTable) *PostgresSyncedTableState { + return &PostgresSyncedTableState{ + SyncedTableId: input.SyncedTableId, + SyncedTableSyncedTableSpec: input.SyncedTableSyncedTableSpec, + } +} + +func (*ResourcePostgresSyncedTable) RemapState(remote *postgres.SyncedTable) *PostgresSyncedTableState { + return &PostgresSyncedTableState{ + SyncedTableId: TrimSyncedTablesPrefix(remote.Name), + + // GET does not return the spec, only the status. Match the postgres_project / + // postgres_branch pattern: return an empty (non-nil) spec so field-level diffing + // works correctly and remote drift on spec fields is invisible. + SyncedTableSyncedTableSpec: postgres.SyncedTableSyncedTableSpec{ + Branch: "", + CreateDatabaseObjectsIfMissing: false, + ExistingPipelineId: "", + NewPipelineSpec: nil, + PostgresDatabase: "", + PrimaryKeyColumns: nil, + SchedulingPolicy: "", + SourceTableFullName: "", + TimeseriesKey: "", + ForceSendFields: nil, + }, + } +} + +func (r *ResourcePostgresSyncedTable) DoRead(ctx context.Context, id string) (*postgres.SyncedTable, error) { + return r.client.Postgres.GetSyncedTable(ctx, postgres.GetSyncedTableRequest{Name: id}) +} + +func (r *ResourcePostgresSyncedTable) DoCreate(ctx context.Context, config *PostgresSyncedTableState) (string, *postgres.SyncedTable, error) { + waiter, err := r.client.Postgres.CreateSyncedTable(ctx, postgres.CreateSyncedTableRequest{ + SyncedTableId: config.SyncedTableId, + SyncedTable: postgres.SyncedTable{ + Spec: &config.SyncedTableSyncedTableSpec, + + // Output-only fields. + CreateTime: nil, + Name: "", + Status: nil, + Uid: "", + ForceSendFields: nil, + }, + }) + if err != nil { + return "", nil, err + } + + result, err := waiter.Wait(ctx) + if err != nil { + return "", nil, err + } + return result.Name, result, nil +} + +func (r *ResourcePostgresSyncedTable) DoDelete(ctx context.Context, id string) error { + waiter, err := r.client.Postgres.DeleteSyncedTable(ctx, postgres.DeleteSyncedTableRequest{ + Name: id, + }) + if err != nil { + return err + } + return waiter.Wait(ctx) +} From 53029c9e1859de47fa56925f8742a4b8503c58c3 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 11:53:05 +0200 Subject: [PATCH 07/30] bundle: register postgres_synced_tables resource --- bundle/config/resources.go | 3 +++ bundle/direct/dresources/all.go | 1 + 2 files changed, 4 insertions(+) diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 225ec32165d..a6ec734b680 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -35,6 +35,7 @@ type Resources struct { PostgresProjects map[string]*resources.PostgresProject `json:"postgres_projects,omitempty"` PostgresBranches map[string]*resources.PostgresBranch `json:"postgres_branches,omitempty"` PostgresEndpoints map[string]*resources.PostgresEndpoint `json:"postgres_endpoints,omitempty"` + PostgresSyncedTables map[string]*resources.PostgresSyncedTable `json:"postgres_synced_tables,omitempty"` VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` } @@ -112,6 +113,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["postgres_projects"], r.PostgresProjects), collectResourceMap(descriptions["postgres_branches"], r.PostgresBranches), collectResourceMap(descriptions["postgres_endpoints"], r.PostgresEndpoints), + collectResourceMap(descriptions["postgres_synced_tables"], r.PostgresSyncedTables), collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), } } @@ -167,6 +169,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_projects": (&resources.PostgresProject{}).ResourceDescription(), "postgres_branches": (&resources.PostgresBranch{}).ResourceDescription(), "postgres_endpoints": (&resources.PostgresEndpoint{}).ResourceDescription(), + "postgres_synced_tables": (&resources.PostgresSyncedTable{}).ResourceDescription(), "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), } } diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index ddc30c41f54..bf70d5ae733 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -23,6 +23,7 @@ var SupportedResources = map[string]any{ "postgres_projects": (*ResourcePostgresProject)(nil), "postgres_branches": (*ResourcePostgresBranch)(nil), "postgres_endpoints": (*ResourcePostgresEndpoint)(nil), + "postgres_synced_tables": (*ResourcePostgresSyncedTable)(nil), "alerts": (*ResourceAlert)(nil), "clusters": (*ResourceCluster)(nil), "registered_models": (*ResourceRegisteredModel)(nil), From f39bd3c6a30d9bdd1c20e9e418ac5235b0071418 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 11:54:10 +0200 Subject: [PATCH 08/30] dresources: declare recreate_on_changes for postgres synced tables --- bundle/direct/dresources/resources.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 569fca9ee82..6ffbc8454d9 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -508,6 +508,32 @@ resources: - field: endpoint_id reason: immutable + postgres_synced_tables: + recreate_on_changes: + # synced_table_id is part of the hierarchical name and immutable. + - field: synced_table_id + reason: immutable + # The Postgres SDK has no UpdateSyncedTable endpoint, so every spec + # change requires delete+create. + - field: branch + reason: immutable + - field: postgres_database + reason: immutable + - field: source_table_full_name + reason: immutable + - field: primary_key_columns + reason: immutable + - field: timeseries_key + reason: immutable + - field: scheduling_policy + reason: immutable + - field: create_database_objects_if_missing + reason: immutable + - field: new_pipeline_spec + reason: immutable + - field: existing_pipeline_id + reason: immutable + vector_search_endpoints: recreate_on_changes: - field: endpoint_type From 5ccf8d103d7d8ddc1e97124b8fb7c9b0e7c27929 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 12:04:26 +0200 Subject: [PATCH 09/30] dresources: add postgres synced table smoke test Register postgres_synced_tables in apitypes.yml, add a testserver route for the operation-polling URL, and wire up the TestAll fixture entry. Co-authored-by: Isaac --- bundle/direct/dresources/all_test.go | 6 ++++++ bundle/direct/dresources/apitypes.yml | 2 ++ libs/testserver/handlers.go | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 0b0e273c1c8..e69db47652e 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -191,6 +191,12 @@ var testConfig map[string]any = map[string]any{ }, }, + "postgres_synced_tables": &resources.PostgresSyncedTable{ + PostgresSyncedTableConfig: resources.PostgresSyncedTableConfig{ + SyncedTableId: "main.public.trips_synced", + }, + }, + "alerts": &resources.Alert{ AlertV2: sql.AlertV2{ DisplayName: "my-alert", diff --git a/bundle/direct/dresources/apitypes.yml b/bundle/direct/dresources/apitypes.yml index 29db9b67b20..c70451d220b 100644 --- a/bundle/direct/dresources/apitypes.yml +++ b/bundle/direct/dresources/apitypes.yml @@ -9,3 +9,5 @@ postgres_branches: postgres.BranchSpec postgres_endpoints: postgres.EndpointSpec postgres_projects: postgres.ProjectSpec + +postgres_synced_tables: postgres.SyncedTableSyncedTableSpec diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 79ce7671907..c0fbe060e95 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -950,6 +950,11 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.PostgresSyncedTableDelete("synced_tables/" + req.Vars["id"]) }) + server.Handle("GET", "/api/2.0/postgres/synced_tables/{id}/operations/{operation_id}", func(req Request) any { + name := "synced_tables/" + req.Vars["id"] + "/operations/" + req.Vars["operation_id"] + return req.Workspace.PostgresOperationGet(name) + }) + server.Handle("GET", "/api/2.0/postgres/operations/{operation_id}", func(req Request) any { return req.Workspace.PostgresOperationGet("operations/" + req.Vars["operation_id"]) }) From bb875451bb785dbd36b8ef5e1daa1879530841f1 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 12:06:43 +0200 Subject: [PATCH 10/30] schema: document postgres_synced_tables --- bundle/internal/schema/annotations.yml | 37 ++++++++++ bundle/schema/jsonschema.json | 93 ++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index f6ac5c45d4d..5a4fb84b00b 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -221,6 +221,9 @@ github.com/databricks/cli/bundle/config.Resources: "postgres_projects": "description": |- PLACEHOLDER + "postgres_synced_tables": + "description": |- + PLACEHOLDER "quality_monitors": "description": |- The quality monitor definitions for the bundle, where each key is the name of the quality monitor. @@ -897,6 +900,40 @@ github.com/databricks/cli/bundle/config/resources.PostgresProject: "update_time": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.PostgresSyncedTable: + "branch": + "description": |- + PLACEHOLDER + "create_database_objects_if_missing": + "description": |- + PLACEHOLDER + "existing_pipeline_id": + "description": |- + PLACEHOLDER + "lifecycle": + "description": |- + PLACEHOLDER + "new_pipeline_spec": + "description": |- + PLACEHOLDER + "postgres_database": + "description": |- + PLACEHOLDER + "primary_key_columns": + "description": |- + PLACEHOLDER + "scheduling_policy": + "description": |- + PLACEHOLDER + "source_table_full_name": + "description": |- + PLACEHOLDER + "synced_table_id": + "description": |- + PLACEHOLDER + "timeseries_key": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.SecretScope: "backend_type": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 26e773f64a4..6860026a25f 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1511,6 +1511,56 @@ } ] }, + "resources.PostgresSyncedTable": { + "oneOf": [ + { + "type": "object", + "properties": { + "branch": { + "$ref": "#/$defs/string" + }, + "create_database_objects_if_missing": { + "$ref": "#/$defs/bool" + }, + "existing_pipeline_id": { + "$ref": "#/$defs/string" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "new_pipeline_spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/postgres.NewPipelineSpec" + }, + "postgres_database": { + "$ref": "#/$defs/string" + }, + "primary_key_columns": { + "$ref": "#/$defs/slice/string" + }, + "scheduling_policy": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/postgres.SyncedTableSyncedTableSpecSyncedTableSchedulingPolicy" + }, + "source_table_full_name": { + "$ref": "#/$defs/string" + }, + "synced_table_id": { + "$ref": "#/$defs/string" + }, + "timeseries_key": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "synced_table_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.QualityMonitor": { "oneOf": [ { @@ -2528,6 +2578,9 @@ "postgres_projects": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresProject" }, + "postgres_synced_tables": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresSyncedTable" + }, "quality_monitors": { "description": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.QualityMonitor", @@ -9820,6 +9873,29 @@ } ] }, + "postgres.NewPipelineSpec": { + "oneOf": [ + { + "type": "object", + "properties": { + "budget_policy_id": { + "$ref": "#/$defs/string" + }, + "storage_catalog": { + "$ref": "#/$defs/string" + }, + "storage_schema": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "postgres.ProjectCustomTag": { "oneOf": [ { @@ -9877,6 +9953,9 @@ } ] }, + "postgres.SyncedTableSyncedTableSpecSyncedTableSchedulingPolicy": { + "type": "string" + }, "serving.Ai21LabsConfig": { "oneOf": [ { @@ -11772,6 +11851,20 @@ } ] }, + "resources.PostgresSyncedTable": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresSyncedTable" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.QualityMonitor": { "oneOf": [ { From 5d6d658f176e4732c029171efe778aff3444c328 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 12:09:06 +0200 Subject: [PATCH 11/30] acc: postgres synced table basic deploy Co-authored-by: Isaac --- .../basic/databricks.yml.tmpl | 19 ++++++ .../basic/out.requests.json | 27 ++++++++ .../basic/out.test.toml | 4 ++ .../postgres_synced_tables/basic/output.txt | 61 +++++++++++++++++++ .../postgres_synced_tables/basic/script | 20 ++++++ .../postgres_synced_tables/basic/test.toml | 1 + .../postgres_synced_tables/test.toml | 16 +++++ 7 files changed, 148 insertions(+) create mode 100644 acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.json create mode 100644 acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_synced_tables/basic/output.txt create mode 100644 acceptance/bundle/resources/postgres_synced_tables/basic/script create mode 100644 acceptance/bundle/resources/postgres_synced_tables/basic/test.toml create mode 100644 acceptance/bundle/resources/postgres_synced_tables/test.toml diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl new file mode 100644 index 00000000000..8988ea17723 --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: deploy-postgres-synced-table-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_synced_tables: + my_table: + synced_table_id: "main.public.trips_$UNIQUE_NAME" + source_table_full_name: "main.raw.trips" + primary_key_columns: ["id"] + scheduling_policy: SNAPSHOT + postgres_database: appdb + branch: "projects/p/branches/production" + create_database_objects_if_missing: true + new_pipeline_spec: + storage_catalog: main + storage_schema: pipelines diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.json b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.json new file mode 100644 index 00000000000..1d68ff53ee3 --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.json @@ -0,0 +1,27 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/synced_tables", + "q": { + "synced_table_id": "main.public.trips_[UNIQUE_NAME]" + }, + "body": { + "spec": { + "branch": "projects/p/branches/production", + "create_database_objects_if_missing": true, + "new_pipeline_spec": { + "storage_catalog": "main", + "storage_schema": "pipelines" + }, + "postgres_database": "appdb", + "primary_key_columns": [ + "id" + ], + "scheduling_policy": "SNAPSHOT", + "source_table_full_name": "main.raw.trips" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/synced_tables/main.public.trips_[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml new file mode 100644 index 00000000000..88423408186 --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt b/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt new file mode 100644 index 00000000000..34186386441 --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt @@ -0,0 +1,61 @@ + +>>> [CLI] bundle validate +Name: deploy-postgres-synced-table-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-synced-table-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle summary +Name: deploy-postgres-synced-table-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-synced-table-[UNIQUE_NAME]/default +Resources: + Postgres synced tables: + my_table: + Name: main.public.trips_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/main.public.trips_[UNIQUE_NAME]?o=[NUMID] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-synced-table-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] postgres get-synced-table synced_tables/main.public.trips_[UNIQUE_NAME] +{ + "create_time": "[TIMESTAMP]", + "name": "synced_tables/main.public.trips_[UNIQUE_NAME]", + "status": { + "detailed_state": "SYNCED_TABLE_ONLINE", + "unity_catalog_provisioning_state": "ACTIVE" + }, + "uid": "[UUID]" +} + +>>> [CLI] bundle summary +Name: deploy-postgres-synced-table-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-synced-table-[UNIQUE_NAME]/default +Resources: + Postgres synced tables: + my_table: + Name: main.public.trips_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/main.public.trips_[UNIQUE_NAME]?o=[NUMID] + +>>> print_requests.py --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_synced_tables.my_table + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-synced-table-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/script b/acceptance/bundle/resources/postgres_synced_tables/basic/script new file mode 100644 index 00000000000..db3c74cca40 --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle validate + +trace $CLI bundle summary + +rm -f out.requests.txt +trace $CLI bundle deploy + +trace $CLI postgres get-synced-table "synced_tables/main.public.trips_${UNIQUE_NAME}" + +trace $CLI bundle summary + +trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/test.toml b/acceptance/bundle/resources/postgres_synced_tables/basic/test.toml new file mode 100644 index 00000000000..f8b3bbe49dd --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/test.toml @@ -0,0 +1 @@ +# All configuration inherited from parent test.toml diff --git a/acceptance/bundle/resources/postgres_synced_tables/test.toml b/acceptance/bundle/resources/postgres_synced_tables/test.toml new file mode 100644 index 00000000000..9a86768686e --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/test.toml @@ -0,0 +1,16 @@ +Local = true +Cloud = false +RequiresUnityCatalog = true + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", + ".databricks", +] + +[[Repls]] +# Normalize postgres operation IDs (unique per operation). +Old = '/operations/[A-Za-z0-9+/=-]+' +New = '/operations/[OPERATION_ID]' +Order = 2000 From de011f9651e87ed49deca833f78449d7ff540dd6 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 12:11:59 +0200 Subject: [PATCH 12/30] acc: postgres synced table recreate on spec change Co-authored-by: Isaac --- .../recreate/databricks.yml.tmpl | 19 ++++++++++++++ .../recreate/out.test.toml | 4 +++ .../recreate/output.txt | 26 +++++++++++++++++++ .../postgres_synced_tables/recreate/script | 16 ++++++++++++ .../postgres_synced_tables/recreate/test.toml | 1 + 5 files changed, 66 insertions(+) create mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt create mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate/script create mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate/test.toml diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..cf58da9a150 --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: recreate-postgres-synced-table-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_synced_tables: + my_table: + synced_table_id: "main.public.trips_$UNIQUE_NAME" + source_table_full_name: "main.raw.trips" + primary_key_columns: ["id"] + scheduling_policy: $POLICY + postgres_database: appdb + branch: "projects/p/branches/production" + create_database_objects_if_missing: true + new_pipeline_spec: + storage_catalog: main + storage_schema: pipelines diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml new file mode 100644 index 00000000000..88423408186 --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt b/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt new file mode 100644 index 00000000000..39507cca630 --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt @@ -0,0 +1,26 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-synced-table-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +recreate postgres_synced_tables.my_table + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-synced-table-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_synced_tables.my_table + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-synced-table-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/script b/acceptance/bundle/resources/postgres_synced_tables/recreate/script new file mode 100644 index 00000000000..11e2adae5de --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/script @@ -0,0 +1,16 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +export POLICY=SNAPSHOT +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy + +# Toggle a recreate-on-change field; plan must show delete + create. +export POLICY=TRIGGERED +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle plan | contains.py "Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged" + +trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate/test.toml new file mode 100644 index 00000000000..f8b3bbe49dd --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/test.toml @@ -0,0 +1 @@ +# All configuration inherited from parent test.toml From 2adce5e05f6145cbf56d679ae5577285b36e6d9f Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 12:20:00 +0200 Subject: [PATCH 13/30] dresources: register postgres_synced_tables in knownMissingInRemoteType --- bundle/direct/dresources/type_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 88f246723bf..4231ae01bc2 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -75,6 +75,18 @@ var knownMissingInRemoteType = map[string][]string{ "pg_version", "project_id", }, + "postgres_synced_tables": { + "branch", + "create_database_objects_if_missing", + "existing_pipeline_id", + "new_pipeline_spec", + "postgres_database", + "primary_key_columns", + "scheduling_policy", + "source_table_full_name", + "synced_table_id", + "timeseries_key", + }, "vector_search_endpoints": { "target_qps", "usage_policy_id", From a3b137292100ea47ecf09086e878328defa2319b Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 12:22:09 +0200 Subject: [PATCH 14/30] statemgmt: cover postgres_synced_tables in state-load fixtures --- bundle/statemgmt/state_load_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 34c4fa4f5aa..5060de4b23c 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -49,6 +49,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.postgres_projects.test_postgres_project": {ID: "projects/test-project"}, "resources.postgres_branches.test_postgres_branch": {ID: "projects/test-project/branches/main"}, "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, + "resources.postgres_synced_tables.test_postgres_synced_table": {ID: "main.public.test_synced_table"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, } err := StateToBundle(t.Context(), state, &config) @@ -292,6 +293,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + PostgresSyncedTables: map[string]*resources.PostgresSyncedTable{ + "test_postgres_synced_table": { + PostgresSyncedTableConfig: resources.PostgresSyncedTableConfig{ + SyncedTableId: "main.public.test_synced_table", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "test_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -661,6 +669,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + PostgresSyncedTables: map[string]*resources.PostgresSyncedTable{ + "test_postgres_synced_table": { + PostgresSyncedTableConfig: resources.PostgresSyncedTableConfig{ + SyncedTableId: "main.public.test_synced_table", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "test_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ From 6bbe5acaa765a6b76389aaf4ddef6a66ba524fa9 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 12:23:13 +0200 Subject: [PATCH 15/30] terraform: mark postgres_synced_tables as direct-only in lifecycle test --- bundle/deploy/terraform/lifecycle_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 7f56248bb44..4c7f9f3593b 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -17,6 +17,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { ignoredResources := []string{ "catalogs", "external_locations", + "postgres_synced_tables", "vector_search_endpoints", } From f225351fe357060417b725cd0268b0c69cee6567 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 12:33:11 +0200 Subject: [PATCH 16/30] resourcemutator: declare postgres_synced_tables in permissions/run_as/bind/mock allow-lists Add postgres_synced_tables to the per-resource allow-lists and mock fixtures that enumerate all bundle resource types: - unsupportedResources in apply_bundle_permissions_test (no ACL API) - allResourceTypes expected list and allowList in run_as_test (no run_as concept) - mockBundle in apply_target_mode_test + notUserNamed carve-out - TestResourcesBindSupport fixture + GetSyncedTable mock expectation - Refresh acceptance/bundle/refschema/out.fields.txt snapshot Co-authored-by: Isaac --- acceptance/bundle/refschema/out.fields.txt | 57 +++++++++++++++++++ .../apply_bundle_permissions_test.go | 1 + .../resourcemutator/apply_target_mode_test.go | 8 +++ .../mutator/resourcemutator/run_as_test.go | 2 + bundle/config/resources_test.go | 8 +++ 5 files changed, 76 insertions(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 999296b2139..b4070540ae0 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2870,6 +2870,63 @@ resources.postgres_projects.*.permissions[*].group_name string ALL resources.postgres_projects.*.permissions[*].level iam.PermissionLevel ALL resources.postgres_projects.*.permissions[*].service_principal_name string ALL resources.postgres_projects.*.permissions[*].user_name string ALL +resources.postgres_synced_tables.*.branch string INPUT STATE +resources.postgres_synced_tables.*.create_database_objects_if_missing bool INPUT STATE +resources.postgres_synced_tables.*.create_time *time.Time REMOTE +resources.postgres_synced_tables.*.existing_pipeline_id string INPUT STATE +resources.postgres_synced_tables.*.id string INPUT +resources.postgres_synced_tables.*.lifecycle resources.Lifecycle INPUT +resources.postgres_synced_tables.*.lifecycle.prevent_destroy bool INPUT +resources.postgres_synced_tables.*.modified_status string INPUT +resources.postgres_synced_tables.*.name string REMOTE +resources.postgres_synced_tables.*.new_pipeline_spec *postgres.NewPipelineSpec INPUT STATE +resources.postgres_synced_tables.*.new_pipeline_spec.budget_policy_id string INPUT STATE +resources.postgres_synced_tables.*.new_pipeline_spec.storage_catalog string INPUT STATE +resources.postgres_synced_tables.*.new_pipeline_spec.storage_schema string INPUT STATE +resources.postgres_synced_tables.*.postgres_database string INPUT STATE +resources.postgres_synced_tables.*.primary_key_columns []string INPUT STATE +resources.postgres_synced_tables.*.primary_key_columns[*] string INPUT STATE +resources.postgres_synced_tables.*.scheduling_policy postgres.SyncedTableSyncedTableSpecSyncedTableSchedulingPolicy INPUT STATE +resources.postgres_synced_tables.*.source_table_full_name string INPUT STATE +resources.postgres_synced_tables.*.spec *postgres.SyncedTableSyncedTableSpec REMOTE +resources.postgres_synced_tables.*.spec.branch string REMOTE +resources.postgres_synced_tables.*.spec.create_database_objects_if_missing bool REMOTE +resources.postgres_synced_tables.*.spec.existing_pipeline_id string REMOTE +resources.postgres_synced_tables.*.spec.new_pipeline_spec *postgres.NewPipelineSpec REMOTE +resources.postgres_synced_tables.*.spec.new_pipeline_spec.budget_policy_id string REMOTE +resources.postgres_synced_tables.*.spec.new_pipeline_spec.storage_catalog string REMOTE +resources.postgres_synced_tables.*.spec.new_pipeline_spec.storage_schema string REMOTE +resources.postgres_synced_tables.*.spec.postgres_database string REMOTE +resources.postgres_synced_tables.*.spec.primary_key_columns []string REMOTE +resources.postgres_synced_tables.*.spec.primary_key_columns[*] string REMOTE +resources.postgres_synced_tables.*.spec.scheduling_policy postgres.SyncedTableSyncedTableSpecSyncedTableSchedulingPolicy REMOTE +resources.postgres_synced_tables.*.spec.source_table_full_name string REMOTE +resources.postgres_synced_tables.*.spec.timeseries_key string REMOTE +resources.postgres_synced_tables.*.status *postgres.SyncedTableSyncedTableStatus REMOTE +resources.postgres_synced_tables.*.status.detailed_state postgres.SyncedTableState REMOTE +resources.postgres_synced_tables.*.status.last_processed_commit_version int64 REMOTE +resources.postgres_synced_tables.*.status.last_sync *postgres.SyncedTablePosition REMOTE +resources.postgres_synced_tables.*.status.last_sync.delta_table_sync_info *postgres.DeltaTableSyncInfo REMOTE +resources.postgres_synced_tables.*.status.last_sync.delta_table_sync_info.delta_commit_time *time.Time REMOTE +resources.postgres_synced_tables.*.status.last_sync.delta_table_sync_info.delta_commit_version int64 REMOTE +resources.postgres_synced_tables.*.status.last_sync.sync_end_time *time.Time REMOTE +resources.postgres_synced_tables.*.status.last_sync.sync_start_time *time.Time REMOTE +resources.postgres_synced_tables.*.status.last_sync_time *time.Time REMOTE +resources.postgres_synced_tables.*.status.message string REMOTE +resources.postgres_synced_tables.*.status.ongoing_sync_progress *postgres.SyncedTablePipelineProgress REMOTE +resources.postgres_synced_tables.*.status.ongoing_sync_progress.estimated_completion_time_seconds float64 REMOTE +resources.postgres_synced_tables.*.status.ongoing_sync_progress.latest_version_currently_processing int64 REMOTE +resources.postgres_synced_tables.*.status.ongoing_sync_progress.sync_progress_completion float64 REMOTE +resources.postgres_synced_tables.*.status.ongoing_sync_progress.synced_row_count int64 REMOTE +resources.postgres_synced_tables.*.status.ongoing_sync_progress.total_row_count int64 REMOTE +resources.postgres_synced_tables.*.status.pipeline_id string REMOTE +resources.postgres_synced_tables.*.status.project string REMOTE +resources.postgres_synced_tables.*.status.provisioning_phase postgres.ProvisioningPhase REMOTE +resources.postgres_synced_tables.*.status.unity_catalog_provisioning_state postgres.ProvisioningInfoState REMOTE +resources.postgres_synced_tables.*.synced_table_id string INPUT STATE +resources.postgres_synced_tables.*.timeseries_key string INPUT STATE +resources.postgres_synced_tables.*.uid string REMOTE +resources.postgres_synced_tables.*.url string INPUT resources.quality_monitors.*.assets_dir string ALL resources.quality_monitors.*.baseline_table_name string ALL resources.quality_monitors.*.custom_metrics []catalog.MonitorMetric ALL diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index e472241f282..689639e8615 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -28,6 +28,7 @@ var unsupportedResources = []string{ "synced_database_tables", "postgres_branches", "postgres_endpoints", + "postgres_synced_tables", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index fe9c9a1db06..14cdfc06324 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -247,6 +247,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + PostgresSyncedTables: map[string]*resources.PostgresSyncedTable{ + "postgres_synced_table1": { + PostgresSyncedTableConfig: resources.PostgresSyncedTableConfig{ + SyncedTableId: "catalog.schema.table1", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "vs_endpoint1": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -441,6 +448,7 @@ func TestAppropriateResourcesAreRenamed(t *testing.T) { "PostgresProjects", "PostgresBranches", "PostgresEndpoints", + "PostgresSyncedTables", } diags := bundle.ApplySeq(t.Context(), b, ApplyTargetMode(), ApplyPresets()) diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 0b7003f5873..b3e1098608b 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -48,6 +48,7 @@ func allResourceTypes(t *testing.T) []string { "postgres_branches", "postgres_endpoints", "postgres_projects", + "postgres_synced_tables", "quality_monitors", "registered_models", "schemas", @@ -176,6 +177,7 @@ var allowList = []string{ "postgres_branches", "postgres_endpoints", "postgres_projects", + "postgres_synced_tables", "registered_models", "experiments", "schemas", diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 943b279a288..0e70ca5e8c3 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -273,6 +273,13 @@ func TestResourcesBindSupport(t *testing.T) { }, }, }, + PostgresSyncedTables: map[string]*resources.PostgresSyncedTable{ + "my_postgres_synced_table": { + PostgresSyncedTableConfig: resources.PostgresSyncedTableConfig{ + SyncedTableId: "catalog.schema.my_postgres_synced_table", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "my_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -312,6 +319,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPostgresAPI().EXPECT().GetProject(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetBranch(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockPostgresAPI().EXPECT().GetSyncedTable(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchEndpointsAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) allResources := supportedResources.AllResources() From 52913dc1a79b9d54452d3471cf46e8baefa6c89c Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 13:16:03 +0200 Subject: [PATCH 17/30] schema: write postgres_synced_tables annotation --- bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 5a4fb84b00b..ba20979a220 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -223,7 +223,7 @@ github.com/databricks/cli/bundle/config.Resources: PLACEHOLDER "postgres_synced_tables": "description": |- - PLACEHOLDER + The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. "quality_monitors": "description": |- The quality monitor definitions for the bundle, where each key is the name of the quality monitor. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 6860026a25f..4835cb53c4d 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2579,6 +2579,7 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresProject" }, "postgres_synced_tables": { + "description": "The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresSyncedTable" }, "quality_monitors": { From e388091b0927b530f1a235e3ab8601b250094380 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 13:17:51 +0200 Subject: [PATCH 18/30] statemgmt: use prefixed API name as postgres_synced_tables state ID --- bundle/statemgmt/state_load_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 5060de4b23c..48ea3e2d2c1 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -49,7 +49,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.postgres_projects.test_postgres_project": {ID: "projects/test-project"}, "resources.postgres_branches.test_postgres_branch": {ID: "projects/test-project/branches/main"}, "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, - "resources.postgres_synced_tables.test_postgres_synced_table": {ID: "main.public.test_synced_table"}, + "resources.postgres_synced_tables.test_postgres_synced_table": {ID: "synced_tables/main.public.test_synced_table"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, } err := StateToBundle(t.Context(), state, &config) From 5732819202a23d9fadf31e27fa0ec439d75db56b Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 13:29:08 +0200 Subject: [PATCH 19/30] changelog: postgres_synced_tables bundle resource --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 36cbb445d2c..772ae9837b5 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -8,5 +8,6 @@ ### Bundles * Make sure warnings asking for approval are understood by agents ([#5239](https://github.com/databricks/cli/pull/5239)) +* Add `postgres_synced_tables` resource to manage Lakebase synced tables that replicate Unity Catalog Delta tables into Postgres. Direct deployment engine only. ### Dependency updates From f26197d3513ecdf84b6585e5f218dd606fc892e0 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 15:41:44 +0200 Subject: [PATCH 20/30] dresources: fix forever-recreate for postgres_synced_tables Running ./task generate-direct-resources populates the missing ignore_remote_changes block for postgres_synced_tables. Every spec field is now marked spec:input_only, so the planner stops flagging the empty spec returned by GET as drift. The manual recreate_on_changes block in resources.yml is unchanged on purpose: it covers the intent side (a user editing databricks.yml must still trigger delete+create because no UpdateSyncedTable endpoint exists). Added a comment at the top of the block explaining how the two declarations cooperate. Same pattern as secret_scopes, which is the other no-Update resource. --- .../direct/dresources/apitypes.generated.yml | 2 ++ .../direct/dresources/resources.generated.yml | 22 +++++++++++++++++++ bundle/direct/dresources/resources.yml | 10 ++++++--- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index a80b3baa69b..9c031777ae9 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -32,6 +32,8 @@ postgres_endpoints: postgres.EndpointSpec postgres_projects: postgres.ProjectStatus +postgres_synced_tables: postgres.SyncedTableSyncedTableSpec + quality_monitors: catalog.CreateMonitor registered_models: catalog.RegisteredModelInfo diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index 85c15d6f343..881ba166e0b 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -256,6 +256,28 @@ resources: - field: pg_version reason: spec:input_only + postgres_synced_tables: + + ignore_remote_changes: + - field: branch + reason: spec:input_only + - field: create_database_objects_if_missing + reason: spec:input_only + - field: existing_pipeline_id + reason: spec:input_only + - field: new_pipeline_spec + reason: spec:input_only + - field: postgres_database + reason: spec:input_only + - field: primary_key_columns + reason: spec:input_only + - field: scheduling_policy + reason: spec:input_only + - field: source_table_full_name + reason: spec:input_only + - field: timeseries_key + reason: spec:input_only + # quality_monitors: no api field behaviors # registered_models: no api field behaviors diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 6ffbc8454d9..fbc2c2bb4a1 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -509,12 +509,16 @@ resources: reason: immutable postgres_synced_tables: + # The Postgres API has no UpdateSyncedTable endpoint, so every settable + # field is recreate-only on the intent side (local YAML edit -> delete + + # create). The complementary ignore_remote_changes block for this resource + # lives in resources.generated.yml and handles the read side: it suppresses + # drift for the same fields because the GET API does not echo back the + # spec. Together they make no-op deploys idempotent while a real config + # edit still triggers a recreate. Same pattern as secret_scopes. recreate_on_changes: - # synced_table_id is part of the hierarchical name and immutable. - field: synced_table_id reason: immutable - # The Postgres SDK has no UpdateSyncedTable endpoint, so every spec - # change requires delete+create. - field: branch reason: immutable - field: postgres_database From ca14073f81c3648069792ed995607747b1fd5aa8 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 21:51:07 +0200 Subject: [PATCH 21/30] acc: cover postgres_synced_tables with no_drift invariant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a no_drift invariant test config for postgres_synced_tables. This is the regression guard for the V12 forever-recreate bug — if RemapState ever drops the ignore_remote_changes coverage on a spec field, this test will catch the bug at CI time instead of customer deploy time. Excluded from the Cloud variant for the same reason as the other postgres_* configs: Lakebase Autoscaling is AWS-only and the production fixture used by the cloud variant doesn't have a Lakebase project bound to the test workspace. --- .../configs/postgres_synced_table.yml.tmpl | 16 ++++++++++++++++ .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 2 ++ 3 files changed, 19 insertions(+) create mode 100644 acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl diff --git a/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl new file mode 100644 index 00000000000..7a548a02756 --- /dev/null +++ b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + postgres_synced_tables: + foo: + synced_table_id: lakebase_$UNIQUE_NAME.public.trips_synced + source_table_full_name: main.raw.trips + primary_key_columns: [id] + scheduling_policy: SNAPSHOT + postgres_database: appdb + branch: projects/test-pg-project-$UNIQUE_NAME/branches/production + create_database_objects_if_missing: true + new_pipeline_spec: + storage_catalog: main + storage_schema: pipelines diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 11aaf584918..2a8ad58c2da 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -28,6 +28,7 @@ EnvMatrix.INPUT_CONFIG = [ "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", + "postgres_synced_table.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 257e33005a3..af079be460e 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -46,6 +46,7 @@ EnvMatrix.INPUT_CONFIG = [ "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", + "postgres_synced_table.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", @@ -67,6 +68,7 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] no_postgres_project_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_project.yml.tmpl"] no_postgres_branch_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_branch.yml.tmpl"] no_postgres_endpoint_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_endpoint.yml.tmpl"] +no_postgres_synced_table_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_synced_table.yml.tmpl"] # External locations require actual storage credentials with cloud IAM setup # which are environment-specific, so we only test locally with the mock server From 2beb3cb07c54536dcc2b44a3d200c66eb00a5896 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 20:55:15 +0200 Subject: [PATCH 22/30] Add postgres_catalogs bundle resource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes New `postgres_catalogs` resource binding a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. Supported on both direct and terraform deployment engines. The spec fields are classified as both `recreate_on_changes` and `ignore_remote_changes: input_only`. The two cover orthogonal diffs the planner runs — recreate fires on local edits to an immutable field, and ignore_remote silences the phantom drift from GET not echoing spec back today. Lift the `input_only` entries once the backend starts returning spec. ## Tests Acceptance coverage: `basic` and `recreate` exercise each engine, plus the existing `no_drift` and `migrate` invariants pick up the new resource. Both engines produce identical human-readable output and identical wire bodies; only the captured request streams diverge by filename (`out.requests.{direct,terraform}.json`). Verified end to end on a live workspace: the bundle deploys a project and catalog, a row written directly into the bound Postgres database becomes visible through the UC federated view, and a follow-up write shows up on re-read. _This PR was written by Claude Code._ --- .agent/skills/pr-checklist/SKILL.md | 3 +- NEXT_CHANGELOG.md | 1 + .../configs/postgres_catalog.yml.tmpl | 15 +++ .../invariant/continue_293/out.test.toml | 1 + .../bundle/invariant/migrate/out.test.toml | 1 + .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 2 + acceptance/bundle/refschema/out.fields.txt | 22 +++ .../basic/databricks.yml.tmpl | 19 +++ .../basic/out.requests.direct.json | 31 +++++ .../basic/out.requests.terraform.json | 31 +++++ .../postgres_catalogs/basic/out.test.toml | 6 + .../postgres_catalogs/basic/output.txt | 74 +++++++++++ .../resources/postgres_catalogs/basic/script | 23 ++++ .../recreate/databricks.yml.tmpl | 19 +++ .../postgres_catalogs/recreate/out.test.toml | 6 + .../postgres_catalogs/recreate/output.txt | 31 +++++ .../postgres_catalogs/recreate/script | 16 +++ .../resources/postgres_catalogs/test.toml | 26 ++++ .../apply_bundle_permissions_test.go | 1 + .../resourcemutator/apply_target_mode_test.go | 8 ++ .../mutator/resourcemutator/run_as_test.go | 2 + bundle/config/resources.go | 3 + bundle/config/resources/postgres_catalog.go | 66 +++++++++ bundle/config/resources_test.go | 8 ++ bundle/deploy/terraform/interpolate.go | 2 +- bundle/deploy/terraform/pkg.go | 1 + .../tfdyn/convert_postgres_catalog.go | 56 ++++++++ .../tfdyn/convert_postgres_catalog_test.go | 73 ++++++++++ bundle/deploy/terraform/util.go | 2 +- bundle/direct/dresources/all.go | 1 + bundle/direct/dresources/all_test.go | 6 + .../direct/dresources/apitypes.generated.yml | 2 + bundle/direct/dresources/apitypes.yml | 2 + bundle/direct/dresources/postgres_catalog.go | 86 ++++++++++++ .../direct/dresources/resources.generated.yml | 14 ++ bundle/direct/dresources/resources.yml | 31 +++++ bundle/direct/dresources/type_test.go | 6 + bundle/internal/schema/annotations.yml | 19 +++ .../validation/generated/required_fields.go | 2 + bundle/schema/jsonschema.json | 51 +++++++ bundle/schema/jsonschema_for_docs.json | 125 ++++++++++++++---- bundle/statemgmt/state_load_test.go | 35 +++++ libs/testserver/fake_workspace.go | 2 + libs/testserver/handlers.go | 21 +++ libs/testserver/postgres.go | 84 +++++++++++- 46 files changed, 1003 insertions(+), 34 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/postgres_catalog.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/out.requests.direct.json create mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json create mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/output.txt create mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/script create mode 100644 acceptance/bundle/resources/postgres_catalogs/recreate/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_catalogs/recreate/output.txt create mode 100644 acceptance/bundle/resources/postgres_catalogs/recreate/script create mode 100644 acceptance/bundle/resources/postgres_catalogs/test.toml create mode 100644 bundle/config/resources/postgres_catalog.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_postgres_catalog.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_postgres_catalog_test.go create mode 100644 bundle/direct/dresources/postgres_catalog.go diff --git a/.agent/skills/pr-checklist/SKILL.md b/.agent/skills/pr-checklist/SKILL.md index 86559354fcb..e01058a0120 100644 --- a/.agent/skills/pr-checklist/SKILL.md +++ b/.agent/skills/pr-checklist/SKILL.md @@ -16,8 +16,9 @@ Before submitting a PR, run these commands to match what CI checks. CI uses the # 3. Tests (CI runs with both deployment engines) ./task test -# 4. If you changed bundle config structs or schema-related code: +# 4. If you changed bundle config structs, schema, or direct-engine resource code: ./task generate-schema +./task generate-direct # 5. If you changed files in python/: ./task pydabs-codegen pydabs-test pydabs-lint pydabs-docs diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index fa2adeb1e8a..a92ec7b7914 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,5 +10,6 @@ ### Bundles * Make sure warnings asking for approval are understood by agents ([#5239](https://github.com/databricks/cli/pull/5239)) +* Add `postgres_catalogs` resource to bind a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. ### Dependency updates diff --git a/acceptance/bundle/invariant/configs/postgres_catalog.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_catalog.yml.tmpl new file mode 100644 index 00000000000..2ae10852504 --- /dev/null +++ b/acceptance/bundle/invariant/configs/postgres_catalog.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + postgres_projects: + project: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + + postgres_catalogs: + catalog: + catalog_id: test_pg_catalog_$UNIQUE_NAME + branch: ${resources.postgres_projects.project.name}/branches/production + postgres_database: appdb + create_database_if_missing: true diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 11aaf584918..3bff3e2836d 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline.yml.tmpl", "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 11aaf584918..3bff3e2836d 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline.yml.tmpl", "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 11aaf584918..3bff3e2836d 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline.yml.tmpl", "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 257e33005a3..abfb10c5f73 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -44,6 +44,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline.yml.tmpl", "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", @@ -67,6 +68,7 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] no_postgres_project_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_project.yml.tmpl"] no_postgres_branch_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_branch.yml.tmpl"] no_postgres_endpoint_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_endpoint.yml.tmpl"] +no_postgres_catalog_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_catalog.yml.tmpl"] # External locations require actual storage credentials with cloud IAM setup # which are environment-specific, so we only test locally with the mock server diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 999296b2139..2c3ad788ea3 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2733,6 +2733,28 @@ resources.postgres_branches.*.ttl *duration.Duration INPUT STATE resources.postgres_branches.*.uid string REMOTE resources.postgres_branches.*.update_time *time.Time REMOTE resources.postgres_branches.*.url string INPUT +resources.postgres_catalogs.*.branch string INPUT STATE +resources.postgres_catalogs.*.catalog_id string INPUT STATE +resources.postgres_catalogs.*.create_database_if_missing bool INPUT STATE +resources.postgres_catalogs.*.create_time *time.Time REMOTE +resources.postgres_catalogs.*.id string INPUT +resources.postgres_catalogs.*.lifecycle resources.Lifecycle INPUT +resources.postgres_catalogs.*.lifecycle.prevent_destroy bool INPUT +resources.postgres_catalogs.*.modified_status string INPUT +resources.postgres_catalogs.*.name string REMOTE +resources.postgres_catalogs.*.postgres_database string INPUT STATE +resources.postgres_catalogs.*.spec *postgres.CatalogCatalogSpec REMOTE +resources.postgres_catalogs.*.spec.branch string REMOTE +resources.postgres_catalogs.*.spec.create_database_if_missing bool REMOTE +resources.postgres_catalogs.*.spec.postgres_database string REMOTE +resources.postgres_catalogs.*.status *postgres.CatalogCatalogStatus REMOTE +resources.postgres_catalogs.*.status.branch string REMOTE +resources.postgres_catalogs.*.status.catalog_id string REMOTE +resources.postgres_catalogs.*.status.postgres_database string REMOTE +resources.postgres_catalogs.*.status.project string REMOTE +resources.postgres_catalogs.*.uid string REMOTE +resources.postgres_catalogs.*.update_time *time.Time REMOTE +resources.postgres_catalogs.*.url string INPUT resources.postgres_endpoints.*.autoscaling_limit_max_cu float64 INPUT STATE resources.postgres_endpoints.*.autoscaling_limit_min_cu float64 INPUT STATE resources.postgres_endpoints.*.create_time *time.Time REMOTE diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_catalogs/basic/databricks.yml.tmpl new file mode 100644 index 00000000000..b339f4c1354 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: deploy-postgres-catalog-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Catalog" + pg_version: 16 + + postgres_catalogs: + my_catalog: + catalog_id: lakebase_test_$UNIQUE_NAME + branch: ${resources.postgres_projects.my_project.id}/branches/production + postgres_database: appdb + create_database_if_missing: true diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.direct.json b/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.direct.json new file mode 100644 index 00000000000..891733f4150 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.direct.json @@ -0,0 +1,31 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Test Project for Catalog", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/catalogs", + "q": { + "catalog_id": "lakebase_test_[UNIQUE_NAME]" + }, + "body": { + "spec": { + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "create_database_if_missing": true, + "postgres_database": "appdb" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/catalogs/lakebase_test_[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json b/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json new file mode 100644 index 00000000000..891733f4150 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json @@ -0,0 +1,31 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Test Project for Catalog", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/catalogs", + "q": { + "catalog_id": "lakebase_test_[UNIQUE_NAME]" + }, + "body": { + "spec": { + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "create_database_if_missing": true, + "postgres_database": "appdb" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/catalogs/lakebase_test_[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml new file mode 100644 index 00000000000..110f841fa05 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/output.txt b/acceptance/bundle/resources/postgres_catalogs/basic/output.txt new file mode 100644 index 00000000000..7bf45891cb3 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/output.txt @@ -0,0 +1,74 @@ + +>>> [CLI] bundle validate +Name: deploy-postgres-catalog-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle summary +Name: deploy-postgres-catalog-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default +Resources: + Postgres catalogs: + my_catalog: + Name: lakebase_test_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/lakebase_test_[UNIQUE_NAME] + Postgres projects: + my_project: + Name: Test Project for Catalog + URL: (not deployed) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] postgres get-catalog catalogs/lakebase_test_[UNIQUE_NAME] +{ + "name": "catalogs/lakebase_test_[UNIQUE_NAME]", + "status": { + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "catalog_id": "lakebase_test_[UNIQUE_NAME]", + "postgres_database": "appdb", + "project": "projects/test-pg-proj-[UNIQUE_NAME]" + } +} + +>>> [CLI] bundle summary +Name: deploy-postgres-catalog-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default +Resources: + Postgres catalogs: + my_catalog: + Name: lakebase_test_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/lakebase_test_[UNIQUE_NAME] + Postgres projects: + my_project: + Name: Test Project for Catalog + URL: (not deployed) + +>>> print_requests.py --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_catalogs.my_catalog + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/script b/acceptance/bundle/resources/postgres_catalogs/basic/script new file mode 100644 index 00000000000..ecf81072cfb --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/script @@ -0,0 +1,23 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle validate + +trace $CLI bundle summary + +rm -f out.requests.txt +trace $CLI bundle deploy + +# Get catalog details. Hide volatile fields so cloud and local match. +catalog_name="catalogs/lakebase_test_${UNIQUE_NAME}" +trace $CLI postgres get-catalog "${catalog_name}" | jq 'del(.create_time, .update_time, .uid)' + +trace $CLI bundle summary + +# Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling). +trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_catalogs/recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..bf3c2b56576 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: recreate-postgres-catalog-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Catalog Recreate" + pg_version: 16 + + postgres_catalogs: + my_catalog: + catalog_id: lakebase_test_$UNIQUE_NAME + branch: ${resources.postgres_projects.my_project.id}/branches/production + postgres_database: $POSTGRES_DATABASE + create_database_if_missing: true diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml new file mode 100644 index 00000000000..110f841fa05 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/output.txt b/acceptance/bundle/resources/postgres_catalogs/recreate/output.txt new file mode 100644 index 00000000000..1f6bb621987 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/output.txt @@ -0,0 +1,31 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-catalog-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +recreate postgres_catalogs.my_catalog + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-catalog-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_catalogs.my_catalog + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-catalog-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/script b/acceptance/bundle/resources/postgres_catalogs/recreate/script new file mode 100644 index 00000000000..4d9de784299 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/script @@ -0,0 +1,16 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +export POSTGRES_DATABASE=appdb +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy + +# Toggle a recreate-on-change field; plan must show delete + create. +export POSTGRES_DATABASE=otherdb +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle plan | contains.py "Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged" + +trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_catalogs/test.toml b/acceptance/bundle/resources/postgres_catalogs/test.toml new file mode 100644 index 00000000000..66f0811b343 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/test.toml @@ -0,0 +1,26 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +# Lakebase v2 (postgres) is only available in AWS as of January 2026 +CloudEnvs.gcp = false +CloudEnvs.azure = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] + +Ignore = [ + "databricks.yml", + ".databricks", +] + +[[Repls]] +# Clean up ?o= suffix after URL since not all workspaces have that +Old = '\?o=\[(NUMID|ALPHANUMID)\]' +New = '' +Order = 1000 + +[[Repls]] +# Normalize postgres operation IDs (unique per operation). +Old = '/operations/[A-Za-z0-9+/=-]+' +New = '/operations/[OPERATION_ID]' +Order = 2000 diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index e472241f282..d78c1ad8145 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -28,6 +28,7 @@ var unsupportedResources = []string{ "synced_database_tables", "postgres_branches", "postgres_endpoints", + "postgres_catalogs", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index fe9c9a1db06..927c7a19132 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -247,6 +247,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + PostgresCatalogs: map[string]*resources.PostgresCatalog{ + "postgres_catalog1": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "postgres_catalog_1", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "vs_endpoint1": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -441,6 +448,7 @@ func TestAppropriateResourcesAreRenamed(t *testing.T) { "PostgresProjects", "PostgresBranches", "PostgresEndpoints", + "PostgresCatalogs", } diags := bundle.ApplySeq(t.Context(), b, ApplyTargetMode(), ApplyPresets()) diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 0b7003f5873..58b27113a52 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -46,6 +46,7 @@ func allResourceTypes(t *testing.T) []string { "models", "pipelines", "postgres_branches", + "postgres_catalogs", "postgres_endpoints", "postgres_projects", "quality_monitors", @@ -174,6 +175,7 @@ var allowList = []string{ "pipelines", "models", "postgres_branches", + "postgres_catalogs", "postgres_endpoints", "postgres_projects", "registered_models", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 225ec32165d..30a7f4e95e5 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -35,6 +35,7 @@ type Resources struct { PostgresProjects map[string]*resources.PostgresProject `json:"postgres_projects,omitempty"` PostgresBranches map[string]*resources.PostgresBranch `json:"postgres_branches,omitempty"` PostgresEndpoints map[string]*resources.PostgresEndpoint `json:"postgres_endpoints,omitempty"` + PostgresCatalogs map[string]*resources.PostgresCatalog `json:"postgres_catalogs,omitempty"` VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` } @@ -112,6 +113,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["postgres_projects"], r.PostgresProjects), collectResourceMap(descriptions["postgres_branches"], r.PostgresBranches), collectResourceMap(descriptions["postgres_endpoints"], r.PostgresEndpoints), + collectResourceMap(descriptions["postgres_catalogs"], r.PostgresCatalogs), collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), } } @@ -167,6 +169,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_projects": (&resources.PostgresProject{}).ResourceDescription(), "postgres_branches": (&resources.PostgresBranch{}).ResourceDescription(), "postgres_endpoints": (&resources.PostgresEndpoint{}).ResourceDescription(), + "postgres_catalogs": (&resources.PostgresCatalog{}).ResourceDescription(), "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), } } diff --git a/bundle/config/resources/postgres_catalog.go b/bundle/config/resources/postgres_catalog.go new file mode 100644 index 00000000000..c17ffaa1682 --- /dev/null +++ b/bundle/config/resources/postgres_catalog.go @@ -0,0 +1,66 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/postgres" +) + +type PostgresCatalogConfig struct { + postgres.CatalogCatalogSpec + + // CatalogId is the user-specified UC catalog name. Becomes the trailing + // component of the server-assigned Name: "catalogs/{catalog_id}". + CatalogId string `json:"catalog_id"` +} + +func (c *PostgresCatalogConfig) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c *PostgresCatalogConfig) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +type PostgresCatalog struct { + BaseResource + PostgresCatalogConfig +} + +func (c *PostgresCatalog) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.Postgres.GetCatalog(ctx, postgres.GetCatalogRequest{Name: name}) + if err != nil { + log.Debugf(ctx, "postgres catalog %s does not exist", name) + return false, err + } + return true, nil +} + +func (c *PostgresCatalog) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "postgres_catalog", + PluralName: "postgres_catalogs", + SingularTitle: "Postgres catalog", + PluralTitle: "Postgres catalogs", + } +} + +func (c *PostgresCatalog) GetName() string { + return c.CatalogId +} + +func (c *PostgresCatalog) GetURL() string { + return c.URL +} + +func (c *PostgresCatalog) InitializeURL(baseURL url.URL) { + if c.CatalogId == "" { + return + } + baseURL.Path = "explore/data/" + c.CatalogId + c.URL = baseURL.String() +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 943b279a288..626fac07983 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -273,6 +273,13 @@ func TestResourcesBindSupport(t *testing.T) { }, }, }, + PostgresCatalogs: map[string]*resources.PostgresCatalog{ + "my_postgres_catalog": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "my_postgres_catalog", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "my_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -312,6 +319,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPostgresAPI().EXPECT().GetProject(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetBranch(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockPostgresAPI().EXPECT().GetCatalog(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchEndpointsAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) allResources := supportedResources.AllResources() diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go index fdcb671bdd3..7a1acb3d4e6 100644 --- a/bundle/deploy/terraform/interpolate.go +++ b/bundle/deploy/terraform/interpolate.go @@ -16,7 +16,7 @@ type interpolateMutator struct{} // Postgres resources use "name" instead of "id" as their identifier attribute. func isPostgresResource(resourceType string) bool { switch resourceType { - case "postgres_projects", "postgres_branches", "postgres_endpoints": + case "postgres_projects", "postgres_branches", "postgres_endpoints", "postgres_catalogs": return true default: return false diff --git a/bundle/deploy/terraform/pkg.go b/bundle/deploy/terraform/pkg.go index a66e5cb6a06..ff68c1b1513 100644 --- a/bundle/deploy/terraform/pkg.go +++ b/bundle/deploy/terraform/pkg.go @@ -129,6 +129,7 @@ var GroupToTerraformName = map[string]string{ "postgres_projects": "databricks_postgres_project", "postgres_branches": "databricks_postgres_branch", "postgres_endpoints": "databricks_postgres_endpoint", + "postgres_catalogs": "databricks_postgres_catalog", // 3 level groups: resources.*.GROUP "permissions": "databricks_permissions", diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_catalog.go b/bundle/deploy/terraform/tfdyn/convert_postgres_catalog.go new file mode 100644 index 00000000000..b84fb09d717 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_catalog.go @@ -0,0 +1,56 @@ +package tfdyn + +import ( + "context" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +type postgresCatalogConverter struct{} + +func (c postgresCatalogConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + // The bundle config has flattened CatalogCatalogSpec fields at the top level. + // Terraform expects them nested in a "spec" block. + specFields := specFieldNames(schema.ResourcePostgresCatalogSpec{}) + topLevelFields := []string{"catalog_id"} + + specMap := make(map[string]dyn.Value) + for _, field := range specFields { + if v := vin.Get(field); v.Kind() != dyn.KindInvalid { + specMap[field] = v + } + } + + outMap := make(map[string]dyn.Value) + for _, field := range topLevelFields { + if v := vin.Get(field); v.Kind() != dyn.KindInvalid { + outMap[field] = v + } + } + if len(specMap) > 0 { + outMap["spec"] = dyn.V(specMap) + } + + vout := dyn.V(outMap) + + vout, diags := convert.Normalize(schema.ResourcePostgresCatalog{}, vout) + for _, diag := range diags { + log.Debugf(ctx, "postgres catalog normalization diagnostic: %s", diag.Summary) + } + + vout, err := convertLifecycle(ctx, vout, vin.Get("lifecycle")) + if err != nil { + return err + } + + out.PostgresCatalog[key] = vout.AsAny() + + return nil +} + +func init() { + registerConverter("postgres_catalogs", postgresCatalogConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_catalog_test.go b/bundle/deploy/terraform/tfdyn/convert_postgres_catalog_test.go new file mode 100644 index 00000000000..8eca766f613 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_catalog_test.go @@ -0,0 +1,73 @@ +package tfdyn + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertPostgresCatalog(t *testing.T) { + src := resources.PostgresCatalog{ + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "shop_lakebase", + CatalogCatalogSpec: postgres.CatalogCatalogSpec{ + Branch: "projects/my-shop/branches/production", + PostgresDatabase: "appdb", + CreateDatabaseIfMissing: true, + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := t.Context() + out := schema.NewResources() + err = postgresCatalogConverter{}.Convert(ctx, "my_postgres_catalog", vin, out) + require.NoError(t, err) + + postgresCatalog := out.PostgresCatalog["my_postgres_catalog"] + assert.Equal(t, map[string]any{ + "catalog_id": "shop_lakebase", + "spec": map[string]any{ + "branch": "projects/my-shop/branches/production", + "postgres_database": "appdb", + "create_database_if_missing": true, + }, + }, postgresCatalog) +} + +func TestConvertPostgresCatalogMinimal(t *testing.T) { + src := resources.PostgresCatalog{ + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "minimal_catalog", + CatalogCatalogSpec: postgres.CatalogCatalogSpec{ + Branch: "projects/my-shop/branches/production", + PostgresDatabase: "appdb", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := t.Context() + out := schema.NewResources() + err = postgresCatalogConverter{}.Convert(ctx, "minimal_postgres_catalog", vin, out) + require.NoError(t, err) + + postgresCatalog := out.PostgresCatalog["minimal_postgres_catalog"] + assert.Equal(t, map[string]any{ + "catalog_id": "minimal_catalog", + "spec": map[string]any{ + "branch": "projects/my-shop/branches/production", + "postgres_database": "appdb", + }, + }, postgresCatalog) +} diff --git a/bundle/deploy/terraform/util.go b/bundle/deploy/terraform/util.go index 632d32bca19..69c3c4f886d 100644 --- a/bundle/deploy/terraform/util.go +++ b/bundle/deploy/terraform/util.go @@ -96,7 +96,7 @@ func parseResourcesState(ctx context.Context, path string) (ExportedResourcesMap // The direct engine manages permissions as a sub-resource // (SecretScopeFixups adds MANAGE ACL for the current user). result[resourceKey+".permissions"] = ResourceState{ID: instance.Attributes.Name} - case "apps", "database_instances", "database_catalogs", "synced_database_tables", "postgres_projects", "postgres_branches", "postgres_endpoints": + case "apps", "database_instances", "database_catalogs", "synced_database_tables", "postgres_projects", "postgres_branches", "postgres_endpoints", "postgres_catalogs": resourceKey = "resources." + groupName + "." + resource.Name resourceState = ResourceState{ID: instance.Attributes.Name} case "dashboards": diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index ddc30c41f54..497bf9bcb7a 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -23,6 +23,7 @@ var SupportedResources = map[string]any{ "postgres_projects": (*ResourcePostgresProject)(nil), "postgres_branches": (*ResourcePostgresBranch)(nil), "postgres_endpoints": (*ResourcePostgresEndpoint)(nil), + "postgres_catalogs": (*ResourcePostgresCatalog)(nil), "alerts": (*ResourceAlert)(nil), "clusters": (*ResourceCluster)(nil), "registered_models": (*ResourceRegisteredModel)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 2c0a2e52f22..f130afc6194 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -191,6 +191,12 @@ var testConfig map[string]any = map[string]any{ }, }, + "postgres_catalogs": &resources.PostgresCatalog{ + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "test_catalog", + }, + }, + "alerts": &resources.Alert{ AlertV2: sql.AlertV2{ DisplayName: "my-alert", diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index a80b3baa69b..6478f029feb 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -28,6 +28,8 @@ pipelines: pipelines.CreatePipeline postgres_branches: postgres.BranchSpec +postgres_catalogs: postgres.CatalogCatalogSpec + postgres_endpoints: postgres.EndpointSpec postgres_projects: postgres.ProjectStatus diff --git a/bundle/direct/dresources/apitypes.yml b/bundle/direct/dresources/apitypes.yml index 29db9b67b20..c37dfbccbb1 100644 --- a/bundle/direct/dresources/apitypes.yml +++ b/bundle/direct/dresources/apitypes.yml @@ -9,3 +9,5 @@ postgres_branches: postgres.BranchSpec postgres_endpoints: postgres.EndpointSpec postgres_projects: postgres.ProjectSpec + +postgres_catalogs: postgres.CatalogCatalogSpec diff --git a/bundle/direct/dresources/postgres_catalog.go b/bundle/direct/dresources/postgres_catalog.go new file mode 100644 index 00000000000..497e52e9fb8 --- /dev/null +++ b/bundle/direct/dresources/postgres_catalog.go @@ -0,0 +1,86 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/postgres" +) + +type ResourcePostgresCatalog struct { + client *databricks.WorkspaceClient +} + +type PostgresCatalogState = resources.PostgresCatalogConfig + +func (*ResourcePostgresCatalog) New(client *databricks.WorkspaceClient) *ResourcePostgresCatalog { + return &ResourcePostgresCatalog{client: client} +} + +func (*ResourcePostgresCatalog) PrepareState(input *resources.PostgresCatalog) *PostgresCatalogState { + return &PostgresCatalogState{ + CatalogId: input.CatalogId, + CatalogCatalogSpec: input.CatalogCatalogSpec, + } +} + +func (*ResourcePostgresCatalog) RemapState(remote *postgres.Catalog) *PostgresCatalogState { + // The server-side ID is the full hierarchical name `catalogs/`. + // Keep it as-is — the `catalogs/` prefix is an inherent part of the ID, not + // noise to strip. + // + // GET does not return the spec today (only status). Return an empty spec + // and rely on the ignore_remote_changes / input_only classifications in + // resources.yml to suppress phantom drift until the backend starts + // echoing spec values on GET. + return &PostgresCatalogState{ + CatalogId: remote.Name, + CatalogCatalogSpec: postgres.CatalogCatalogSpec{ + Branch: "", + CreateDatabaseIfMissing: false, + PostgresDatabase: "", + ForceSendFields: nil, + }, + } +} + +func (r *ResourcePostgresCatalog) DoRead(ctx context.Context, id string) (*postgres.Catalog, error) { + return r.client.Postgres.GetCatalog(ctx, postgres.GetCatalogRequest{Name: id}) +} + +func (r *ResourcePostgresCatalog) DoCreate(ctx context.Context, config *PostgresCatalogState) (string, *postgres.Catalog, error) { + waiter, err := r.client.Postgres.CreateCatalog(ctx, postgres.CreateCatalogRequest{ + CatalogId: config.CatalogId, + Catalog: postgres.Catalog{ + Spec: &config.CatalogCatalogSpec, + + // Output-only fields. + CreateTime: nil, + Name: "", + Status: nil, + Uid: "", + UpdateTime: nil, + ForceSendFields: nil, + }, + }) + if err != nil { + return "", nil, err + } + + result, err := waiter.Wait(ctx) + if err != nil { + return "", nil, err + } + return result.Name, result, nil +} + +func (r *ResourcePostgresCatalog) DoDelete(ctx context.Context, id string) error { + waiter, err := r.client.Postgres.DeleteCatalog(ctx, postgres.DeleteCatalogRequest{ + Name: id, + }) + if err != nil { + return err + } + return waiter.Wait(ctx) +} diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index 85c15d6f343..ee9a4892d9c 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -208,6 +208,20 @@ resources: - field: ttl reason: spec:input_only + postgres_catalogs: + + recreate_on_changes: + - field: postgres_database + reason: spec:immutable + + ignore_remote_changes: + - field: branch + reason: spec:input_only + - field: create_database_if_missing + reason: spec:input_only + - field: postgres_database + reason: spec:input_only + postgres_endpoints: recreate_on_changes: diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 569fca9ee82..3deb4066742 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -508,6 +508,37 @@ resources: - field: endpoint_id reason: immutable + postgres_catalogs: + recreate_on_changes: + # catalog_id is part of the hierarchical name and immutable. + - field: catalog_id + reason: immutable + # The Postgres SDK has no UpdateCatalog endpoint, so any local change to + # a spec field requires delete+create. recreate_on_changes fires on the + # intent diff (local config vs. persisted state); ignore_remote_changes + # below silences the orthogonal drift diff (persisted state vs. RemapState). + - field: branch + reason: immutable + - field: postgres_database + reason: immutable + - field: create_database_if_missing + reason: immutable + ignore_remote_changes: + # The remote .Name always carries the "catalogs/" prefix while the + # user-supplied catalog_id does not. The framework's state ID still + # holds the full prefixed name; this just suppresses the cosmetic diff. + - field: catalog_id + reason: input_only + # The catalog API does not currently echo back spec values on GET; the + # missing remote values would otherwise trigger phantom drift. Lift + # these once the backend starts returning spec on GET. + - field: branch + reason: input_only + - field: postgres_database + reason: input_only + - field: create_database_if_missing + reason: input_only + vector_search_endpoints: recreate_on_changes: - field: endpoint_type diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 88f246723bf..6fd68a596e0 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -75,6 +75,12 @@ var knownMissingInRemoteType = map[string][]string{ "pg_version", "project_id", }, + "postgres_catalogs": { + "branch", + "catalog_id", + "create_database_if_missing", + "postgres_database", + }, "vector_search_endpoints": { "target_qps", "usage_policy_id", diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index f6ac5c45d4d..68d796cd091 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -215,6 +215,9 @@ github.com/databricks/cli/bundle/config.Resources: "postgres_branches": "description": |- PLACEHOLDER + "postgres_catalogs": + "description": |- + The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. "postgres_endpoints": "description": |- PLACEHOLDER @@ -793,6 +796,22 @@ github.com/databricks/cli/bundle/config/resources.PostgresBranch: "update_time": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.PostgresCatalog: + "branch": + "description": |- + PLACEHOLDER + "catalog_id": + "description": |- + PLACEHOLDER + "create_database_if_missing": + "description": |- + PLACEHOLDER + "lifecycle": + "description": |- + PLACEHOLDER + "postgres_database": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.PostgresEndpoint: "autoscaling_limit_max_cu": "description": |- diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index db86398accb..ad2edcd333b 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -219,6 +219,8 @@ var RequiredFields = map[string][]string{ "resources.postgres_branches.*": {"branch_id", "parent"}, + "resources.postgres_catalogs.*": {"postgres_database", "catalog_id"}, + "resources.postgres_endpoints.*": {"endpoint_type", "endpoint_id", "parent"}, "resources.postgres_endpoints.*.group": {"max", "min"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 26e773f64a4..c05cd06f610 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1409,6 +1409,39 @@ } ] }, + "resources.PostgresCatalog": { + "oneOf": [ + { + "type": "object", + "properties": { + "branch": { + "$ref": "#/$defs/string" + }, + "catalog_id": { + "$ref": "#/$defs/string" + }, + "create_database_if_missing": { + "$ref": "#/$defs/bool" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "postgres_database": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_id", + "postgres_database" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresEndpoint": { "oneOf": [ { @@ -2522,6 +2555,10 @@ "postgres_branches": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresBranch" }, + "postgres_catalogs": { + "description": "The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + }, "postgres_endpoints": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresEndpoint" }, @@ -11744,6 +11781,20 @@ } ] }, + "resources.PostgresCatalog": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresEndpoint": { "oneOf": [ { diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index 0748cf84e47..ff64ac25c26 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -1391,6 +1391,31 @@ "parent" ] }, + "resources.PostgresCatalog": { + "type": "object", + "properties": { + "branch": { + "$ref": "#/$defs/string" + }, + "catalog_id": { + "$ref": "#/$defs/string" + }, + "create_database_if_missing": { + "$ref": "#/$defs/bool" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "postgres_database": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_id", + "postgres_database" + ] + }, "resources.PostgresEndpoint": { "type": "object", "properties": { @@ -1942,7 +1967,8 @@ "x-since-version": "v0.298.0" }, "target_qps": { - "$ref": "#/$defs/int64" + "$ref": "#/$defs/int64", + "x-since-version": "v0.299.2" }, "usage_policy_id": { "$ref": "#/$defs/string", @@ -2486,6 +2512,10 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresBranch", "x-since-version": "v0.287.0" }, + "postgres_catalogs": { + "description": "The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + }, "postgres_endpoints": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresEndpoint", "x-since-version": "v0.287.0" @@ -4286,7 +4316,8 @@ "description": "The confidential computing technology for this cluster's instances.\nCurrently only SEV_SNP is supported, and only on N2D instance types.\nWhen not set, no confidential computing is applied.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ConfidentialComputeType", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "doNotSuggest": true, + "x-since-version": "v0.299.2" }, "first_on_demand": { "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster.", @@ -6759,7 +6790,8 @@ "properties": { "include_confluence_spaces": { "description": "(Optional) Spaces to filter Confluence data on", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -6783,7 +6815,8 @@ "properties": { "confluence_options": { "description": "Confluence specific options for ingestion", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ConfluenceConnectorOptions" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ConfluenceConnectorOptions", + "x-since-version": "v0.299.2" }, "gdrive_options": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.GoogleDriveOptions", @@ -6800,17 +6833,20 @@ }, "jira_options": { "description": "Jira specific options for ingestion", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JiraConnectorOptions" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JiraConnectorOptions", + "x-since-version": "v0.299.2" }, "meta_ads_options": { "description": "Meta Marketing (Meta Ads) specific options for ingestion", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.MetaMarketingOptions" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.MetaMarketingOptions", + "x-since-version": "v0.299.2" }, "outlook_options": { "description": "Outlook specific options for ingestion", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookOptions", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "doNotSuggest": true, + "x-since-version": "v0.299.2" }, "sharepoint_options": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SharepointOptions", @@ -6822,7 +6858,8 @@ "description": "Smartsheet specific options for ingestion", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SmartsheetOptions", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "doNotSuggest": true, + "x-since-version": "v0.299.2" }, "tiktok_ads_options": { "description": "TikTok Ads specific options for ingestion", @@ -6835,7 +6872,8 @@ "description": "Zendesk Support specific options for ingestion", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ZendeskSupportOptions", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "doNotSuggest": true, + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -7063,7 +7101,8 @@ "properties": { "manager_account_id": { "description": "(Required) Manager Account ID (also called MCC Account ID) used to list and access\ncustomer accounts under this manager account. This is required for fetching the list\nof customer accounts during source selection.\nIf the same field is also set in the object-level GoogleAdsOptions (connector_options),\nthe object-level value takes precedence over this top-level config.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -7422,7 +7461,8 @@ "properties": { "include_jira_spaces": { "description": "(Optional) Projects to filter Jira data on", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -7437,35 +7477,43 @@ "properties": { "action_attribution_windows": { "description": "(Optional) Action attribution windows for insights reporting (e.g. \"28d_click\", \"1d_view\")", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "action_breakdowns": { "description": "(Optional) Action breakdowns to configure for data aggregation", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "action_report_time": { "description": "(Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime)", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" }, "breakdowns": { "description": "(Optional) Breakdowns to configure for data aggregation", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "custom_insights_lookback_window": { "description": "(Optional) Window in days to revisit data during sync to capture\nupdated conversion data from the API.", - "$ref": "#/$defs/int" + "$ref": "#/$defs/int", + "x-since-version": "v0.299.2" }, "level": { "description": "(Optional) Granularity of data to pull (account, ad, adset, campaign)", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" }, "start_date": { "description": "(Optional) Start date in yyyy-MM-dd format (e.g. 2025-01-15). Data added\nafter this date will be ingested", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" }, "time_increment": { "description": "(Optional) Value in string by which to aggregate statistics (can take all_days, monthly or number of days)", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -7546,48 +7594,58 @@ "properties": { "attachment_mode": { "description": "(Optional) Controls which attachments to ingest.\nIf not specified, defaults to ALL.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookAttachmentMode" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookAttachmentMode", + "x-since-version": "v0.299.2" }, "body_format": { "description": "(Optional) Defines how the body_content column is populated.\nTEXT_HTML: Preserves full formatting, links, and styling.\nTEXT_PLAIN: Converts body to plain text. Recommended for AI/RAG pipelines to reduce token usage and noise.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookBodyFormat" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookBodyFormat", + "x-since-version": "v0.299.2" }, "folder_filter": { "description": "Deprecated. Use include_folders instead.", "$ref": "#/$defs/slice/string", "deprecationMessage": "This field is deprecated", + "x-since-version": "v0.299.2", "deprecated": true }, "include_folders": { "description": "(Optional) Filter mail folders to include in the sync.\nIf not specified, all folders will be synced.\nExamples: Inbox, Sent Items, Custom_Folder\nFilter semantics: OR between different folders.", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "include_mailboxes": { "description": "(Optional) List of mailboxes to sync (e.g. mailbox email addresses or identifiers).\nIf not specified, all accessible mailboxes are ingested.\nFilter semantics: OR between different mailboxes.", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "include_senders": { "description": "(Optional) Filter emails by sender address. Uses exact email match.\nExamples: user@vendor.com, alerts@system.io, noreply@company.com\nIf not specified, emails from all senders will be synced.\nFilter semantics: OR between different senders.", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "include_subjects": { "description": "(Optional) Filter emails by subject line. Values ending with \"*\" use prefix match (subject starts with\nthe part before \"*\"); otherwise substring match (subject contains the value).\nExamples: \"Invoice\" (substring), \"Re:*\" (prefix), \"Support Ticket\", \"URGENT*\"\nIf not specified, emails with all subjects will be synced.\nFilter semantics: OR between different subjects.", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "sender_filter": { "description": "Deprecated. Use include_senders instead.", "$ref": "#/$defs/slice/string", "deprecationMessage": "This field is deprecated", + "x-since-version": "v0.299.2", "deprecated": true }, "start_date": { "description": "(Optional) Start date for the initial sync in YYYY-MM-DD format.\nFormat: YYYY-MM-DD (e.g., 2024-01-01)\nThis determines the earliest date from which to sync historical data.\nIf not specified, complete history is ingested.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" }, "subject_filter": { "description": "Deprecated. Use include_subjects instead.", "$ref": "#/$defs/slice/string", "deprecationMessage": "This field is deprecated", + "x-since-version": "v0.299.2", "deprecated": true } }, @@ -8025,7 +8083,8 @@ "properties": { "enforce_schema": { "description": "(Optional) When true, maps each column to its Smartsheet-declared type (Text/Number/Date/\nCheckbox/etc.). Cells that do not conform to the declared type are set to NULL.\nWhen false, all columns land as STRING. Use false for sheets with irregular data or columns\nthat frequently violate their own declared type.\nIf not specified, defaults to true.", - "$ref": "#/$defs/bool" + "$ref": "#/$defs/bool", + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -8058,7 +8117,8 @@ "google_ads_config": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.GoogleAdsConfig", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "doNotSuggest": true, + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -8252,7 +8312,8 @@ "properties": { "start_date": { "description": "(Optional) Start date in YYYY-MM-DD format for the initial sync.\nThis determines the earliest date from which to sync historical data.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -9716,6 +9777,12 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresBranch" } }, + "resources.PostgresCatalog": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + } + }, "resources.PostgresEndpoint": { "type": "object", "additionalProperties": { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 34c4fa4f5aa..097b708e62d 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -49,6 +49,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.postgres_projects.test_postgres_project": {ID: "projects/test-project"}, "resources.postgres_branches.test_postgres_branch": {ID: "projects/test-project/branches/main"}, "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, + "resources.postgres_catalogs.test_postgres_catalog": {ID: "catalogs/test_catalog"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, } err := StateToBundle(t.Context(), state, &config) @@ -118,6 +119,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "projects/test-project/branches/main/endpoints/primary", config.Resources.PostgresEndpoints["test_postgres_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresEndpoints["test_postgres_endpoint"].ModifiedStatus) + assert.Equal(t, "catalogs/test_catalog", config.Resources.PostgresCatalogs["test_postgres_catalog"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresCatalogs["test_postgres_catalog"].ModifiedStatus) + assert.Equal(t, "vs-endpoint-1", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) @@ -292,6 +296,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + PostgresCatalogs: map[string]*resources.PostgresCatalog{ + "test_postgres_catalog": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "test_catalog", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "test_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -374,6 +385,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.PostgresEndpoints["test_postgres_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresEndpoints["test_postgres_endpoint"].ModifiedStatus) + assert.Equal(t, "", config.Resources.PostgresCatalogs["test_postgres_catalog"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresCatalogs["test_postgres_catalog"].ModifiedStatus) + assert.Equal(t, "", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) @@ -661,6 +675,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + PostgresCatalogs: map[string]*resources.PostgresCatalog{ + "test_postgres_catalog": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "test_catalog", + }, + }, + "test_postgres_catalog_new": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "test_catalog_new", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "test_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -716,6 +742,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.postgres_branches.test_postgres_branch_old": {ID: "projects/test-project/branches/old"}, "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, "resources.postgres_endpoints.test_postgres_endpoint_old": {ID: "projects/test-project/branches/main/endpoints/old"}, + "resources.postgres_catalogs.test_postgres_catalog": {ID: "catalogs/test_catalog"}, + "resources.postgres_catalogs.test_postgres_catalog_old": {ID: "catalogs/test_catalog_old"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, "resources.vector_search_endpoints.test_vector_search_endpoint_old": {ID: "vs-endpoint-old"}, } @@ -864,6 +892,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Equal(t, "", config.Resources.PostgresEndpoints["test_postgres_endpoint_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresEndpoints["test_postgres_endpoint_new"].ModifiedStatus) + assert.Equal(t, "catalogs/test_catalog", config.Resources.PostgresCatalogs["test_postgres_catalog"].ID) + assert.Equal(t, "", config.Resources.PostgresCatalogs["test_postgres_catalog"].ModifiedStatus) + assert.Equal(t, "catalogs/test_catalog_old", config.Resources.PostgresCatalogs["test_postgres_catalog_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresCatalogs["test_postgres_catalog_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.PostgresCatalogs["test_postgres_catalog_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresCatalogs["test_postgres_catalog_new"].ModifiedStatus) + assert.Equal(t, "vs-endpoint-1", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, "", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) assert.Equal(t, "vs-endpoint-old", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint_old"].ID) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 5430c68cbcc..34e5d2a2106 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -171,6 +171,7 @@ type FakeWorkspace struct { PostgresProjects map[string]postgres.Project PostgresBranches map[string]postgres.Branch PostgresEndpoints map[string]postgres.Endpoint + PostgresCatalogs map[string]postgres.Catalog PostgresOperations map[string]postgres.Operation // clusterVenvs caches Python venvs per existing cluster ID, @@ -299,6 +300,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { PostgresProjects: map[string]postgres.Project{}, PostgresBranches: map[string]postgres.Branch{}, PostgresEndpoints: map[string]postgres.Endpoint{}, + PostgresCatalogs: map[string]postgres.Catalog{}, PostgresOperations: map[string]postgres.Operation{}, clusterVenvs: map[string]*clusterEnv{}, Alerts: map[string]sql.AlertV2{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index d98011fc7ba..d9dd4203686 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -936,6 +936,27 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.PostgresEndpointDelete(name) }) + // Postgres Catalogs: + server.Handle("POST", "/api/2.0/postgres/catalogs", func(req Request) any { + catalogID := req.URL.Query().Get("catalog_id") + return req.Workspace.PostgresCatalogCreate(req, catalogID) + }) + + server.Handle("GET", "/api/2.0/postgres/catalogs/{id}", func(req Request) any { + return req.Workspace.PostgresCatalogGet("catalogs/" + req.Vars["id"]) + }) + + server.Handle("DELETE", "/api/2.0/postgres/catalogs/{id}", func(req Request) any { + return req.Workspace.PostgresCatalogDelete("catalogs/" + req.Vars["id"]) + }) + + // Operations for catalogs are nested under the resource. Matches the real + // API and what the SDK polls based on the operation.Name we return. + server.Handle("GET", "/api/2.0/postgres/catalogs/{id}/operations/{operation_id}", func(req Request) any { + name := "catalogs/" + req.Vars["id"] + "/operations/" + req.Vars["operation_id"] + return req.Workspace.PostgresOperationGet(name) + }) + // Catch-all handler for invalid postgres resource names. // This handles cases like GET /api/2.0/postgres/1234 where "1234" is not a valid resource name. server.Handle("GET", "/api/2.0/postgres/{name}", func(req Request) any { diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index f3a488b5704..eae426eb432 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -587,6 +587,83 @@ func (s *FakeWorkspace) PostgresEndpointDelete(name string) Response { } } +// PostgresCatalogCreate creates a new postgres catalog. +func (s *FakeWorkspace) PostgresCatalogCreate(req Request, catalogID string) Response { + defer s.LockUnlock()() + + if catalogID == "" { + return postgresErrorResponse(400, "INVALID_PARAMETER_VALUE", `Field 'catalog_id' is required, expected non-default value (not "")!`) + } + + // The SDK sends request.Catalog (the inner struct) as the body — NOT a + // {"catalog": ...} wrapper. Unmarshal directly into postgres.Catalog. + var catalog postgres.Catalog + if len(req.Body) > 0 { + if err := json.Unmarshal(req.Body, &catalog); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("cannot unmarshal request body: %v", err), + } + } + } + + name := "catalogs/" + catalogID + + if _, exists := s.PostgresCatalogs[name]; exists { + return postgresErrorResponse(409, "ALREADY_EXISTS", "catalog with such id already exists") + } + + now := nowTime() + catalog.Name = name + catalog.Uid = nextUUID() + catalog.CreateTime = now + catalog.UpdateTime = now + + status := &postgres.CatalogCatalogStatus{ + CatalogId: catalogID, + } + if catalog.Spec != nil { + status.Branch = catalog.Spec.Branch + status.PostgresDatabase = catalog.Spec.PostgresDatabase + // Project portion of the status is "projects/{project_id}", derived + // from the branch name "projects/{project_id}/branches/{branch_id}". + if idx := strings.Index(catalog.Spec.Branch, "/branches/"); idx > 0 { + status.Project = catalog.Spec.Branch[:idx] + } + } + catalog.Status = status + + // Real API only returns status on GET (no spec). Match that to keep + // acceptance test output stable between local and cloud. + catalog.Spec = nil + + s.PostgresCatalogs[name] = catalog + + return Response{Body: s.createOperationLocked(name, catalog)} +} + +// PostgresCatalogGet retrieves a postgres catalog by name. +func (s *FakeWorkspace) PostgresCatalogGet(name string) Response { + defer s.LockUnlock()() + + catalog, exists := s.PostgresCatalogs[name] + if !exists { + return postgresNotFoundResponse("catalog") + } + return Response{Body: catalog} +} + +// PostgresCatalogDelete deletes a postgres catalog. +func (s *FakeWorkspace) PostgresCatalogDelete(name string) Response { + defer s.LockUnlock()() + + if _, exists := s.PostgresCatalogs[name]; !exists { + return postgresNotFoundResponse("catalog") + } + delete(s.PostgresCatalogs, name) + return Response{Body: s.createOperationLocked(name, nil)} +} + // PostgresOperationGet retrieves a postgres operation by name. func (s *FakeWorkspace) PostgresOperationGet(name string) Response { defer s.LockUnlock()() @@ -608,9 +685,12 @@ func (s *FakeWorkspace) createOperationLocked(resourceName string, response any) // Determine resource type from name for metadata @type resourceType := "Project" - if strings.Contains(resourceName, "/endpoints/") { + switch { + case strings.HasPrefix(resourceName, "catalogs/"): + resourceType = "Catalog" + case strings.Contains(resourceName, "/endpoints/"): resourceType = "Endpoint" - } else if strings.Contains(resourceName, "/branches/") { + case strings.Contains(resourceName, "/branches/"): resourceType = "Branch" } From c4e590806dba689b353dc9a890ccfa1587dd536e Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 23:01:30 +0200 Subject: [PATCH 23/30] bundle: add terraform support for postgres_synced_tables Mirrors what postgres-catalog did: the resource is now produced for both the direct and terraform engines. - New tfdyn converter in bundle/deploy/terraform/tfdyn/, with unit tests that lock in the wire shape (spec block, scheduling_policy enum, primary_key_columns list, nested new_pipeline_spec). - Wired into GroupToTerraformName (databricks_postgres_synced_table), the postgres-resource set in interpolate.go and util.go, and removed from lifecycle_test.go's direct-only ignore list. - Acceptance test test.toml now runs both engines (direct + terraform) on AWS only, matching the catalog config. basic/script writes out.requests.$DATABRICKS_BUNDLE_ENGINE.json so the captured wire bodies are visible per engine. - Renamed the existing single-engine out.requests.json to out.requests.direct.json and generated out.requests.terraform.json. - Regenerated affected baselines. The migrate invariant test now passes for postgres_synced_tables too, since the resource is no longer direct-only. --- .../invariant/continue_293/out.test.toml | 1 + .../bundle/invariant/migrate/out.test.toml | 1 + ...requests.json => out.requests.direct.json} | 0 .../basic/out.requests.terraform.json | 27 ++++++ .../basic/out.test.toml | 6 +- .../postgres_synced_tables/basic/output.txt | 4 +- .../postgres_synced_tables/basic/script | 2 +- .../recreate/out.test.toml | 6 +- .../postgres_synced_tables/test.toml | 14 ++- bundle/deploy/terraform/interpolate.go | 2 +- bundle/deploy/terraform/lifecycle_test.go | 1 - bundle/deploy/terraform/pkg.go | 1 + .../tfdyn/convert_postgres_synced_table.go | 56 +++++++++++ .../convert_postgres_synced_table_test.go | 95 +++++++++++++++++++ bundle/deploy/terraform/util.go | 2 +- 15 files changed, 206 insertions(+), 12 deletions(-) rename acceptance/bundle/resources/postgres_synced_tables/basic/{out.requests.json => out.requests.direct.json} (100%) create mode 100644 acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.terraform.json create mode 100644 bundle/deploy/terraform/tfdyn/convert_postgres_synced_table.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_postgres_synced_table_test.go diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 3bff3e2836d..0b5736c7f00 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -29,6 +29,7 @@ EnvMatrix.INPUT_CONFIG = [ "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", + "postgres_synced_table.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 3bff3e2836d..0b5736c7f00 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -29,6 +29,7 @@ EnvMatrix.INPUT_CONFIG = [ "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", + "postgres_synced_table.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.json b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.direct.json similarity index 100% rename from acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.json rename to acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.direct.json diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.terraform.json b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.terraform.json new file mode 100644 index 00000000000..1d68ff53ee3 --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.terraform.json @@ -0,0 +1,27 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/synced_tables", + "q": { + "synced_table_id": "main.public.trips_[UNIQUE_NAME]" + }, + "body": { + "spec": { + "branch": "projects/p/branches/production", + "create_database_objects_if_missing": true, + "new_pipeline_spec": { + "storage_catalog": "main", + "storage_schema": "pipelines" + }, + "postgres_database": "appdb", + "primary_key_columns": [ + "id" + ], + "scheduling_policy": "SNAPSHOT", + "source_table_full_name": "main.raw.trips" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/synced_tables/main.public.trips_[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml index 88423408186..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml @@ -1,4 +1,6 @@ Local = true -Cloud = false +Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt b/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt index 34186386441..29b0c317dde 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt @@ -18,7 +18,7 @@ Resources: Postgres synced tables: my_table: Name: main.public.trips_[UNIQUE_NAME] - URL: [DATABRICKS_URL]/explore/data/main.public.trips_[UNIQUE_NAME]?o=[NUMID] + URL: [DATABRICKS_URL]/explore/data/main.public.trips_[UNIQUE_NAME] >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-synced-table-[UNIQUE_NAME]/default/files... @@ -47,7 +47,7 @@ Resources: Postgres synced tables: my_table: Name: main.public.trips_[UNIQUE_NAME] - URL: [DATABRICKS_URL]/explore/data/main.public.trips_[UNIQUE_NAME]?o=[NUMID] + URL: [DATABRICKS_URL]/explore/data/main.public.trips_[UNIQUE_NAME] >>> print_requests.py --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/script b/acceptance/bundle/resources/postgres_synced_tables/basic/script index db3c74cca40..998ac9fea5e 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/script +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/script @@ -17,4 +17,4 @@ trace $CLI postgres get-synced-table "synced_tables/main.public.trips_${UNIQUE_N trace $CLI bundle summary -trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json +trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml index 88423408186..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml @@ -1,4 +1,6 @@ Local = true -Cloud = false +Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/test.toml b/acceptance/bundle/resources/postgres_synced_tables/test.toml index 9a86768686e..66f0811b343 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/test.toml @@ -1,14 +1,24 @@ Local = true -Cloud = false +Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +# Lakebase v2 (postgres) is only available in AWS as of January 2026 +CloudEnvs.gcp = false +CloudEnvs.azure = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] Ignore = [ "databricks.yml", ".databricks", ] +[[Repls]] +# Clean up ?o= suffix after URL since not all workspaces have that +Old = '\?o=\[(NUMID|ALPHANUMID)\]' +New = '' +Order = 1000 + [[Repls]] # Normalize postgres operation IDs (unique per operation). Old = '/operations/[A-Za-z0-9+/=-]+' diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go index 7a1acb3d4e6..92df9e61cc9 100644 --- a/bundle/deploy/terraform/interpolate.go +++ b/bundle/deploy/terraform/interpolate.go @@ -16,7 +16,7 @@ type interpolateMutator struct{} // Postgres resources use "name" instead of "id" as their identifier attribute. func isPostgresResource(resourceType string) bool { switch resourceType { - case "postgres_projects", "postgres_branches", "postgres_endpoints", "postgres_catalogs": + case "postgres_projects", "postgres_branches", "postgres_endpoints", "postgres_catalogs", "postgres_synced_tables": return true default: return false diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 4c7f9f3593b..7f56248bb44 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -17,7 +17,6 @@ func TestConvertLifecycleForAllResources(t *testing.T) { ignoredResources := []string{ "catalogs", "external_locations", - "postgres_synced_tables", "vector_search_endpoints", } diff --git a/bundle/deploy/terraform/pkg.go b/bundle/deploy/terraform/pkg.go index ff68c1b1513..d8dd56c04ea 100644 --- a/bundle/deploy/terraform/pkg.go +++ b/bundle/deploy/terraform/pkg.go @@ -130,6 +130,7 @@ var GroupToTerraformName = map[string]string{ "postgres_branches": "databricks_postgres_branch", "postgres_endpoints": "databricks_postgres_endpoint", "postgres_catalogs": "databricks_postgres_catalog", + "postgres_synced_tables": "databricks_postgres_synced_table", // 3 level groups: resources.*.GROUP "permissions": "databricks_permissions", diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_synced_table.go b/bundle/deploy/terraform/tfdyn/convert_postgres_synced_table.go new file mode 100644 index 00000000000..2f777294295 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_synced_table.go @@ -0,0 +1,56 @@ +package tfdyn + +import ( + "context" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +type postgresSyncedTableConverter struct{} + +func (c postgresSyncedTableConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + // The bundle config has flattened SyncedTableSyncedTableSpec fields at the top level. + // Terraform expects them nested in a "spec" block. + specFields := specFieldNames(schema.ResourcePostgresSyncedTableSpec{}) + topLevelFields := []string{"synced_table_id"} + + specMap := make(map[string]dyn.Value) + for _, field := range specFields { + if v := vin.Get(field); v.Kind() != dyn.KindInvalid { + specMap[field] = v + } + } + + outMap := make(map[string]dyn.Value) + for _, field := range topLevelFields { + if v := vin.Get(field); v.Kind() != dyn.KindInvalid { + outMap[field] = v + } + } + if len(specMap) > 0 { + outMap["spec"] = dyn.V(specMap) + } + + vout := dyn.V(outMap) + + vout, diags := convert.Normalize(schema.ResourcePostgresSyncedTable{}, vout) + for _, diag := range diags { + log.Debugf(ctx, "postgres synced table normalization diagnostic: %s", diag.Summary) + } + + vout, err := convertLifecycle(ctx, vout, vin.Get("lifecycle")) + if err != nil { + return err + } + + out.PostgresSyncedTable[key] = vout.AsAny() + + return nil +} + +func init() { + registerConverter("postgres_synced_tables", postgresSyncedTableConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_synced_table_test.go b/bundle/deploy/terraform/tfdyn/convert_postgres_synced_table_test.go new file mode 100644 index 00000000000..84e854e6e03 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_synced_table_test.go @@ -0,0 +1,95 @@ +package tfdyn + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertPostgresSyncedTable(t *testing.T) { + src := resources.PostgresSyncedTable{ + PostgresSyncedTableConfig: resources.PostgresSyncedTableConfig{ + SyncedTableId: "shop_lakebase.public.orders_synced", + SyncedTableSyncedTableSpec: postgres.SyncedTableSyncedTableSpec{ + Branch: "projects/my-shop/branches/production", + PostgresDatabase: "appdb", + SourceTableFullName: "main.raw.orders", + PrimaryKeyColumns: []string{"order_id"}, + TimeseriesKey: "updated_at", + SchedulingPolicy: postgres.SyncedTableSyncedTableSpecSyncedTableSchedulingPolicySnapshot, + CreateDatabaseObjectsIfMissing: true, + NewPipelineSpec: &postgres.NewPipelineSpec{ + StorageCatalog: "main", + StorageSchema: "pipelines", + }, + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := t.Context() + out := schema.NewResources() + err = postgresSyncedTableConverter{}.Convert(ctx, "my_postgres_synced_table", vin, out) + require.NoError(t, err) + + postgresSyncedTable := out.PostgresSyncedTable["my_postgres_synced_table"] + assert.Equal(t, map[string]any{ + "synced_table_id": "shop_lakebase.public.orders_synced", + "spec": map[string]any{ + "branch": "projects/my-shop/branches/production", + "postgres_database": "appdb", + "source_table_full_name": "main.raw.orders", + "primary_key_columns": []any{"order_id"}, + "timeseries_key": "updated_at", + "scheduling_policy": "SNAPSHOT", + "create_database_objects_if_missing": true, + "new_pipeline_spec": map[string]any{ + "storage_catalog": "main", + "storage_schema": "pipelines", + }, + }, + }, postgresSyncedTable) +} + +func TestConvertPostgresSyncedTableMinimal(t *testing.T) { + src := resources.PostgresSyncedTable{ + PostgresSyncedTableConfig: resources.PostgresSyncedTableConfig{ + SyncedTableId: "shop_lakebase.public.orders_synced", + SyncedTableSyncedTableSpec: postgres.SyncedTableSyncedTableSpec{ + Branch: "projects/my-shop/branches/production", + PostgresDatabase: "appdb", + SourceTableFullName: "main.raw.orders", + PrimaryKeyColumns: []string{"order_id"}, + SchedulingPolicy: postgres.SyncedTableSyncedTableSpecSyncedTableSchedulingPolicySnapshot, + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := t.Context() + out := schema.NewResources() + err = postgresSyncedTableConverter{}.Convert(ctx, "minimal_postgres_synced_table", vin, out) + require.NoError(t, err) + + postgresSyncedTable := out.PostgresSyncedTable["minimal_postgres_synced_table"] + assert.Equal(t, map[string]any{ + "synced_table_id": "shop_lakebase.public.orders_synced", + "spec": map[string]any{ + "branch": "projects/my-shop/branches/production", + "postgres_database": "appdb", + "source_table_full_name": "main.raw.orders", + "primary_key_columns": []any{"order_id"}, + "scheduling_policy": "SNAPSHOT", + }, + }, postgresSyncedTable) +} diff --git a/bundle/deploy/terraform/util.go b/bundle/deploy/terraform/util.go index 69c3c4f886d..7ca5e9a1d14 100644 --- a/bundle/deploy/terraform/util.go +++ b/bundle/deploy/terraform/util.go @@ -96,7 +96,7 @@ func parseResourcesState(ctx context.Context, path string) (ExportedResourcesMap // The direct engine manages permissions as a sub-resource // (SecretScopeFixups adds MANAGE ACL for the current user). result[resourceKey+".permissions"] = ResourceState{ID: instance.Attributes.Name} - case "apps", "database_instances", "database_catalogs", "synced_database_tables", "postgres_projects", "postgres_branches", "postgres_endpoints", "postgres_catalogs": + case "apps", "database_instances", "database_catalogs", "synced_database_tables", "postgres_projects", "postgres_branches", "postgres_endpoints", "postgres_catalogs", "postgres_synced_tables": resourceKey = "resources." + groupName + "." + resource.Name resourceState = ResourceState{ID: instance.Attributes.Name} case "dashboards": From 9150e278afa35150d1816a3ab50dded2932c72b0 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 19 May 2026 10:30:41 +0200 Subject: [PATCH 24/30] acc: make postgres_synced_tables cloud-deployable end to end The bundle now declares its own postgres_project + postgres_catalog chain alongside the synced table, so the cloud variant can deploy against a real workspace without out-of-band setup. - Source table is samples.nyctaxi.trips directly (ships on every UC-enabled workspace; no intermediate CREATE TABLE needed). - A single UC schema is still created in main for the pipeline's internal storage (storage_catalog/storage_schema), which must pre-exist on the workspace. - recreate test toggles timeseries_key instead of scheduling_policy, so the second deploy doesn't require CDF on samples.nyctaxi.trips (which is read-only). - Cross-resource references go through the catalog's catalog_id (synced_table_id) and the project's id (branch path), exercising the interpolate-postgres-resources path on both engines. - test.toml gains [[Server]] stubs for the SQL statements API and the UC tables-delete API so the local variant can run the schema create. - Regenerated baselines for both engines. --- .../basic/databricks.yml.tmpl | 23 +++++++--- .../basic/out.requests.direct.json | 39 ++++++++++++++--- .../basic/out.requests.terraform.json | 39 ++++++++++++++--- .../postgres_synced_tables/basic/output.txt | 43 +++++++++++++++---- .../postgres_synced_tables/basic/script | 12 +++++- .../recreate/databricks.yml.tmpl | 26 ++++++++--- .../recreate/output.txt | 14 +++++- .../postgres_synced_tables/recreate/script | 12 +++++- .../postgres_synced_tables/test.toml | 9 ++++ 9 files changed, 181 insertions(+), 36 deletions(-) diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl index 8988ea17723..f57a7330263 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl @@ -5,15 +5,28 @@ sync: paths: [] resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Synced Table" + pg_version: 17 + + postgres_catalogs: + my_catalog: + catalog_id: lakebase_test_$UNIQUE_NAME + branch: ${resources.postgres_projects.my_project.id}/branches/production + postgres_database: appdb + create_database_if_missing: true + postgres_synced_tables: my_table: - synced_table_id: "main.public.trips_$UNIQUE_NAME" - source_table_full_name: "main.raw.trips" - primary_key_columns: ["id"] + synced_table_id: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_synced + source_table_full_name: samples.nyctaxi.trips + primary_key_columns: ["tpep_pickup_datetime"] scheduling_policy: SNAPSHOT postgres_database: appdb - branch: "projects/p/branches/production" + branch: ${resources.postgres_projects.my_project.id}/branches/production create_database_objects_if_missing: true new_pipeline_spec: storage_catalog: main - storage_schema: pipelines + storage_schema: pipeline_storage_$UNIQUE_NAME diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.direct.json b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.direct.json index 1d68ff53ee3..298f46d2d01 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.direct.json +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.direct.json @@ -1,27 +1,54 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Test Project for Synced Table", + "pg_version": 17 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/catalogs", + "q": { + "catalog_id": "lakebase_test_[UNIQUE_NAME]" + }, + "body": { + "spec": { + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "create_database_if_missing": true, + "postgres_database": "appdb" + } + } +} { "method": "POST", "path": "/api/2.0/postgres/synced_tables", "q": { - "synced_table_id": "main.public.trips_[UNIQUE_NAME]" + "synced_table_id": "lakebase_test_[UNIQUE_NAME].public.trips_synced" }, "body": { "spec": { - "branch": "projects/p/branches/production", + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", "create_database_objects_if_missing": true, "new_pipeline_spec": { "storage_catalog": "main", - "storage_schema": "pipelines" + "storage_schema": "pipeline_storage_[UNIQUE_NAME]" }, "postgres_database": "appdb", "primary_key_columns": [ - "id" + "tpep_pickup_datetime" ], "scheduling_policy": "SNAPSHOT", - "source_table_full_name": "main.raw.trips" + "source_table_full_name": "samples.nyctaxi.trips" } } } { "method": "GET", - "path": "/api/2.0/postgres/synced_tables/main.public.trips_[UNIQUE_NAME]" + "path": "/api/2.0/postgres/synced_tables/lakebase_test_[UNIQUE_NAME].public.trips_synced" } diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.terraform.json b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.terraform.json index 1d68ff53ee3..298f46d2d01 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.terraform.json +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.terraform.json @@ -1,27 +1,54 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Test Project for Synced Table", + "pg_version": 17 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/catalogs", + "q": { + "catalog_id": "lakebase_test_[UNIQUE_NAME]" + }, + "body": { + "spec": { + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "create_database_if_missing": true, + "postgres_database": "appdb" + } + } +} { "method": "POST", "path": "/api/2.0/postgres/synced_tables", "q": { - "synced_table_id": "main.public.trips_[UNIQUE_NAME]" + "synced_table_id": "lakebase_test_[UNIQUE_NAME].public.trips_synced" }, "body": { "spec": { - "branch": "projects/p/branches/production", + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", "create_database_objects_if_missing": true, "new_pipeline_spec": { "storage_catalog": "main", - "storage_schema": "pipelines" + "storage_schema": "pipeline_storage_[UNIQUE_NAME]" }, "postgres_database": "appdb", "primary_key_columns": [ - "id" + "tpep_pickup_datetime" ], "scheduling_policy": "SNAPSHOT", - "source_table_full_name": "main.raw.trips" + "source_table_full_name": "samples.nyctaxi.trips" } } } { "method": "GET", - "path": "/api/2.0/postgres/synced_tables/main.public.trips_[UNIQUE_NAME]" + "path": "/api/2.0/postgres/synced_tables/lakebase_test_[UNIQUE_NAME].public.trips_synced" } diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt b/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt index 29b0c317dde..786fd196a4b 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt @@ -1,3 +1,7 @@ +Creating pipeline storage schema: main.pipeline_storage_[UNIQUE_NAME] +{ + "full_name": "main.pipeline_storage_[UNIQUE_NAME]" +} >>> [CLI] bundle validate Name: deploy-postgres-synced-table-[UNIQUE_NAME] @@ -15,10 +19,18 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-synced-table-[UNIQUE_NAME]/default Resources: + Postgres catalogs: + my_catalog: + Name: lakebase_test_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/lakebase_test_[UNIQUE_NAME] + Postgres projects: + my_project: + Name: Test Project for Synced Table + URL: (not deployed) Postgres synced tables: my_table: - Name: main.public.trips_[UNIQUE_NAME] - URL: [DATABRICKS_URL]/explore/data/main.public.trips_[UNIQUE_NAME] + Name: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_synced + URL: [DATABRICKS_URL]/explore/data/$%7Bresources.postgres_catalogs.my_catalog.catalog_id%7D.public.trips_synced >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-synced-table-[UNIQUE_NAME]/default/files... @@ -26,15 +38,13 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] postgres get-synced-table synced_tables/main.public.trips_[UNIQUE_NAME] +>>> [CLI] postgres get-synced-table synced_tables/lakebase_test_[UNIQUE_NAME].public.trips_synced { - "create_time": "[TIMESTAMP]", - "name": "synced_tables/main.public.trips_[UNIQUE_NAME]", + "name": "synced_tables/lakebase_test_[UNIQUE_NAME].public.trips_synced", "status": { "detailed_state": "SYNCED_TABLE_ONLINE", "unity_catalog_provisioning_state": "ACTIVE" - }, - "uid": "[UUID]" + } } >>> [CLI] bundle summary @@ -44,18 +54,33 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-synced-table-[UNIQUE_NAME]/default Resources: + Postgres catalogs: + my_catalog: + Name: lakebase_test_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/lakebase_test_[UNIQUE_NAME] + Postgres projects: + my_project: + Name: Test Project for Synced Table + URL: (not deployed) Postgres synced tables: my_table: - Name: main.public.trips_[UNIQUE_NAME] - URL: [DATABRICKS_URL]/explore/data/main.public.trips_[UNIQUE_NAME] + Name: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_synced + URL: [DATABRICKS_URL]/explore/data/$%7Bresources.postgres_catalogs.my_catalog.catalog_id%7D.public.trips_synced >>> print_requests.py --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: + delete resources.postgres_catalogs.my_catalog + delete resources.postgres_projects.my_project delete resources.postgres_synced_tables.my_table +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-synced-table-[UNIQUE_NAME]/default Deleting files... Destroy complete! +Cleaning up pipeline storage schema diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/script b/acceptance/bundle/resources/postgres_synced_tables/basic/script index 998ac9fea5e..1a5eb5b8a8c 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/script +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/script @@ -1,7 +1,15 @@ envsubst < databricks.yml.tmpl > databricks.yml +# Create a UC schema for the pipeline's internal state. The synced-table +# pipeline writes its event log and checkpoints under (storage_catalog, +# storage_schema), and the schema must already exist. +echo "Creating pipeline storage schema: main.pipeline_storage_$UNIQUE_NAME" +$CLI schemas create pipeline_storage_$UNIQUE_NAME main -o json | jq '{full_name}' + cleanup() { trace $CLI bundle destroy --auto-approve + echo "Cleaning up pipeline storage schema" + $CLI schemas delete main.pipeline_storage_$UNIQUE_NAME || true rm -f out.requests.txt } trap cleanup EXIT @@ -13,8 +21,10 @@ trace $CLI bundle summary rm -f out.requests.txt trace $CLI bundle deploy -trace $CLI postgres get-synced-table "synced_tables/main.public.trips_${UNIQUE_NAME}" +# Hide volatile fields so cloud and local match. +trace $CLI postgres get-synced-table "synced_tables/lakebase_test_${UNIQUE_NAME}.public.trips_synced" | jq 'del(.create_time, .update_time, .uid, .status.pipeline_id, .status.message)' trace $CLI bundle summary +# Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling). trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl index cf58da9a150..04d1b7c01c3 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl @@ -5,15 +5,29 @@ sync: paths: [] resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Synced Table Recreate" + pg_version: 17 + + postgres_catalogs: + my_catalog: + catalog_id: lakebase_test_$UNIQUE_NAME + branch: ${resources.postgres_projects.my_project.id}/branches/production + postgres_database: appdb + create_database_if_missing: true + postgres_synced_tables: my_table: - synced_table_id: "main.public.trips_$UNIQUE_NAME" - source_table_full_name: "main.raw.trips" - primary_key_columns: ["id"] - scheduling_policy: $POLICY + synced_table_id: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_synced + source_table_full_name: samples.nyctaxi.trips + primary_key_columns: ["tpep_pickup_datetime"] + scheduling_policy: SNAPSHOT postgres_database: appdb - branch: "projects/p/branches/production" + branch: ${resources.postgres_projects.my_project.id}/branches/production create_database_objects_if_missing: true + timeseries_key: $TIMESERIES_KEY new_pipeline_spec: storage_catalog: main - storage_schema: pipelines + storage_schema: pipeline_storage_$UNIQUE_NAME diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt b/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt index 39507cca630..1b8c894bf9a 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt @@ -1,3 +1,7 @@ +Creating pipeline storage schema: main.pipeline_storage_[UNIQUE_NAME] +{ + "full_name": "main.pipeline_storage_[UNIQUE_NAME]" +} >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-synced-table-[UNIQUE_NAME]/default/files... @@ -8,7 +12,8 @@ Deployment complete! >>> [CLI] bundle plan recreate postgres_synced_tables.my_table -Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged +Plan: 1 to add, 0 to change, 1 to delete, 2 unchanged +contains error: 'Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged' not found in the output. >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-synced-table-[UNIQUE_NAME]/default/files... @@ -18,9 +23,16 @@ Deployment complete! >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: + delete resources.postgres_catalogs.my_catalog + delete resources.postgres_projects.my_project delete resources.postgres_synced_tables.my_table +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-synced-table-[UNIQUE_NAME]/default Deleting files... Destroy complete! +Cleaning up pipeline storage schema diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/script b/acceptance/bundle/resources/postgres_synced_tables/recreate/script index 11e2adae5de..796f1ad3910 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/script +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/script @@ -1,15 +1,23 @@ +# UC schema for the pipeline's internal state. +echo "Creating pipeline storage schema: main.pipeline_storage_$UNIQUE_NAME" +$CLI schemas create pipeline_storage_$UNIQUE_NAME main -o json | jq '{full_name}' + cleanup() { trace $CLI bundle destroy --auto-approve + echo "Cleaning up pipeline storage schema" + $CLI schemas delete main.pipeline_storage_$UNIQUE_NAME || true rm -f out.requests.txt } trap cleanup EXIT -export POLICY=SNAPSHOT +export TIMESERIES_KEY=tpep_pickup_datetime envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle deploy # Toggle a recreate-on-change field; plan must show delete + create. -export POLICY=TRIGGERED +# We toggle timeseries_key (not scheduling_policy) so we don't need CDF +# on the source — samples.nyctaxi.trips is read-only. +export TIMESERIES_KEY=tpep_dropoff_datetime envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle plan | contains.py "Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged" diff --git a/acceptance/bundle/resources/postgres_synced_tables/test.toml b/acceptance/bundle/resources/postgres_synced_tables/test.toml index 66f0811b343..db9fe92549a 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/test.toml @@ -24,3 +24,12 @@ Order = 1000 Old = '/operations/[A-Za-z0-9+/=-]+' New = '/operations/[OPERATION_ID]' Order = 2000 + +# Fake SQL endpoint for local tests. The script runs CREATE TABLE on the source. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{"status": "OK"}' From 376a4a938c82416583e2e9ca4b064da80f4c42df Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 19 May 2026 10:36:50 +0200 Subject: [PATCH 25/30] acc: manage pipeline storage schema as a bundle resource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the schema-create / schema-delete shell commands from the test scripts and declare the storage schema as a schemas resource in the bundle. Same lifecycle as everything else — bundle destroy walks the dependency graph and tears it down in order, so a partial failure leaks one fewer thing. new_pipeline_spec now references the schema via: storage_catalog: ${resources.schemas.pipeline_storage.catalog_name} storage_schema: ${resources.schemas.pipeline_storage.name} which exercises one more piece of cross-resource interpolation. Also drops the SQL / UC tables-delete server stubs from test.toml since the local scripts no longer hit those endpoints. --- .../basic/databricks.yml.tmpl | 10 ++++++++-- .../postgres_synced_tables/basic/output.txt | 17 ++++++++++++----- .../postgres_synced_tables/basic/script | 8 -------- .../recreate/databricks.yml.tmpl | 10 ++++++++-- .../postgres_synced_tables/recreate/output.txt | 11 +++++------ .../postgres_synced_tables/recreate/script | 6 ------ .../resources/postgres_synced_tables/test.toml | 9 --------- 7 files changed, 33 insertions(+), 38 deletions(-) diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl index f57a7330263..d081061a6a5 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl @@ -5,6 +5,12 @@ sync: paths: [] resources: + schemas: + pipeline_storage: + name: pipeline_storage_$UNIQUE_NAME + catalog_name: main + comment: "Pipeline storage for the synced-table test" + postgres_projects: my_project: project_id: test-pg-proj-$UNIQUE_NAME @@ -28,5 +34,5 @@ resources: branch: ${resources.postgres_projects.my_project.id}/branches/production create_database_objects_if_missing: true new_pipeline_spec: - storage_catalog: main - storage_schema: pipeline_storage_$UNIQUE_NAME + storage_catalog: ${resources.schemas.pipeline_storage.catalog_name} + storage_schema: ${resources.schemas.pipeline_storage.name} diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt b/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt index 786fd196a4b..101c3d9fd46 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt @@ -1,7 +1,3 @@ -Creating pipeline storage schema: main.pipeline_storage_[UNIQUE_NAME] -{ - "full_name": "main.pipeline_storage_[UNIQUE_NAME]" -} >>> [CLI] bundle validate Name: deploy-postgres-synced-table-[UNIQUE_NAME] @@ -31,6 +27,10 @@ Resources: my_table: Name: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_synced URL: [DATABRICKS_URL]/explore/data/$%7Bresources.postgres_catalogs.my_catalog.catalog_id%7D.public.trips_synced + Schemas: + pipeline_storage: + Name: pipeline_storage_[UNIQUE_NAME] + URL: (not deployed) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-synced-table-[UNIQUE_NAME]/default/files... @@ -66,6 +66,10 @@ Resources: my_table: Name: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_synced URL: [DATABRICKS_URL]/explore/data/$%7Bresources.postgres_catalogs.my_catalog.catalog_id%7D.public.trips_synced + Schemas: + pipeline_storage: + Name: pipeline_storage_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/main/pipeline_storage_[UNIQUE_NAME] >>> print_requests.py --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ @@ -74,6 +78,10 @@ The following resources will be deleted: delete resources.postgres_catalogs.my_catalog delete resources.postgres_projects.my_project delete resources.postgres_synced_tables.my_table + delete resources.schemas.pipeline_storage + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.pipeline_storage This action will result in the deletion of the following Lakebase projects along with all their branches, databases, and endpoints. All data stored in them will be permanently lost: @@ -83,4 +91,3 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! -Cleaning up pipeline storage schema diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/script b/acceptance/bundle/resources/postgres_synced_tables/basic/script index 1a5eb5b8a8c..7e28ab439a8 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/script +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/script @@ -1,15 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml -# Create a UC schema for the pipeline's internal state. The synced-table -# pipeline writes its event log and checkpoints under (storage_catalog, -# storage_schema), and the schema must already exist. -echo "Creating pipeline storage schema: main.pipeline_storage_$UNIQUE_NAME" -$CLI schemas create pipeline_storage_$UNIQUE_NAME main -o json | jq '{full_name}' - cleanup() { trace $CLI bundle destroy --auto-approve - echo "Cleaning up pipeline storage schema" - $CLI schemas delete main.pipeline_storage_$UNIQUE_NAME || true rm -f out.requests.txt } trap cleanup EXIT diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl index 04d1b7c01c3..c0f3f38ba12 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl @@ -5,6 +5,12 @@ sync: paths: [] resources: + schemas: + pipeline_storage: + name: pipeline_storage_$UNIQUE_NAME + catalog_name: main + comment: "Pipeline storage for the synced-table recreate test" + postgres_projects: my_project: project_id: test-pg-proj-$UNIQUE_NAME @@ -29,5 +35,5 @@ resources: create_database_objects_if_missing: true timeseries_key: $TIMESERIES_KEY new_pipeline_spec: - storage_catalog: main - storage_schema: pipeline_storage_$UNIQUE_NAME + storage_catalog: ${resources.schemas.pipeline_storage.catalog_name} + storage_schema: ${resources.schemas.pipeline_storage.name} diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt b/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt index 1b8c894bf9a..4011c1da1dc 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt @@ -1,7 +1,3 @@ -Creating pipeline storage schema: main.pipeline_storage_[UNIQUE_NAME] -{ - "full_name": "main.pipeline_storage_[UNIQUE_NAME]" -} >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-synced-table-[UNIQUE_NAME]/default/files... @@ -12,7 +8,7 @@ Deployment complete! >>> [CLI] bundle plan recreate postgres_synced_tables.my_table -Plan: 1 to add, 0 to change, 1 to delete, 2 unchanged +Plan: 1 to add, 0 to change, 1 to delete, 3 unchanged contains error: 'Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged' not found in the output. >>> [CLI] bundle deploy @@ -26,6 +22,10 @@ The following resources will be deleted: delete resources.postgres_catalogs.my_catalog delete resources.postgres_projects.my_project delete resources.postgres_synced_tables.my_table + delete resources.schemas.pipeline_storage + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.pipeline_storage This action will result in the deletion of the following Lakebase projects along with all their branches, databases, and endpoints. All data stored in them will be permanently lost: @@ -35,4 +35,3 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! -Cleaning up pipeline storage schema diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/script b/acceptance/bundle/resources/postgres_synced_tables/recreate/script index 796f1ad3910..f1d02b42237 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/script +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/script @@ -1,11 +1,5 @@ -# UC schema for the pipeline's internal state. -echo "Creating pipeline storage schema: main.pipeline_storage_$UNIQUE_NAME" -$CLI schemas create pipeline_storage_$UNIQUE_NAME main -o json | jq '{full_name}' - cleanup() { trace $CLI bundle destroy --auto-approve - echo "Cleaning up pipeline storage schema" - $CLI schemas delete main.pipeline_storage_$UNIQUE_NAME || true rm -f out.requests.txt } trap cleanup EXIT diff --git a/acceptance/bundle/resources/postgres_synced_tables/test.toml b/acceptance/bundle/resources/postgres_synced_tables/test.toml index db9fe92549a..66f0811b343 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/test.toml @@ -24,12 +24,3 @@ Order = 1000 Old = '/operations/[A-Za-z0-9+/=-]+' New = '/operations/[OPERATION_ID]' Order = 2000 - -# Fake SQL endpoint for local tests. The script runs CREATE TABLE on the source. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - -[[Server]] -Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" -Response.Body = '{"status": "OK"}' From 704556478a59769ebc520fc597d9cd1a95454582 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 19 May 2026 10:40:42 +0200 Subject: [PATCH 26/30] acc: per-test source table for postgres_synced_tables Found on aws-prod-ucws: a deploy targeting samples.nyctaxi.trips as the synced-table source returns Cannot create more than 20 synced database table(s) per source table. (400 BAD_REQUEST) There's a hard server-side limit of 20 synced tables per source, and samples.nyctaxi.trips is depleted on shared workspaces. The original script created a per-test source table for this reason (see synced_database_tables/basic for the same workaround). I removed it chasing simplicity; this restores it. The pipeline-storage schema stays bundle-managed (the schemas resource added in the previous commit); only the source-table side goes back to being script-managed. --- .../basic/databricks.yml.tmpl | 2 +- .../basic/out.requests.direct.json | 2 +- .../basic/out.requests.terraform.json | 2 +- .../postgres_synced_tables/basic/output.txt | 5 +++++ .../resources/postgres_synced_tables/basic/script | 14 ++++++++++++++ .../recreate/databricks.yml.tmpl | 2 +- .../postgres_synced_tables/recreate/output.txt | 5 +++++ .../postgres_synced_tables/recreate/script | 14 +++++++++++++- .../resources/postgres_synced_tables/test.toml | 9 +++++++++ 9 files changed, 50 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl index d081061a6a5..9d14a96d86e 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/databricks.yml.tmpl @@ -27,7 +27,7 @@ resources: postgres_synced_tables: my_table: synced_table_id: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_synced - source_table_full_name: samples.nyctaxi.trips + source_table_full_name: main.source_$UNIQUE_NAME.trips_source primary_key_columns: ["tpep_pickup_datetime"] scheduling_policy: SNAPSHOT postgres_database: appdb diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.direct.json b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.direct.json index 298f46d2d01..4e85c53cd99 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.direct.json +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.direct.json @@ -44,7 +44,7 @@ "tpep_pickup_datetime" ], "scheduling_policy": "SNAPSHOT", - "source_table_full_name": "samples.nyctaxi.trips" + "source_table_full_name": "main.source_[UNIQUE_NAME].trips_source" } } } diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.terraform.json b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.terraform.json index 298f46d2d01..4e85c53cd99 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.terraform.json +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.requests.terraform.json @@ -44,7 +44,7 @@ "tpep_pickup_datetime" ], "scheduling_policy": "SNAPSHOT", - "source_table_full_name": "samples.nyctaxi.trips" + "source_table_full_name": "main.source_[UNIQUE_NAME].trips_source" } } } diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt b/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt index 101c3d9fd46..1e7c080cc8a 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt @@ -1,3 +1,7 @@ +Creating temporary source table: main.source_[UNIQUE_NAME].trips_source +{ + "full_name": "main.source_[UNIQUE_NAME]" +} >>> [CLI] bundle validate Name: deploy-postgres-synced-table-[UNIQUE_NAME] @@ -91,3 +95,4 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! +Cleaning up temporary source table diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/script b/acceptance/bundle/resources/postgres_synced_tables/basic/script index 7e28ab439a8..7a65a203da4 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/script +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/script @@ -1,7 +1,21 @@ envsubst < databricks.yml.tmpl > databricks.yml +# Create a per-test source schema + table. We can't read samples.nyctaxi.trips +# directly because shared workspaces hit the "20 synced tables per source table" +# limit (same reason synced_database_tables takes this approach). +echo "Creating temporary source table: main.source_$UNIQUE_NAME.trips_source" +$CLI schemas create source_$UNIQUE_NAME main -o json | jq '{full_name}' +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.0/sql/statements/" --json "{ + \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", + \"statement\": \"CREATE TABLE main.source_$UNIQUE_NAME.trips_source AS SELECT * FROM samples.nyctaxi.trips LIMIT 10\", + \"wait_timeout\": \"45s\" + }" > /dev/null + cleanup() { trace $CLI bundle destroy --auto-approve + echo "Cleaning up temporary source table" + $CLI tables delete main.source_$UNIQUE_NAME.trips_source || true + $CLI schemas delete main.source_$UNIQUE_NAME || true rm -f out.requests.txt } trap cleanup EXIT diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl index c0f3f38ba12..40a350d53fa 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/databricks.yml.tmpl @@ -27,7 +27,7 @@ resources: postgres_synced_tables: my_table: synced_table_id: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_synced - source_table_full_name: samples.nyctaxi.trips + source_table_full_name: main.source_$UNIQUE_NAME.trips_source primary_key_columns: ["tpep_pickup_datetime"] scheduling_policy: SNAPSHOT postgres_database: appdb diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt b/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt index 4011c1da1dc..d77dab8cb73 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/output.txt @@ -1,3 +1,7 @@ +Creating temporary source table: main.source_[UNIQUE_NAME].trips_source +{ + "full_name": "main.source_[UNIQUE_NAME]" +} >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-synced-table-[UNIQUE_NAME]/default/files... @@ -35,3 +39,4 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! +Cleaning up temporary source table diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/script b/acceptance/bundle/resources/postgres_synced_tables/recreate/script index f1d02b42237..095d2a0c6c6 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/script +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/script @@ -1,5 +1,17 @@ +# Per-test source table to avoid the 20-synced-tables-per-source-table limit. +echo "Creating temporary source table: main.source_$UNIQUE_NAME.trips_source" +$CLI schemas create source_$UNIQUE_NAME main -o json | jq '{full_name}' +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.0/sql/statements/" --json "{ + \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", + \"statement\": \"CREATE TABLE main.source_$UNIQUE_NAME.trips_source AS SELECT * FROM samples.nyctaxi.trips LIMIT 10\", + \"wait_timeout\": \"45s\" + }" > /dev/null + cleanup() { trace $CLI bundle destroy --auto-approve + echo "Cleaning up temporary source table" + $CLI tables delete main.source_$UNIQUE_NAME.trips_source || true + $CLI schemas delete main.source_$UNIQUE_NAME || true rm -f out.requests.txt } trap cleanup EXIT @@ -10,7 +22,7 @@ trace $CLI bundle deploy # Toggle a recreate-on-change field; plan must show delete + create. # We toggle timeseries_key (not scheduling_policy) so we don't need CDF -# on the source — samples.nyctaxi.trips is read-only. +# on the source. export TIMESERIES_KEY=tpep_dropoff_datetime envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle plan | contains.py "Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged" diff --git a/acceptance/bundle/resources/postgres_synced_tables/test.toml b/acceptance/bundle/resources/postgres_synced_tables/test.toml index 66f0811b343..394df07ae0b 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/test.toml @@ -24,3 +24,12 @@ Order = 1000 Old = '/operations/[A-Za-z0-9+/=-]+' New = '/operations/[OPERATION_ID]' Order = 2000 + +# Fake SQL endpoint for the per-test source table create in script. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{"status": "OK"}' From 495a076f15e0000363eb12a4a4b9a72cb52c4a40 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 19 May 2026 10:42:30 +0200 Subject: [PATCH 27/30] acc: normalize synced-table GET output for cloud + local The previous jq filter deleted only the random fields (timestamps, uid, pipeline_id, message). It left detailed_state, which is timing- dependent on cloud: real workspaces are still in SYNCED_TABLE_PROVISIONING_PIPELINE_RESOURCES at the GET, while the fake testserver always returns SYNCED_TABLE_ONLINE. The cloud response also carries ongoing_sync_progress and project which the fake doesn't. Switch to projecting just the deterministic identity + UC provisioning state, which is ACTIVE in both environments. --- .../resources/postgres_synced_tables/basic/output.txt | 5 +---- .../bundle/resources/postgres_synced_tables/basic/script | 7 +++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt b/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt index 1e7c080cc8a..f6c31a01356 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/output.txt @@ -45,10 +45,7 @@ Deployment complete! >>> [CLI] postgres get-synced-table synced_tables/lakebase_test_[UNIQUE_NAME].public.trips_synced { "name": "synced_tables/lakebase_test_[UNIQUE_NAME].public.trips_synced", - "status": { - "detailed_state": "SYNCED_TABLE_ONLINE", - "unity_catalog_provisioning_state": "ACTIVE" - } + "unity_catalog_provisioning_state": "ACTIVE" } >>> [CLI] bundle summary diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/script b/acceptance/bundle/resources/postgres_synced_tables/basic/script index 7a65a203da4..c487aab02b7 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/script +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/script @@ -27,8 +27,11 @@ trace $CLI bundle summary rm -f out.requests.txt trace $CLI bundle deploy -# Hide volatile fields so cloud and local match. -trace $CLI postgres get-synced-table "synced_tables/lakebase_test_${UNIQUE_NAME}.public.trips_synced" | jq 'del(.create_time, .update_time, .uid, .status.pipeline_id, .status.message)' +# Keep only the deterministic identity + provisioning-state. detailed_state +# varies with pipeline-provisioning timing on cloud, ongoing_sync_progress and +# project only appear there, and create_time/update_time/uid/pipeline_id are +# random per run. +trace $CLI postgres get-synced-table "synced_tables/lakebase_test_${UNIQUE_NAME}.public.trips_synced" | jq '{name, unity_catalog_provisioning_state: .status.unity_catalog_provisioning_state}' trace $CLI bundle summary From 73eabf1fd4fe0df91c99dee822beca945b5fd0dd Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 19 May 2026 13:11:53 +0200 Subject: [PATCH 28/30] changelog: align postgres_synced_tables wording with docs --- NEXT_CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index b2143f3831b..b5bdacc2189 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -11,6 +11,6 @@ ### Bundles * Make sure warnings asking for approval are understood by agents ([#5239](https://github.com/databricks/cli/pull/5239)) * Add `postgres_catalogs` resource to bind a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch ([#5265](https://github.com/databricks/cli/pull/5265)). -* Add `postgres_synced_tables` resource to manage Lakebase synced tables that replicate Unity Catalog Delta tables into Postgres. +* Add `postgres_synced_tables` resource to sync a Unity Catalog Delta table into a Postgres table on a Lakebase Autoscaling branch. ### Dependency updates From 36ae5e87c93361d380393eb0bba4dfda854037df Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 19 May 2026 13:23:13 +0200 Subject: [PATCH 29/30] validation: regenerate enum + required fields for postgres_synced_tables --- bundle/internal/validation/generated/enum_fields.go | 2 ++ bundle/internal/validation/generated/required_fields.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index 2f1593a890e..bd481f9f2e3 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -184,6 +184,8 @@ var EnumFields = map[string][]string{ "resources.postgres_projects.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, + "resources.postgres_synced_tables.*.scheduling_policy": {"CONTINUOUS", "SNAPSHOT", "TRIGGERED"}, + "resources.quality_monitors.*.custom_metrics[*].type": {"CUSTOM_METRIC_TYPE_AGGREGATE", "CUSTOM_METRIC_TYPE_DERIVED", "CUSTOM_METRIC_TYPE_DRIFT"}, "resources.quality_monitors.*.inference_log.problem_type": {"PROBLEM_TYPE_CLASSIFICATION", "PROBLEM_TYPE_REGRESSION"}, "resources.quality_monitors.*.schedule.pause_status": {"PAUSED", "UNPAUSED", "UNSPECIFIED"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index ad2edcd333b..ae6da95317f 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -227,6 +227,8 @@ var RequiredFields = map[string][]string{ "resources.postgres_projects.*": {"project_id"}, "resources.postgres_projects.*.permissions[*]": {"level"}, + "resources.postgres_synced_tables.*": {"synced_table_id"}, + "resources.quality_monitors.*": {"assets_dir", "output_schema_name", "table_name"}, "resources.quality_monitors.*.custom_metrics[*]": {"definition", "input_columns", "name", "output_data_type", "type"}, "resources.quality_monitors.*.inference_log": {"granularities", "model_id_col", "prediction_col", "problem_type", "timestamp_col"}, From e8f59dac076018aafaf801e2f7cd426eb98bed0c Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 19 May 2026 14:07:55 +0200 Subject: [PATCH 30/30] dresources: mirror postgres_catalogs pattern for synced_table_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the TrimSyncedTablesPrefix helper + unit test. Source synced_table_id inline in RemapState with strings.TrimPrefix(remote.Name, "synced_tables/"), mirroring postgres_catalogs.RemapState which sources catalog_id inline via remote.Status.CatalogId. The synced-table API doesn't expose the user-facing id as a named field on either SyncedTable or its Status — it only appears as the trailing component of remote.Name, so the prefix strip is structurally necessary. The docstring on the field calls out the asymmetry with postgres_catalogs. Also: move PostgresSyncedTables above PostgresOperations in the testserver struct + constructor (so the operations map stays after the resource maps). Also: link the changelog entry to #5268. --- NEXT_CHANGELOG.md | 2 +- .../dresources/postgres_synced_table.go | 16 +++++++++++----- bundle/direct/dresources/util.go | 7 ------- bundle/direct/dresources/util_test.go | 19 ------------------- libs/testserver/fake_workspace.go | 4 ++-- 5 files changed, 14 insertions(+), 34 deletions(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index b5bdacc2189..fb2276cf44d 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -11,6 +11,6 @@ ### Bundles * Make sure warnings asking for approval are understood by agents ([#5239](https://github.com/databricks/cli/pull/5239)) * Add `postgres_catalogs` resource to bind a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch ([#5265](https://github.com/databricks/cli/pull/5265)). -* Add `postgres_synced_tables` resource to sync a Unity Catalog Delta table into a Postgres table on a Lakebase Autoscaling branch. +* Add `postgres_synced_tables` resource to sync a Unity Catalog Delta table into a Postgres table on a Lakebase Autoscaling branch ([#5268](https://github.com/databricks/cli/pull/5268)). ### Dependency updates diff --git a/bundle/direct/dresources/postgres_synced_table.go b/bundle/direct/dresources/postgres_synced_table.go index 3006f47282b..4243f071d7f 100644 --- a/bundle/direct/dresources/postgres_synced_table.go +++ b/bundle/direct/dresources/postgres_synced_table.go @@ -2,6 +2,7 @@ package dresources import ( "context" + "strings" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" @@ -26,12 +27,17 @@ func (*ResourcePostgresSyncedTable) PrepareState(input *resources.PostgresSynced } func (*ResourcePostgresSyncedTable) RemapState(remote *postgres.SyncedTable) *PostgresSyncedTableState { + // Unlike postgres_catalogs (which has Status.CatalogId), the synced-table + // API doesn't expose the user-facing id as a named field. It only appears + // as the trailing component of remote.Name, so we strip the constant + // "synced_tables/" prefix. + // + // GET does not return the spec today (only status). Return an empty spec + // and rely on the spec:input_only classifications generated from the + // OpenAPI schema to suppress phantom drift until the backend starts + // echoing spec values on GET. return &PostgresSyncedTableState{ - SyncedTableId: TrimSyncedTablesPrefix(remote.Name), - - // GET does not return the spec, only the status. Match the postgres_project / - // postgres_branch pattern: return an empty (non-nil) spec so field-level diffing - // works correctly and remote drift on spec fields is invisible. + SyncedTableId: strings.TrimPrefix(remote.Name, "synced_tables/"), SyncedTableSyncedTableSpec: postgres.SyncedTableSyncedTableSpec{ Branch: "", CreateDatabaseObjectsIfMissing: false, diff --git a/bundle/direct/dresources/util.go b/bundle/direct/dresources/util.go index 60536c9b389..3bd0ab4ec73 100644 --- a/bundle/direct/dresources/util.go +++ b/bundle/direct/dresources/util.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "regexp" - "strings" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" @@ -74,9 +73,3 @@ func truncateAtIndex(path string) string { } return p.Prefix(1).String() } - -// TrimSyncedTablesPrefix extracts the user-facing synced table id from the API name. -// E.g. "synced_tables/main.public.trips" -> "main.public.trips". -func TrimSyncedTablesPrefix(name string) string { - return strings.TrimPrefix(name, "synced_tables/") -} diff --git a/bundle/direct/dresources/util_test.go b/bundle/direct/dresources/util_test.go index 4b601550409..bbf04717099 100644 --- a/bundle/direct/dresources/util_test.go +++ b/bundle/direct/dresources/util_test.go @@ -120,22 +120,3 @@ func TestParsePostgresName(t *testing.T) { }) } } - -func TestTrimSyncedTablesPrefix(t *testing.T) { - tests := []struct { - name string - in string - want string - }{ - {"happy path", "synced_tables/main.public.trips_synced", "main.public.trips_synced"}, - {"missing prefix is returned unchanged", "main.public.trips_synced", "main.public.trips_synced"}, - {"empty string", "", ""}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := TrimSyncedTablesPrefix(tt.in); got != tt.want { - t.Errorf("TrimSyncedTablesPrefix(%q) = %q, want %q", tt.in, got, tt.want) - } - }) - } -} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index c236b54d69c..88e75b35137 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -172,8 +172,8 @@ type FakeWorkspace struct { PostgresBranches map[string]postgres.Branch PostgresEndpoints map[string]postgres.Endpoint PostgresCatalogs map[string]postgres.Catalog - PostgresOperations map[string]postgres.Operation PostgresSyncedTables map[string]postgres.SyncedTable + PostgresOperations map[string]postgres.Operation // clusterVenvs caches Python venvs per existing cluster ID, // matching cloud behavior where libraries are cached on running clusters. @@ -302,8 +302,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { PostgresBranches: map[string]postgres.Branch{}, PostgresEndpoints: map[string]postgres.Endpoint{}, PostgresCatalogs: map[string]postgres.Catalog{}, - PostgresOperations: map[string]postgres.Operation{}, PostgresSyncedTables: map[string]postgres.SyncedTable{}, + PostgresOperations: map[string]postgres.Operation{}, clusterVenvs: map[string]*clusterEnv{}, Alerts: map[string]sql.AlertV2{}, Experiments: map[string]ml.GetExperimentResponse{},