diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 15a7a3de2cd..527aee74cba 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -17,6 +17,7 @@ * direct: Cluster resize now falls back to regular update if resize fails due to `INVALID_STATE` ([#5716](https://github.com/databricks/cli/pull/5716)). * `bundle generate dashboard` now honors the `--key` flag when naming the generated resource, and rejects combining `--existing-path`, `--existing-id`, and `--resource` instead of silently ignoring all but one ([#5492](https://github.com/databricks/cli/pull/5492)). * Fixed `bundle deployment migrate` failing on `model_serving_endpoints`/`database_instances` with permissions (regression since v1.5.0) ([#5775](https://github.com/databricks/cli/pull/5775)). + * Support `replace_existing: true` on `postgres_databases` and `postgres_roles` so bundles can take over a database or role that already exists on a Lakebase branch instead of failing with `ALREADY_EXISTS` ([#5803](https://github.com/databricks/cli/pull/5803)). * After a terraform deploy, the CLI now dry-runs a migration to the direct engine (writing nothing locally or remotely) and reports the outcome via telemetry, warning if the migration could not be completed ([#5797](https://github.com/databricks/cli/pull/5797)). ### Dependency updates diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 7f9cf9c94f5..0beec53332a 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2940,6 +2940,7 @@ resources.postgres_databases.*.modified_status string INPUT resources.postgres_databases.*.name string REMOTE resources.postgres_databases.*.parent string ALL resources.postgres_databases.*.postgres_database string ALL +resources.postgres_databases.*.replace_existing bool INPUT STATE resources.postgres_databases.*.role string ALL resources.postgres_databases.*.status *postgres.DatabaseDatabaseStatus REMOTE resources.postgres_databases.*.status.database_id string REMOTE @@ -3075,6 +3076,7 @@ resources.postgres_roles.*.modified_status string INPUT resources.postgres_roles.*.name string REMOTE resources.postgres_roles.*.parent string ALL resources.postgres_roles.*.postgres_role string ALL +resources.postgres_roles.*.replace_existing bool INPUT STATE resources.postgres_roles.*.role_id string ALL resources.postgres_roles.*.status *postgres.RoleRoleStatus REMOTE resources.postgres_roles.*.status.attributes *postgres.RoleAttributes REMOTE diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_databases/replace_existing/databricks.yml.tmpl new file mode 100644 index 00000000000..ae92537fea5 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/databricks.yml.tmpl @@ -0,0 +1,40 @@ +bundle: + name: replace-existing-database-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Database replace_existing" + pg_version: 16 + history_retention_duration: "604800s" + + postgres_branches: + main: + parent: ${resources.postgres_projects.my_project.id} + branch_id: main + no_expiry: true + + postgres_roles: + owner: + parent: ${resources.postgres_branches.main.id} + role_id: app-owner + postgres_role: app_owner + + # replace_existing takes over a database that already exists on the branch + # instead of erroring with ALREADY_EXISTS. The field is input-only and not + # surfaced in get-database, but it is visible in the recorded request body so + # the diff confirms it was sent. + postgres_databases: + my_database: + parent: ${resources.postgres_branches.main.id} + database_id: my-database + postgres_database: app_db + # The live API requires `role`. Declare an explicit role so the bundle is + # portable across users (the auto-created project-owner role's id is + # derived from the creator's identity). + role: ${resources.postgres_roles.owner.id} + replace_existing: true diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/out.requests.deploy.direct.json b/acceptance/bundle/resources/postgres_databases/replace_existing/out.requests.deploy.direct.json new file mode 100644 index 00000000000..c1dd61f0000 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/out.requests.deploy.direct.json @@ -0,0 +1,42 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases", + "q": { + "database_id": "my-database", + "replace_existing": "true" + }, + "body": { + "spec": { + "postgres_database": "app_db", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases/my-database" +} diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/out.requests.deploy.terraform.json b/acceptance/bundle/resources/postgres_databases/replace_existing/out.requests.deploy.terraform.json new file mode 100644 index 00000000000..186913bda05 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/out.requests.deploy.terraform.json @@ -0,0 +1,31 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases", + "q": { + "database_id": "my-database", + "replace_existing": "true" + }, + "body": { + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "spec": { + "postgres_database": "app_db", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases/my-database" +} diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml new file mode 100644 index 00000000000..48a09b93850 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false +RequiresUnityCatalog = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/output.txt b/acceptance/bundle/resources/postgres_databases/replace_existing/output.txt new file mode 100644 index 00000000000..af920827cb3 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/output.txt @@ -0,0 +1,63 @@ + +=== Deploy project, branch, role, and database +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/replace-existing-database-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Unbind the database: it now exists on the branch but is no longer tracked by the bundle +>>> [CLI] bundle deployment unbind my_database +Updating deployment state... + +=== Plan after unbind: the bundle wants to (re-)create the database +>>> [CLI] bundle plan +create postgres_databases.my_database + +Plan: 1 to add, 0 to change, 0 to delete, 3 unchanged + +=== Re-deploy: replace_existing adopts the already-existing database instead of erroring +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/replace-existing-database-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Confirm the database is under management again +>>> [CLI] postgres get-database projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases/my-database +{ + "database_id": "my-database", + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases/my-database", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "status": { + "database_id": "my-database", + "postgres_database": "app_db", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" + } +} + +>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_branches.main + delete resources.postgres_databases.my_database + delete resources.postgres_projects.my_project + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.main + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.my_database + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/replace-existing-database-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/script b/acceptance/bundle/resources/postgres_databases/replace_existing/script new file mode 100644 index 00000000000..83139ed1b84 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/script @@ -0,0 +1,27 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy project, branch, role, and database" +trace $CLI bundle deploy + +title "Unbind the database: it now exists on the branch but is no longer tracked by the bundle" +# Mirrors a database that already exists on the branch but is not created or +# owned by this bundle. +trace $CLI bundle deployment unbind my_database + +title "Plan after unbind: the bundle wants to (re-)create the database" +trace $CLI bundle plan + +title "Re-deploy: replace_existing adopts the already-existing database instead of erroring" +rm -f out.requests.txt +trace $CLI bundle deploy + +title "Confirm the database is under management again" +trace $CLI postgres get-database "projects/test-pg-proj-${UNIQUE_NAME}/branches/main/databases/my-database" | jq 'del(.create_time, .update_time)' + +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml new file mode 100644 index 00000000000..54a665b74ea --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml @@ -0,0 +1,3 @@ +# The pre-existing database is staged locally via `bundle deployment unbind`, +# so this is local-only. +Cloud = false diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_roles/replace_existing/databricks.yml.tmpl new file mode 100644 index 00000000000..55fa325caa5 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/databricks.yml.tmpl @@ -0,0 +1,30 @@ +bundle: + name: replace-existing-role-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Role replace_existing" + pg_version: 16 + history_retention_duration: 604800s + + postgres_branches: + main: + parent: ${resources.postgres_projects.my_project.id} + branch_id: main + no_expiry: true + + # replace_existing takes over a role that already exists on the branch (e.g. + # inherited from the parent branch) instead of erroring with ALREADY_EXISTS. + # The field is input-only and not surfaced in get-role, but it is visible in + # the recorded request body so the diff confirms it was sent. + postgres_roles: + my_role: + parent: ${resources.postgres_branches.main.id} + role_id: test-role + postgres_role: app_role + replace_existing: true diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/out.requests.deploy.direct.json b/acceptance/bundle/resources/postgres_roles/replace_existing/out.requests.deploy.direct.json new file mode 100644 index 00000000000..5f289588af5 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/out.requests.deploy.direct.json @@ -0,0 +1,33 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles", + "q": { + "replace_existing": "true", + "role_id": "test-role" + }, + "body": { + "spec": { + "postgres_role": "app_role" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role" +} diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/out.requests.deploy.terraform.json b/acceptance/bundle/resources/postgres_roles/replace_existing/out.requests.deploy.terraform.json new file mode 100644 index 00000000000..2679673de46 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/out.requests.deploy.terraform.json @@ -0,0 +1,26 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles", + "q": { + "replace_existing": "true", + "role_id": "test-role" + }, + "body": { + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "spec": { + "postgres_role": "app_role" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role" +} diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml new file mode 100644 index 00000000000..48a09b93850 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false +RequiresUnityCatalog = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/output.txt b/acceptance/bundle/resources/postgres_roles/replace_existing/output.txt new file mode 100644 index 00000000000..3579054ce40 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/output.txt @@ -0,0 +1,64 @@ + +=== Deploy project, branch, and role +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/replace-existing-role-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Unbind the role: it now exists on the branch but is no longer tracked by the bundle +>>> [CLI] bundle deployment unbind my_role +Updating deployment state... + +=== Plan after unbind: the bundle wants to (re-)create the role +>>> [CLI] bundle plan +create postgres_roles.my_role + +Plan: 1 to add, 0 to change, 0 to delete, 2 unchanged + +=== Re-deploy: replace_existing adopts the already-existing role instead of erroring +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/replace-existing-role-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Confirm the role is under management again +>>> [CLI] postgres get-role projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "role_id": "test-role", + "status": { + "attributes": { + "bypassrls": false, + "createdb": false, + "createrole": false + }, + "auth_method": "PG_PASSWORD_SCRAM_SHA_256", + "identity_type": "IDENTITY_TYPE_UNSPECIFIED", + "postgres_role": "app_role", + "role_id": "test-role" + } +} + +>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_branches.main + delete resources.postgres_projects.my_project + delete resources.postgres_roles.my_role + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.main + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/replace-existing-role-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/script b/acceptance/bundle/resources/postgres_roles/replace_existing/script new file mode 100644 index 00000000000..9fce33a5b2a --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/script @@ -0,0 +1,27 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy project, branch, and role" +trace $CLI bundle deploy + +title "Unbind the role: it now exists on the branch but is no longer tracked by the bundle" +# Mirrors an inherited role on a child branch: present on the branch, but not +# created or owned by this bundle. +trace $CLI bundle deployment unbind my_role + +title "Plan after unbind: the bundle wants to (re-)create the role" +trace $CLI bundle plan + +title "Re-deploy: replace_existing adopts the already-existing role instead of erroring" +rm -f out.requests.txt +trace $CLI bundle deploy + +title "Confirm the role is under management again" +trace $CLI postgres get-role "projects/test-pg-proj-${UNIQUE_NAME}/branches/main/roles/test-role" | jq 'del(.create_time, .update_time)' + +trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml new file mode 100644 index 00000000000..8c557504819 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml @@ -0,0 +1,3 @@ +# The pre-existing role is staged locally via `bundle deployment unbind` (the +# testserver does not model branch role inheritance), so this is local-only. +Cloud = false diff --git a/bundle/config/resources/postgres_database.go b/bundle/config/resources/postgres_database.go index 1830e8586d3..878aa478632 100644 --- a/bundle/config/resources/postgres_database.go +++ b/bundle/config/resources/postgres_database.go @@ -20,6 +20,11 @@ type PostgresDatabaseConfig struct { // Parent is the branch containing this database. Format: "projects/{project_id}/branches/{branch_id}" Parent string `json:"parent"` + + // ReplaceExisting, when true, takes over an existing database with the same ID + // instead of returning ALREADY_EXISTS. Used to manage a database that already + // exists on the branch. Input-only: not returned by the GET API. + ReplaceExisting bool `json:"replace_existing,omitempty"` } func (c *PostgresDatabaseConfig) UnmarshalJSON(b []byte) error { diff --git a/bundle/config/resources/postgres_role.go b/bundle/config/resources/postgres_role.go index 34c2f5fcae4..9bb7ada907a 100644 --- a/bundle/config/resources/postgres_role.go +++ b/bundle/config/resources/postgres_role.go @@ -20,6 +20,12 @@ type PostgresRoleConfig struct { // Parent is the branch containing this role. Format: "projects/{project_id}/branches/{branch_id}" Parent string `json:"parent"` + + // ReplaceExisting, when true, takes over an existing role with the same ID + // instead of returning ALREADY_EXISTS. Used to manage a role that already + // exists on the branch (e.g. inherited from the parent branch). Input-only: + // not returned by the GET API. + ReplaceExisting bool `json:"replace_existing,omitempty"` } func (c *PostgresRoleConfig) UnmarshalJSON(b []byte) error { diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_database.go b/bundle/deploy/terraform/tfdyn/convert_postgres_database.go index cb89f4aed33..cb2f6e88de2 100644 --- a/bundle/deploy/terraform/tfdyn/convert_postgres_database.go +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_database.go @@ -15,7 +15,7 @@ func (c postgresDatabaseConverter) Convert(ctx context.Context, key string, vin // The bundle config has flattened DatabaseSpec fields at the top level. // Terraform expects them nested in a "spec" block. specFields := specFieldNames(schema.ResourcePostgresDatabaseSpec{}) - topLevelFields := []string{"database_id", "parent"} + topLevelFields := []string{"database_id", "parent", "replace_existing"} // Build the spec block from the flattened fields specMap := make(map[string]dyn.Value) diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_role.go b/bundle/deploy/terraform/tfdyn/convert_postgres_role.go index d1a2449f22e..0704db4e50d 100644 --- a/bundle/deploy/terraform/tfdyn/convert_postgres_role.go +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_role.go @@ -15,7 +15,7 @@ func (c postgresRoleConverter) Convert(ctx context.Context, key string, vin dyn. // The bundle config has flattened RoleRoleSpec fields at the top level. // Terraform expects them nested in a "spec" block. specFields := specFieldNames(schema.ResourcePostgresRoleSpec{}) - topLevelFields := []string{"role_id", "parent"} + topLevelFields := []string{"role_id", "parent", "replace_existing"} // Build the spec block from the flattened fields specMap := make(map[string]dyn.Value) diff --git a/bundle/direct/dresources/postgres_database.go b/bundle/direct/dresources/postgres_database.go index 4b8e705cc0b..adfd7326eec 100644 --- a/bundle/direct/dresources/postgres_database.go +++ b/bundle/direct/dresources/postgres_database.go @@ -51,14 +51,20 @@ func (*ResourcePostgresDatabase) PrepareState(input *resources.PostgresDatabase) return &PostgresDatabaseState{ DatabaseId: input.DatabaseId, Parent: input.Parent, + ReplaceExisting: input.ReplaceExisting, DatabaseDatabaseSpec: input.DatabaseDatabaseSpec, } } func (*ResourcePostgresDatabase) RemapState(remote *PostgresDatabaseRemote) *PostgresDatabaseState { return &PostgresDatabaseState{ - DatabaseId: remote.DatabaseId, - Parent: remote.Parent, + DatabaseId: remote.DatabaseId, + Parent: remote.Parent, + + // replace_existing is a create-time-only flag; the GET API never returns + // it, so RemapState leaves it false. + ReplaceExisting: false, + DatabaseDatabaseSpec: remote.DatabaseDatabaseSpec, } } @@ -107,9 +113,7 @@ func (r *ResourcePostgresDatabase) DoCreate(ctx context.Context, config *Postgre UpdateTime: nil, ForceSendFields: nil, }, - // ReplaceExisting adopts an existing database with the same ID instead of - // returning ALREADY_EXISTS. Not exposed in bundle config, so always false. - ReplaceExisting: false, + ReplaceExisting: config.ReplaceExisting, ForceSendFields: nil, }) if err != nil { diff --git a/bundle/direct/dresources/postgres_role.go b/bundle/direct/dresources/postgres_role.go index 204474ef108..99f847dc56d 100644 --- a/bundle/direct/dresources/postgres_role.go +++ b/bundle/direct/dresources/postgres_role.go @@ -29,6 +29,10 @@ type PostgresRoleState struct { // Parent is "projects/{project_id}/branches/{branch_id}". Parent string `json:"parent"` + + // ReplaceExisting takes over an existing role with the same ID on create + // instead of returning ALREADY_EXISTS. Input-only: not returned by the GET API. + ReplaceExisting bool `json:"replace_existing,omitempty"` } // PostgresRoleRemote is the return type for DoRead. It embeds RoleRoleSpec so that @@ -70,16 +74,22 @@ func (*ResourcePostgresRole) New(client *databricks.WorkspaceClient) *ResourcePo func (*ResourcePostgresRole) PrepareState(input *resources.PostgresRole) *PostgresRoleState { return &PostgresRoleState{ - RoleId: input.RoleId, - Parent: input.Parent, - RoleRoleSpec: input.RoleRoleSpec, + RoleId: input.RoleId, + Parent: input.Parent, + ReplaceExisting: input.ReplaceExisting, + RoleRoleSpec: input.RoleRoleSpec, } } func (*ResourcePostgresRole) RemapState(remote *PostgresRoleRemote) *PostgresRoleState { return &PostgresRoleState{ - RoleId: remote.RoleId, - Parent: remote.Parent, + RoleId: remote.RoleId, + Parent: remote.Parent, + + // replace_existing is a create-time-only flag; the GET API never returns + // it, so RemapState leaves it false. + ReplaceExisting: false, + RoleRoleSpec: remote.RoleRoleSpec, } } @@ -128,9 +138,7 @@ func (r *ResourcePostgresRole) DoCreate(ctx context.Context, config *PostgresRol UpdateTime: nil, ForceSendFields: nil, }, - // ReplaceExisting adopts an existing role with the same ID instead of - // returning ALREADY_EXISTS. Not exposed in bundle config, so always false. - ReplaceExisting: false, + ReplaceExisting: config.ReplaceExisting, ForceSendFields: nil, }) if err != nil { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 6c8cd3973ed..3c264b70959 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -641,6 +641,10 @@ resources: reason: id_field - field: database_id reason: id_field + ignore_local_changes: + # replace_existing only takes effect on create; toggling it later is a no-op. + - field: replace_existing + reason: "input_only; cannot be updated after create" postgres_endpoints: provided_id_fields: @@ -705,6 +709,10 @@ resources: reason: immutable postgres_roles: + ignore_local_changes: + # replace_existing only takes effect on create; toggling it later is a no-op. + - field: replace_existing + reason: "input_only; cannot be updated after create" recreate_on_changes: # parent and role_id are immutable (together they form the hierarchical name). - field: parent diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 3474450ed3e..ac42db5d68a 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -38,9 +38,15 @@ var knownMissingInRemoteType = map[string][]string{ "postgres_branches": { "replace_existing", }, + "postgres_databases": { + "replace_existing", + }, "postgres_endpoints": { "replace_existing", }, + "postgres_roles": { + "replace_existing", + }, "postgres_projects": { "purge_on_delete", }, diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index a868bdc98f7..74d2e03f193 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1359,7 +1359,7 @@ resources: PLACEHOLDER "replace_existing": "description": |- - PLACEHOLDER + When true, take over an existing branch with the same ID instead of failing with an ALREADY_EXISTS error. Use it to bring the implicitly-created production branch of a new Lakebase project under bundle management. Only takes effect when the branch is created. "source_branch": "description": |- PLACEHOLDER @@ -1407,6 +1407,9 @@ resources: "postgres_database": "description": |- PLACEHOLDER + "replace_existing": + "description": |- + When true, take over an existing database with the same ID instead of failing with an ALREADY_EXISTS error. Use it to bring a database that already exists on the branch under bundle management. Only takes effect when the database is created. "role": "description": |- PLACEHOLDER @@ -1443,7 +1446,7 @@ resources: PLACEHOLDER "replace_existing": "description": |- - PLACEHOLDER + When true, take over an existing endpoint with the same ID instead of failing with an ALREADY_EXISTS error. Use it to bring the implicitly-created primary read-write endpoint of a branch under bundle management. Only takes effect when the endpoint is created. "settings": "description": |- PLACEHOLDER @@ -1539,6 +1542,9 @@ resources: "postgres_role": "description": |- The name of the Postgres role. Required when creating the role. + "replace_existing": + "description": |- + When true, take over an existing role with the same ID instead of failing with an ALREADY_EXISTS error. Use it to bring a role that already exists on the branch (for example, one inherited from the parent branch) under bundle management. Only takes effect when the role is created. "role_id": "description": |- The user-specified role ID; becomes the final component of the role's resource name. Must be 4-63 characters, lowercase letters, numbers, and hyphens (RFC 1123). diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 87154f7945f..2ecd542e6ec 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1562,6 +1562,7 @@ "$ref": "#/$defs/string" }, "replace_existing": { + "description": "When true, take over an existing branch with the same ID instead of failing with an ALREADY_EXISTS error. Use it to bring the implicitly-created production branch of a new Lakebase project under bundle management. Only takes effect when the branch is created.", "$ref": "#/$defs/bool" }, "source_branch": { @@ -1641,6 +1642,10 @@ "postgres_database": { "$ref": "#/$defs/string" }, + "replace_existing": { + "description": "When true, take over an existing database with the same ID instead of failing with an ALREADY_EXISTS error. Use it to bring a database that already exists on the branch under bundle management. Only takes effect when the database is created.", + "$ref": "#/$defs/bool" + }, "role": { "$ref": "#/$defs/string" } @@ -1691,6 +1696,7 @@ "$ref": "#/$defs/string" }, "replace_existing": { + "description": "When true, take over an existing endpoint with the same ID instead of failing with an ALREADY_EXISTS error. Use it to bring the implicitly-created primary read-write endpoint of a branch under bundle management. Only takes effect when the endpoint is created.", "$ref": "#/$defs/bool" }, "settings": { @@ -1802,6 +1808,10 @@ "description": "The name of the Postgres role. Required when creating the role.", "$ref": "#/$defs/string" }, + "replace_existing": { + "description": "When true, take over an existing role with the same ID instead of failing with an ALREADY_EXISTS error. Use it to bring a role that already exists on the branch (for example, one inherited from the parent branch) under bundle management. Only takes effect when the role is created.", + "$ref": "#/$defs/bool" + }, "role_id": { "description": "The user-specified role ID; becomes the final component of the role's resource name. Must be 4-63 characters, lowercase letters, numbers, and hyphens (RFC 1123).", "$ref": "#/$defs/string" diff --git a/bundle/terraform_dabs_map/generated.go b/bundle/terraform_dabs_map/generated.go index 5a0cf360e9d..39d08ecfbd8 100644 --- a/bundle/terraform_dabs_map/generated.go +++ b/bundle/terraform_dabs_map/generated.go @@ -21,12 +21,10 @@ package terraform_dabs_map // postgres_branches / databricks_postgres_branch: 1 tf-only // postgres_branches / databricks_postgres_branch: 1 unwraps // postgres_catalogs / databricks_postgres_catalog: 1 unwraps -// postgres_databases / databricks_postgres_database: 1 tf-only // postgres_databases / databricks_postgres_database: 1 unwraps // postgres_endpoints / databricks_postgres_endpoint: 1 unwraps // postgres_projects / databricks_postgres_project: 2 tf-only // postgres_projects / databricks_postgres_project: 1 unwraps -// postgres_roles / databricks_postgres_role: 1 tf-only // postgres_roles / databricks_postgres_role: 1 unwraps // postgres_synced_tables / databricks_postgres_synced_table: 1 unwraps // schemas / databricks_schema: 1 dabs-only @@ -565,17 +563,11 @@ var TerraformOnlyFields = map[string]FieldSet{ "postgres_branches": { "purge_on_delete": {}, }, - "postgres_databases": { - "replace_existing": {}, - }, "postgres_projects": { "initial_branch_spec": { "is_protected": {}, // databricks_postgres_project.*.initial_branch_spec.is_protected }, }, - "postgres_roles": { - "replace_existing": {}, - }, "schemas": { "force_destroy": {}, }, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 4be6356eceb..cc430c40577 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -1096,7 +1096,8 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/databases", func(req Request) any { parent := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] databaseID := req.URL.Query().Get("database_id") - return req.Workspace.PostgresDatabaseCreate(req, parent, databaseID) + replaceExisting := req.URL.Query().Get("replace_existing") == "true" + return req.Workspace.PostgresDatabaseCreate(req, parent, databaseID, replaceExisting) }) server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/databases", func(req Request) any { @@ -1123,7 +1124,8 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/roles", func(req Request) any { parent := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] roleID := req.URL.Query().Get("role_id") - return req.Workspace.PostgresRoleCreate(req, parent, roleID) + replaceExisting := req.URL.Query().Get("replace_existing") == "true" + return req.Workspace.PostgresRoleCreate(req, parent, roleID, replaceExisting) }) server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/roles", func(req Request) any { @@ -1173,7 +1175,8 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/roles", func(req Request) any { parent := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] roleID := req.URL.Query().Get("role_id") - return req.Workspace.PostgresRoleCreate(req, parent, roleID) + replaceExisting := req.URL.Query().Get("replace_existing") == "true" + return req.Workspace.PostgresRoleCreate(req, parent, roleID, replaceExisting) }) server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/roles", func(req Request) any { diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index 83b50213a30..33983dab681 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -719,7 +719,11 @@ func (s *FakeWorkspace) PostgresEndpointDelete(name string) Response { } // PostgresDatabaseCreate creates a new postgres database. -func (s *FakeWorkspace) PostgresDatabaseCreate(req Request, parent, databaseID string) Response { +// +// When replaceExisting is true, an existing database with the same ID is updated +// in place instead of returning ALREADY_EXISTS. This mirrors the backend behavior +// that lets users bring an already-existing database under management. +func (s *FakeWorkspace) PostgresDatabaseCreate(req Request, parent, databaseID string, replaceExisting bool) Response { defer s.LockUnlock()() if databaseID == "" { @@ -750,7 +754,8 @@ func (s *FakeWorkspace) PostgresDatabaseCreate(req Request, parent, databaseID s name := fmt.Sprintf("%s/databases/%s", parent, databaseID) - if _, exists := s.PostgresDatabases[name]; exists { + existing, exists := s.PostgresDatabases[name] + if exists && !replaceExisting { // The real Lakebase API returns 400 BAD_REQUEST (not 409) for a duplicate // create, the same as postgres_roles. Match it so the conflict a bundle hits // on a pre-existing database looks the same. @@ -758,6 +763,23 @@ func (s *FakeWorkspace) PostgresDatabaseCreate(req Request, parent, databaseID s } now := nowTime() + if exists { + // Preserve identifying / output-only fields; apply incoming spec to status. + status := &postgres.DatabaseDatabaseStatus{ + DatabaseId: databaseID, + PostgresDatabase: database.Spec.PostgresDatabase, + Role: database.Spec.Role, + } + existing.UpdateTime = now + existing.Status = status + existing.Spec = nil + s.PostgresDatabases[name] = existing + + return Response{ + Body: s.createOperationLocked(existing.Name, existing), + } + } + database.Name = name database.DatabaseId = databaseID database.Parent = parent @@ -1047,7 +1069,11 @@ func roleStatusFromSpec(spec *postgres.RoleRoleSpec) *postgres.RoleRoleStatus { } // PostgresRoleCreate creates a new postgres role. -func (s *FakeWorkspace) PostgresRoleCreate(req Request, parent, roleID string) Response { +// +// When replaceExisting is true, an existing role with the same ID is updated in +// place instead of returning ALREADY_EXISTS. This mirrors the backend behavior +// that lets users bring an inherited/pre-existing role under management. +func (s *FakeWorkspace) PostgresRoleCreate(req Request, parent, roleID string, replaceExisting bool) Response { defer s.LockUnlock()() // Check if parent branch exists @@ -1077,7 +1103,8 @@ func (s *FakeWorkspace) PostgresRoleCreate(req Request, parent, roleID string) R name := fmt.Sprintf("%s/roles/%s", parent, roleID) - if _, exists := s.PostgresRoles[name]; exists { + existing, exists := s.PostgresRoles[name] + if exists && !replaceExisting { // The real Lakebase API returns 400 BAD_REQUEST (not 409) for a duplicate // role, with this message (verified on dogfood 2026-06-10). Match it so the // conflict a bundle hits on an inherited/pre-existing role looks the same. @@ -1085,6 +1112,19 @@ func (s *FakeWorkspace) PostgresRoleCreate(req Request, parent, roleID string) R } now := nowTime() + if exists { + // Preserve identifying / output-only fields; apply incoming spec to status. + existing.UpdateTime = now + existing.Status = roleStatusFromSpec(role.Spec) + existing.Status.RoleId = roleID + existing.Spec = nil + s.PostgresRoles[name] = existing + + return Response{ + Body: s.createOperationLocked(existing.Name, existing), + } + } + role.Name = name role.RoleId = roleID role.Parent = parent