From e1688fe48cbd6dfb3a1f9bf36bddec34136cca79 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 3 Jul 2026 10:50:12 +0000 Subject: [PATCH 1/3] Validate required/unit-constrained UC fields early in bundle validate Reject missing sql_warehouse name, missing grant principal, and out-of-range catalog/schema custom_max_retention_hours at validate/plan time instead of letting them pass and fail later at deploy with low-context backend errors. DECO-27550 --- NEXT_CHANGELOG.md | 1 + .../empty_resources/empty_dict/output.txt | 4 + .../empty_resources/with_grants/output.txt | 4 + .../with_permissions/output.txt | 4 + .../grants_required_principal/databricks.yml | 21 ++++ .../grants_required_principal/out.test.toml | 3 + .../grants_required_principal/output.txt | 15 +++ .../validate/grants_required_principal/script | 1 + .../grants_required_principal/test.toml | 3 + .../retention_hours_range/databricks.yml | 20 ++++ .../retention_hours_range/out.test.toml | 3 + .../validate/retention_hours_range/output.txt | 19 ++++ .../validate/retention_hours_range/script | 1 + .../validate/retention_hours_range/test.toml | 3 + .../databricks.yml | 9 ++ .../sql_warehouse_required_name/out.test.toml | 3 + .../sql_warehouse_required_name/output.txt | 15 +++ .../sql_warehouse_required_name/script | 1 + bundle/config/validate/required.go | 99 +++++++++++++++++++ 19 files changed, 229 insertions(+) create mode 100644 acceptance/bundle/validate/grants_required_principal/databricks.yml create mode 100644 acceptance/bundle/validate/grants_required_principal/out.test.toml create mode 100644 acceptance/bundle/validate/grants_required_principal/output.txt create mode 100644 acceptance/bundle/validate/grants_required_principal/script create mode 100644 acceptance/bundle/validate/grants_required_principal/test.toml create mode 100644 acceptance/bundle/validate/retention_hours_range/databricks.yml create mode 100644 acceptance/bundle/validate/retention_hours_range/out.test.toml create mode 100644 acceptance/bundle/validate/retention_hours_range/output.txt create mode 100644 acceptance/bundle/validate/retention_hours_range/script create mode 100644 acceptance/bundle/validate/retention_hours_range/test.toml create mode 100644 acceptance/bundle/validate/sql_warehouse_required_name/databricks.yml create mode 100644 acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml create mode 100644 acceptance/bundle/validate/sql_warehouse_required_name/output.txt create mode 100644 acceptance/bundle/validate/sql_warehouse_required_name/script diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 57d93cb0fcd..e3f94f8035b 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -11,6 +11,7 @@ ### Bundles * `bundle generate job` now downloads workspace files referenced by `spark_python_task`, rewriting them to a relative path like it already does for notebooks. Git-sourced files and cloud URIs are left untouched ([#5799](https://github.com/databricks/cli/pull/5799)). + * `bundle validate` now fails early with a clear error when a `sql_warehouse` is missing a `name`, a grant is missing a `principal`, or a catalog/schema `custom_max_retention_hours` is outside the allowed range (0 or 168-720 hours), instead of passing validation and failing later at deploy. ### Dependency updates diff --git a/acceptance/bundle/validate/empty_resources/empty_dict/output.txt b/acceptance/bundle/validate/empty_resources/empty_dict/output.txt index efbf60ca1ae..c31d1773e88 100644 --- a/acceptance/bundle/validate/empty_resources/empty_dict/output.txt +++ b/acceptance/bundle/validate/empty_resources/empty_dict/output.txt @@ -160,6 +160,10 @@ app resource 'rname' should have either source_code_path or git_source field } === resources.sql_warehouses.rname === +Error: sql_warehouse name is required + at resources.sql_warehouses.rname + in databricks.yml:6:12 + { "sql_warehouses": { "rname": { diff --git a/acceptance/bundle/validate/empty_resources/with_grants/output.txt b/acceptance/bundle/validate/empty_resources/with_grants/output.txt index e9d371b33d5..2c7e4bd5217 100644 --- a/acceptance/bundle/validate/empty_resources/with_grants/output.txt +++ b/acceptance/bundle/validate/empty_resources/with_grants/output.txt @@ -201,6 +201,10 @@ Warning: unknown field: grants at resources.sql_warehouses.rname in databricks.yml:7:7 +Error: sql_warehouse name is required + at resources.sql_warehouses.rname + in databricks.yml:7:7 + { "sql_warehouses": { "rname": { diff --git a/acceptance/bundle/validate/empty_resources/with_permissions/output.txt b/acceptance/bundle/validate/empty_resources/with_permissions/output.txt index 28957886423..ba521d61c97 100644 --- a/acceptance/bundle/validate/empty_resources/with_permissions/output.txt +++ b/acceptance/bundle/validate/empty_resources/with_permissions/output.txt @@ -176,6 +176,10 @@ app resource 'rname' should have either source_code_path or git_source field } === resources.sql_warehouses.rname === +Error: sql_warehouse name is required + at resources.sql_warehouses.rname + in databricks.yml:7:7 + { "sql_warehouses": { "rname": { diff --git a/acceptance/bundle/validate/grants_required_principal/databricks.yml b/acceptance/bundle/validate/grants_required_principal/databricks.yml new file mode 100644 index 00000000000..3c513a12796 --- /dev/null +++ b/acceptance/bundle/validate/grants_required_principal/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: test-bundle + +resources: + catalogs: + my_catalog: + name: my_catalog + grants: + # Missing principal: required by the backend but modeled as optional + # (json:"principal,omitempty") in the SDK. See DECO-27550. + - privileges: + - ALL_PRIVILEGES + schemas: + my_schema: + catalog_name: my_catalog + name: my_schema + grants: + # Valid grant: principal is set. + - principal: alice@example.com + privileges: + - USE_SCHEMA diff --git a/acceptance/bundle/validate/grants_required_principal/out.test.toml b/acceptance/bundle/validate/grants_required_principal/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/validate/grants_required_principal/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/grants_required_principal/output.txt b/acceptance/bundle/validate/grants_required_principal/output.txt new file mode 100644 index 00000000000..f40aab1bab7 --- /dev/null +++ b/acceptance/bundle/validate/grants_required_principal/output.txt @@ -0,0 +1,15 @@ + +>>> [CLI] bundle validate +Error: grant principal is required + at resources.catalogs.my_catalog.grants[0] + in databricks.yml:11:11 + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/grants_required_principal/script b/acceptance/bundle/validate/grants_required_principal/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/validate/grants_required_principal/script @@ -0,0 +1 @@ +trace $CLI bundle validate diff --git a/acceptance/bundle/validate/grants_required_principal/test.toml b/acceptance/bundle/validate/grants_required_principal/test.toml new file mode 100644 index 00000000000..fc2b3f50667 --- /dev/null +++ b/acceptance/bundle/validate/grants_required_principal/test.toml @@ -0,0 +1,3 @@ +# Catalogs and schemas are only supported by the direct engine. +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/retention_hours_range/databricks.yml b/acceptance/bundle/validate/retention_hours_range/databricks.yml new file mode 100644 index 00000000000..d82953280b1 --- /dev/null +++ b/acceptance/bundle/validate/retention_hours_range/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: test-bundle + +resources: + catalogs: + invalid_catalog: + name: invalid_catalog + # The field is named in hours, but the backend only accepts 0 or 168-720 + # (7-30 days). 7 hours is rejected at deploy. See DECO-27550. + custom_max_retention_hours: 7 + valid_catalog: + name: valid_catalog + # 240 hours = 10 days, within range. + custom_max_retention_hours: 240 + schemas: + invalid_schema: + catalog_name: valid_catalog + name: invalid_schema + # 1000 hours is out of range (> 30 days). + custom_max_retention_hours: 1000 diff --git a/acceptance/bundle/validate/retention_hours_range/out.test.toml b/acceptance/bundle/validate/retention_hours_range/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/validate/retention_hours_range/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/retention_hours_range/output.txt b/acceptance/bundle/validate/retention_hours_range/output.txt new file mode 100644 index 00000000000..e0414c95e4d --- /dev/null +++ b/acceptance/bundle/validate/retention_hours_range/output.txt @@ -0,0 +1,19 @@ + +>>> [CLI] bundle validate +Error: custom_max_retention_hours must be 0 or between 168 and 720 hours (7 to 30 days), got 7 + at resources.catalogs.invalid_catalog.custom_max_retention_hours + in databricks.yml:10:35 + +Error: custom_max_retention_hours must be 0 or between 168 and 720 hours (7 to 30 days), got 1000 + at resources.schemas.invalid_schema.custom_max_retention_hours + in databricks.yml:20:35 + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 2 errors + +Exit code: 1 diff --git a/acceptance/bundle/validate/retention_hours_range/script b/acceptance/bundle/validate/retention_hours_range/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/validate/retention_hours_range/script @@ -0,0 +1 @@ +trace $CLI bundle validate diff --git a/acceptance/bundle/validate/retention_hours_range/test.toml b/acceptance/bundle/validate/retention_hours_range/test.toml new file mode 100644 index 00000000000..fc2b3f50667 --- /dev/null +++ b/acceptance/bundle/validate/retention_hours_range/test.toml @@ -0,0 +1,3 @@ +# Catalogs and schemas are only supported by the direct engine. +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/sql_warehouse_required_name/databricks.yml b/acceptance/bundle/validate/sql_warehouse_required_name/databricks.yml new file mode 100644 index 00000000000..95918ef8c98 --- /dev/null +++ b/acceptance/bundle/validate/sql_warehouse_required_name/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: test-bundle + +resources: + sql_warehouses: + # Missing name: required by the backend on create, but modeled as optional + # (json:"name,omitempty") in the SDK. See DECO-27550. + my_warehouse: + cluster_size: "2X-Small" diff --git a/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml b/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/sql_warehouse_required_name/output.txt b/acceptance/bundle/validate/sql_warehouse_required_name/output.txt new file mode 100644 index 00000000000..726fcec42b5 --- /dev/null +++ b/acceptance/bundle/validate/sql_warehouse_required_name/output.txt @@ -0,0 +1,15 @@ + +>>> [CLI] bundle validate +Error: sql_warehouse name is required + at resources.sql_warehouses.my_warehouse + in databricks.yml:9:7 + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/sql_warehouse_required_name/script b/acceptance/bundle/validate/sql_warehouse_required_name/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/validate/sql_warehouse_required_name/script @@ -0,0 +1 @@ +trace $CLI bundle validate diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index 6f886caab44..b22cd366b67 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -14,6 +14,15 @@ import ( type required struct{} +// The backend expresses custom_max_retention_hours in hours but only accepts 0 +// (disabled) or a value corresponding to 7-30 days. Out-of-range values fail at +// deploy with a low-context "recovery period must be 0 or between 7 and 30 days" +// error, so we reject them early. See DECO-27550. +const ( + minCustomRetentionHours = 168 // 7 days + maxCustomRetentionHours = 720 // 30 days +) + func Required() bundle.Mutator { return &required{} } @@ -119,11 +128,101 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti }) } + // sql_warehouses.name is required by the backend on create, but the SDK models it + // as optional (json:"name,omitempty"), so warnForMissingFields never flags it. + // Without a name deploy fails with a low-context error. See DECO-27550. + _, err := dyn.MapByPattern( + b.Config.Value(), + dyn.NewPattern(dyn.Key("resources"), dyn.Key("sql_warehouses"), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if isMissingOrEmptyString(v.Get("name")) { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "sql_warehouse name is required", + Locations: v.Locations(), + Paths: []dyn.Path{slices.Clone(p)}, + }) + } + return v, nil + }, + ) + if err != nil { + return diag.FromErr(err) + } + + // grants[*].principal is required by the backend but modeled as optional + // (json:"principal,omitempty"). Without it deploy fails with "400 Invalid + // PermissionsChange". Grants exist on every securable (catalogs, schemas, volumes, + // external_locations, registered_models, vector_search_indexes). See DECO-27550. + _, err = dyn.MapByPattern( + b.Config.Value(), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants"), dyn.AnyIndex()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if isMissingOrEmptyString(v.Get("principal")) { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "grant principal is required", + Locations: v.Locations(), + Paths: []dyn.Path{slices.Clone(p)}, + }) + } + return v, nil + }, + ) + if err != nil { + return diag.FromErr(err) + } + + return diags +} + +// isMissingOrEmptyString reports whether v is unset, null, or an empty string. +func isMissingOrEmptyString(v dyn.Value) bool { + switch v.Kind() { + case dyn.KindInvalid, dyn.KindNil: + return true + case dyn.KindString: + return v.MustString() == "" + default: + return false + } +} + +// errorForInvalidRetentionHours rejects out-of-range custom_max_retention_hours on +// catalogs and schemas before deploy. See DECO-27550. +func errorForInvalidRetentionHours(b *bundle.Bundle) diag.Diagnostics { + diags := diag.Diagnostics{} + for _, section := range []string{"catalogs", "schemas"} { + _, err := dyn.MapByPattern( + b.Config.Value(), + dyn.NewPattern(dyn.Key("resources"), dyn.Key(section), dyn.AnyKey(), dyn.Key("custom_max_retention_hours")), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + hours, ok := v.AsInt() + if !ok { + return v, nil + } + if hours == 0 || (hours >= minCustomRetentionHours && hours <= maxCustomRetentionHours) { + return v, nil + } + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("custom_max_retention_hours must be 0 or between %d and %d hours (7 to 30 days), got %d", minCustomRetentionHours, maxCustomRetentionHours, hours), + Locations: v.Locations(), + Paths: []dyn.Path{slices.Clone(p)}, + }) + return v, nil + }, + ) + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + } return diags } func (f *required) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := errorForMissingFields(ctx, b) + diags = diags.Extend(errorForInvalidRetentionHours(b)) if diags.HasError() { return diags } From 8f7c3b63e8d27373f0df49869fd444af6bc8c724 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 3 Jul 2026 11:26:17 +0000 Subject: [PATCH 2/3] Remove internal ticket references and trim validation comments --- .../grants_required_principal/databricks.yml | 4 ++-- .../retention_hours_range/databricks.yml | 3 +-- .../databricks.yml | 4 ++-- bundle/config/validate/required.go | 19 +++++++------------ 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/acceptance/bundle/validate/grants_required_principal/databricks.yml b/acceptance/bundle/validate/grants_required_principal/databricks.yml index 3c513a12796..4e81d9cb165 100644 --- a/acceptance/bundle/validate/grants_required_principal/databricks.yml +++ b/acceptance/bundle/validate/grants_required_principal/databricks.yml @@ -6,8 +6,8 @@ resources: my_catalog: name: my_catalog grants: - # Missing principal: required by the backend but modeled as optional - # (json:"principal,omitempty") in the SDK. See DECO-27550. + # Missing principal: required by the backend, optional in the SDK + # (json:"principal,omitempty"). - privileges: - ALL_PRIVILEGES schemas: diff --git a/acceptance/bundle/validate/retention_hours_range/databricks.yml b/acceptance/bundle/validate/retention_hours_range/databricks.yml index d82953280b1..b14ed9e57e0 100644 --- a/acceptance/bundle/validate/retention_hours_range/databricks.yml +++ b/acceptance/bundle/validate/retention_hours_range/databricks.yml @@ -5,8 +5,7 @@ resources: catalogs: invalid_catalog: name: invalid_catalog - # The field is named in hours, but the backend only accepts 0 or 168-720 - # (7-30 days). 7 hours is rejected at deploy. See DECO-27550. + # Named in hours, but the backend only accepts 0 or 168-720 (7-30 days). custom_max_retention_hours: 7 valid_catalog: name: valid_catalog diff --git a/acceptance/bundle/validate/sql_warehouse_required_name/databricks.yml b/acceptance/bundle/validate/sql_warehouse_required_name/databricks.yml index 95918ef8c98..e0cd3a238de 100644 --- a/acceptance/bundle/validate/sql_warehouse_required_name/databricks.yml +++ b/acceptance/bundle/validate/sql_warehouse_required_name/databricks.yml @@ -3,7 +3,7 @@ bundle: resources: sql_warehouses: - # Missing name: required by the backend on create, but modeled as optional - # (json:"name,omitempty") in the SDK. See DECO-27550. + # Missing name: required by the backend, optional in the SDK + # (json:"name,omitempty"). my_warehouse: cluster_size: "2X-Small" diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index b22cd366b67..c8721046c76 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -14,10 +14,8 @@ import ( type required struct{} -// The backend expresses custom_max_retention_hours in hours but only accepts 0 -// (disabled) or a value corresponding to 7-30 days. Out-of-range values fail at -// deploy with a low-context "recovery period must be 0 or between 7 and 30 days" -// error, so we reject them early. See DECO-27550. +// custom_max_retention_hours accepts 0 (disabled) or 7-30 days; other values +// fail at deploy with a low-context error, so we reject them early. const ( minCustomRetentionHours = 168 // 7 days maxCustomRetentionHours = 720 // 30 days @@ -128,9 +126,8 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti }) } - // sql_warehouses.name is required by the backend on create, but the SDK models it - // as optional (json:"name,omitempty"), so warnForMissingFields never flags it. - // Without a name deploy fails with a low-context error. See DECO-27550. + // sql_warehouses.name is required by the backend but optional in the SDK + // (json:"name,omitempty"), so warnForMissingFields never flags it. _, err := dyn.MapByPattern( b.Config.Value(), dyn.NewPattern(dyn.Key("resources"), dyn.Key("sql_warehouses"), dyn.AnyKey()), @@ -150,10 +147,8 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti return diag.FromErr(err) } - // grants[*].principal is required by the backend but modeled as optional - // (json:"principal,omitempty"). Without it deploy fails with "400 Invalid - // PermissionsChange". Grants exist on every securable (catalogs, schemas, volumes, - // external_locations, registered_models, vector_search_indexes). See DECO-27550. + // grants[*].principal is required by the backend but optional in the SDK + // (json:"principal,omitempty"). Grants exist on every securable, so match any. _, err = dyn.MapByPattern( b.Config.Value(), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants"), dyn.AnyIndex()), @@ -189,7 +184,7 @@ func isMissingOrEmptyString(v dyn.Value) bool { } // errorForInvalidRetentionHours rejects out-of-range custom_max_retention_hours on -// catalogs and schemas before deploy. See DECO-27550. +// catalogs and schemas before deploy. func errorForInvalidRetentionHours(b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} for _, section := range []string{"catalogs", "schemas"} { From cfbbb09278376b404fd544be32f2580d1e1ed053 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 3 Jul 2026 11:41:30 +0000 Subject: [PATCH 3/3] Update acceptance output after comment line shift --- acceptance/bundle/validate/retention_hours_range/output.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/validate/retention_hours_range/output.txt b/acceptance/bundle/validate/retention_hours_range/output.txt index e0414c95e4d..499951a3d13 100644 --- a/acceptance/bundle/validate/retention_hours_range/output.txt +++ b/acceptance/bundle/validate/retention_hours_range/output.txt @@ -2,11 +2,11 @@ >>> [CLI] bundle validate Error: custom_max_retention_hours must be 0 or between 168 and 720 hours (7 to 30 days), got 7 at resources.catalogs.invalid_catalog.custom_max_retention_hours - in databricks.yml:10:35 + in databricks.yml:9:35 Error: custom_max_retention_hours must be 0 or between 168 and 720 hours (7 to 30 days), got 1000 at resources.schemas.invalid_schema.custom_max_retention_hours - in databricks.yml:20:35 + in databricks.yml:19:35 Name: test-bundle Target: default