diff --git a/.nextchanges/bundles/apps-uc-securable-kind-drift.md b/.nextchanges/bundles/apps-uc-securable-kind-drift.md new file mode 100644 index 00000000000..ff99af6def9 --- /dev/null +++ b/.nextchanges/bundles/apps-uc-securable-kind-drift.md @@ -0,0 +1 @@ +`bundle plan` no longer reports a permanent, unconvergeable update on `securable_kind` for Unity Catalog `TABLE` securables declared under an app's `resources`. The field is computed by the backend (output-only), so it is now ignored during drift detection. Direct engine only ([#6342](https://github.com/databricks/cli/issues/6342)). diff --git a/acceptance/bundle/resources/apps/uc-securable-drift/app/app.py b/acceptance/bundle/resources/apps/uc-securable-drift/app/app.py new file mode 100644 index 00000000000..ad1e64f9fb0 --- /dev/null +++ b/acceptance/bundle/resources/apps/uc-securable-drift/app/app.py @@ -0,0 +1,5 @@ +import http.server, os + +http.server.HTTPServer( + ("", int(os.environ["DATABRICKS_APP_PORT"])), http.server.SimpleHTTPRequestHandler +).serve_forever() diff --git a/acceptance/bundle/resources/apps/uc-securable-drift/databricks.yml b/acceptance/bundle/resources/apps/uc-securable-drift/databricks.yml new file mode 100644 index 00000000000..48d0506c8a2 --- /dev/null +++ b/acceptance/bundle/resources/apps/uc-securable-drift/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: apps-uc-securable-drift + +resources: + apps: + myapp: + name: myapp + source_code_path: ./app + resources: + - name: my-table + uc_securable: + securable_full_name: main.default.my_table + securable_type: TABLE + permission: SELECT + - name: my-volume + uc_securable: + securable_full_name: main.default.my_volume + securable_type: VOLUME + permission: READ_VOLUME diff --git a/acceptance/bundle/resources/apps/uc-securable-drift/out.test.toml b/acceptance/bundle/resources/apps/uc-securable-drift/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/apps/uc-securable-drift/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/uc-securable-drift/output.txt b/acceptance/bundle/resources/apps/uc-securable-drift/output.txt new file mode 100644 index 00000000000..aa4af5ba44a --- /dev/null +++ b/acceptance/bundle/resources/apps/uc-securable-drift/output.txt @@ -0,0 +1,29 @@ + +=== Deploy an app with UC TABLE and VOLUME securables +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/apps-uc-securable-drift/default/files... +Created apps.myapp +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Re-plan is clean: the TABLE securable's server-computed securable_kind is output-only and must not drift +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== securable_kind is classified output-only (skip); the VOLUME securable has none +>>> [CLI] bundle plan -o json +{ + "resources[0].uc_securable.securable_kind": { + "action": "skip", + "reason": "spec:output_only", + "remote": "TABLE_DELTA" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/apps-uc-securable-drift/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/apps/uc-securable-drift/script b/acceptance/bundle/resources/apps/uc-securable-drift/script new file mode 100644 index 00000000000..04438951475 --- /dev/null +++ b/acceptance/bundle/resources/apps/uc-securable-drift/script @@ -0,0 +1,13 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy an app with UC TABLE and VOLUME securables" +trace $CLI bundle deploy + +title "Re-plan is clean: the TABLE securable's server-computed securable_kind is output-only and must not drift" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "securable_kind is classified output-only (skip); the VOLUME securable has none" +trace $CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes // {} | with_entries(select(.key | test("uc_securable")))' diff --git a/acceptance/bundle/resources/apps/uc-securable-drift/test.toml b/acceptance/bundle/resources/apps/uc-securable-drift/test.toml new file mode 100644 index 00000000000..5a8fd7adb65 --- /dev/null +++ b/acceptance/bundle/resources/apps/uc-securable-drift/test.toml @@ -0,0 +1,8 @@ +# securable_kind drift is a direct-engine plan-classification concern, and the fake +# server models the backend computing it on TABLE securables. A real workspace would +# need the referenced UC table to exist, so this runs locally only. +RecordRequests = false + +Ignore = [".databricks", "databricks.yml"] + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/output.txt index 066ee507f61..a84408d9139 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/output.txt @@ -15,14 +15,14 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged { "config.served_entities[0].burst_scaling_enabled": { "action": "skip", - "reason": "input_only", + "reason": "spec:input_only", "old": true, "new": true, "remote": false }, "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { "action": "skip", - "reason": "input_only", + "reason": "spec:input_only", "old": "sk-test-plaintext-key", "new": "sk-test-plaintext-key", "remote": "" diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index c121f9b3a0f..10b45fc4fee 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -60,6 +60,8 @@ resources: reason: spec:output_only - field: pending_deployment reason: spec:output_only + - field: resources[*].uc_securable.securable_kind + reason: spec:output_only - field: service_principal_client_id reason: spec:output_only - field: service_principal_id @@ -115,14 +117,16 @@ resources: ignore_remote_changes: - field: custom_tags reason: spec:input_only + - field: custom_tags[*].key + reason: spec:input_only + - field: custom_tags[*].value + reason: spec:input_only - field: enable_pg_native_login reason: spec:input_only - field: enable_readable_secondaries reason: spec:input_only - field: node_count reason: spec:input_only - - field: parent_instance_ref.lsn - reason: spec:input_only - field: retention_window_in_days reason: spec:input_only - field: stopped @@ -132,6 +136,16 @@ resources: - field: child_instance_refs reason: spec:output_only + - field: child_instance_refs[*].branch_time + reason: spec:output_only + - field: child_instance_refs[*].effective_lsn + reason: spec:output_only + - field: child_instance_refs[*].lsn + reason: spec:output_only + - field: child_instance_refs[*].name + reason: spec:output_only + - field: child_instance_refs[*].uid + reason: spec:output_only - field: creation_time reason: spec:output_only - field: creator @@ -152,10 +166,6 @@ resources: reason: spec:output_only - field: effective_usage_policy_id reason: spec:output_only - - field: parent_instance_ref.effective_lsn - reason: spec:output_only - - field: parent_instance_ref.uid - reason: spec:output_only - field: pg_version reason: spec:output_only - field: read_only_dns @@ -200,6 +210,31 @@ resources: model_serving_endpoints: ignore_remote_changes: + - field: config.served_entities[*].burst_scaling_enabled + reason: spec:input_only + - field: config.served_entities[*].external_model.ai21labs_config.ai21labs_api_key_plaintext + reason: spec:input_only + - field: config.served_entities[*].external_model.amazon_bedrock_config.aws_access_key_id_plaintext + reason: spec:input_only + - field: config.served_entities[*].external_model.amazon_bedrock_config.aws_secret_access_key_plaintext + reason: spec:input_only + - field: config.served_entities[*].external_model.anthropic_config.anthropic_api_key_plaintext + reason: spec:input_only + - field: config.served_entities[*].external_model.cohere_config.cohere_api_key_plaintext + reason: spec:input_only + - field: config.served_entities[*].external_model.databricks_model_serving_config.databricks_api_token_plaintext + reason: spec:input_only + - field: config.served_entities[*].external_model.google_cloud_vertex_ai_config.private_key_plaintext + reason: spec:input_only + - field: config.served_entities[*].external_model.openai_config.microsoft_entra_client_secret_plaintext + reason: spec:input_only + - field: config.served_entities[*].external_model.openai_config.openai_api_key_plaintext + reason: spec:input_only + - field: config.served_entities[*].external_model.palm_config.palm_api_key_plaintext + reason: spec:input_only + - field: config.served_models[*].burst_scaling_enabled + reason: spec:input_only + - field: telemetry_config.inference_table_config.name reason: spec:output_only @@ -294,6 +329,10 @@ resources: reason: spec:input_only - field: custom_tags reason: spec:input_only + - field: custom_tags[*].key + reason: spec:input_only + - field: custom_tags[*].value + reason: spec:input_only - field: default_branch reason: spec:input_only - field: default_endpoint_settings @@ -334,6 +373,14 @@ resources: reason: spec:input_only - field: extra_columns reason: spec:input_only + - field: extra_columns[*].column_name + reason: spec:input_only + - field: extra_columns[*].column_type + reason: spec:input_only + - field: extra_columns[*].compute + reason: spec:input_only + - field: extra_columns[*].maintenance + reason: spec:input_only - field: new_pipeline_spec reason: spec:input_only - field: postgres_database @@ -348,6 +395,12 @@ resources: reason: spec:input_only - field: type_overrides reason: spec:input_only + - field: type_overrides[*].column_name + reason: spec:input_only + - field: type_overrides[*].pg_type + reason: spec:input_only + - field: type_overrides[*].size + reason: spec:input_only # quality_monitors: no api field behaviors diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 6a29e861798..9c7b11e19c3 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -317,39 +317,17 @@ resources: # reports empty. This is not a backend default, so suppress remote changes. - field: budget_policy_id reason: no_update_api - # Accepted on write but not returned by GET. - - field: config.served_entities[*].burst_scaling_enabled - reason: input_only # table_names creates a profile and is not returned; inference_table_config round-trips. # The spec annotated this input_only in the v0.171 spec but no longer does, so the rule # has to be manual again. - field: telemetry_config.table_names reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. - - field: config.served_entities[*].external_model.ai21labs_config.ai21labs_api_key_plaintext - reason: input_only - - field: config.served_entities[*].external_model.amazon_bedrock_config.aws_access_key_id_plaintext - reason: input_only - - field: config.served_entities[*].external_model.amazon_bedrock_config.aws_secret_access_key_plaintext - reason: input_only - - field: config.served_entities[*].external_model.anthropic_config.anthropic_api_key_plaintext - reason: input_only - - field: config.served_entities[*].external_model.cohere_config.cohere_api_key_plaintext - reason: input_only + # custom_provider_config is not in the OpenAPI spec so the generator cannot reach it. - field: config.served_entities[*].external_model.custom_provider_config.api_key_auth.value_plaintext reason: input_only - field: config.served_entities[*].external_model.custom_provider_config.bearer_token_auth.token_plaintext reason: input_only - - field: config.served_entities[*].external_model.databricks_model_serving_config.databricks_api_token_plaintext - reason: input_only - - field: config.served_entities[*].external_model.google_cloud_vertex_ai_config.private_key_plaintext - reason: input_only - - field: config.served_entities[*].external_model.openai_config.openai_api_key_plaintext - reason: input_only - - field: config.served_entities[*].external_model.openai_config.microsoft_entra_client_secret_plaintext - reason: input_only - - field: config.served_entities[*].external_model.palm_config.palm_api_key_plaintext - reason: input_only ignore_local_changes: - field: budget_policy_id reason: no_update_api diff --git a/bundle/direct/tools/generate_resources.py b/bundle/direct/tools/generate_resources.py index 081de8b3273..bd3838c7ed4 100644 --- a/bundle/direct/tools/generate_resources.py +++ b/bundle/direct/tools/generate_resources.py @@ -33,30 +33,44 @@ def parse_apitypes(generated_path, override_path): def parse_out_fields(path): - """Parse out.fields.txt to extract STATE field names per resource.""" + """Parse out.fields.txt to extract STATE field names per resource and array element types.""" state_fields = {} + array_element_types = {} # (resource, base_path) -> element_type for line in path.read_text().splitlines(): parts = line.split("\t") if len(parts) < 3 or not parts[0].startswith("resources."): continue - field_path, flags = parts[0], parts[2:] + field_path, field_type, flags = parts[0], parts[1], parts[2:] if "STATE" not in flags and "ALL" not in flags: continue # Field line: resources..*. match = re.match(r"resources\.([a-z_]+)\.\*\.(.+)", field_path) - if match and "[*]" not in match.group(2): - state_fields.setdefault(match.group(1), set()).add(match.group(2)) + if not match: + continue + resource, rest = match.group(1), match.group(2) + + # Array element type declarations (path ends with [*]): record the element type + # so the generator can traverse into array element schemas that lack a ref in the + # OpenAPI schema (e.g. App.resources[] -> AppResource). + if rest.endswith("[*]"): + base_path = rest[:-3] + elem_type = field_type.lstrip("*") + array_element_types[(resource, base_path)] = elem_type - return state_fields + state_fields.setdefault(resource, set()).add(rest) + return state_fields, array_element_types -def get_field_behaviors(schemas, type_name): + +def get_field_behaviors(schemas, type_name, resource_name=None, array_element_types=None): """Extract field behaviors from a schema, propagating INPUT_ONLY/OUTPUT_ONLY from containers.""" if type_name not in schemas: return {} + if array_element_types is None: + array_element_types = {} def extract(schema, prefix, visited, depth, inherited): if depth > 4: @@ -76,6 +90,14 @@ def extract(schema, prefix, visited, depth, inherited): visited.add(ref) propagate = [b for b in behaviors if b in ("INPUT_ONLY", "OUTPUT_ONLY")] results.update(extract(schemas[ref], path, visited, depth + 1, propagate)) + elif resource_name is not None: + # For array fields with no ref in the schema, use the element type from + # out.fields.txt (e.g. App.resources[] -> AppResource). + elem_type = array_element_types.get((resource_name, path)) + if elem_type and elem_type in schemas and elem_type not in visited: + visited.add(elem_type) + propagate = [b for b in behaviors if b in ("INPUT_ONLY", "OUTPUT_ONLY")] + results.update(extract(schemas[elem_type], f"{path}[*]", visited, depth + 1, propagate)) return results # Find INPUT_ONLY/OUTPUT_ONLY from container types that reference this type @@ -177,14 +199,14 @@ def main(): args = parser.parse_args() resource_types = parse_apitypes(args.apitypes, args.apitypes_override) - state_fields = parse_out_fields(args.out_fields) + state_fields, array_element_types = parse_out_fields(args.out_fields) schemas = json.loads(args.apischema.read_text())["schemas"] resource_behaviors = {} for resource, type_name in sorted(resource_types.items()): fields = state_fields.get(resource, set()) print(f"\n{resource}: type={type_name}", file=sys.stderr) - all_behaviors = get_field_behaviors(schemas, type_name) + all_behaviors = get_field_behaviors(schemas, type_name, resource, array_element_types) if all_behaviors: print(f" field behaviors from {type_name}:", file=sys.stderr) for field in sorted(all_behaviors): diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index 6b7b5d4190f..7ef32ae1920 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -16,6 +16,18 @@ const ( appStatusUnavailableMessage = "App status is unavailable." ) +// setUcSecurableKinds mimics the platform computing an output-only +// securable_kind (e.g. TABLE_DELTA) for UC TABLE securables. The client never +// sends it, but the backend returns it on every read; VOLUME and other +// securable types get none. See https://github.com/databricks/cli/issues/6342 +func setUcSecurableKinds(app *apps.App) { + for _, res := range app.Resources { + if res.UcSecurable != nil && res.UcSecurable.SecurableType == apps.AppResourceUcSecurableUcSecurableTypeTable { + res.UcSecurable.SecurableKind = "TABLE_DELTA" + } + } +} + func (s *FakeWorkspace) AppsCreateUpdate(req Request, name string) Response { var updateReq apps.AsyncUpdateAppRequest if err := json.Unmarshal(req.Body, &updateReq); err != nil { @@ -66,6 +78,7 @@ func (s *FakeWorkspace) AppsCreateUpdate(req Request, name string) Response { return Response{Body: fmt.Sprintf("internal error: %s", err), StatusCode: http.StatusInternalServerError} } } + setUcSecurableKinds(&existing) s.Apps[name] = existing return Response{ @@ -356,6 +369,8 @@ func (s *FakeWorkspace) AppsUpsert(req Request, name string) Response { }) } + setUcSecurableKinds(&app) + s.Apps[name] = app return Response{ Body: app,