Fix resource config generator to correctly generate nested fields

.nextchanges/bundles/apps-uc-securable-kind-drift.md

@@ -0,0 +1 @@
+`bundle plan` no longer reports a permanent, unconvergeable update on `securable_kind` for Unity Catalog `TABLE` securables declared under an app's `resources`. The field is computed by the backend (output-only), so it is now ignored during drift detection. Direct engine only ([#6342](https://github.com/databricks/cli/issues/6342)).

acceptance/bundle/resources/apps/uc-securable-drift/app/app.py

@@ -0,0 +1,5 @@
+import http.server, os
+
+http.server.HTTPServer(
+    ("", int(os.environ["DATABRICKS_APP_PORT"])), http.server.SimpleHTTPRequestHandler
+).serve_forever()

acceptance/bundle/resources/apps/uc-securable-drift/databricks.yml

@@ -0,0 +1,19 @@
+bundle:
+  name: apps-uc-securable-drift
+
+resources:
+  apps:
+    myapp:
+      name: myapp
+      source_code_path: ./app
+      resources:
+        - name: my-table
+          uc_securable:
+            securable_full_name: main.default.my_table
+            securable_type: TABLE
+            permission: SELECT
+        - name: my-volume
+          uc_securable:
+            securable_full_name: main.default.my_volume
+            securable_type: VOLUME
+            permission: READ_VOLUME

acceptance/bundle/resources/apps/uc-securable-drift/output.txt

@@ -0,0 +1,29 @@
+
+=== Deploy an app with UC TABLE and VOLUME securables
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/apps-uc-securable-drift/default/files...
+Created apps.myapp
+Files: 5 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Re-plan is clean: the TABLE securable's server-computed securable_kind is output-only and must not drift
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== securable_kind is classified output-only (skip); the VOLUME securable has none
+>>> [CLI] bundle plan -o json
+{
+  "resources[0].uc_securable.securable_kind": {
+    "action": "skip",
+    "reason": "spec:output_only",
+    "remote": "TABLE_DELTA"
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.apps.myapp
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/apps-uc-securable-drift/default
+
+Destroy: 1 deleted

acceptance/bundle/resources/apps/uc-securable-drift/script

@@ -0,0 +1,13 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+title "Deploy an app with UC TABLE and VOLUME securables"
+trace $CLI bundle deploy
+
+title "Re-plan is clean: the TABLE securable's server-computed securable_kind is output-only and must not drift"
+trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged"
+
+title "securable_kind is classified output-only (skip); the VOLUME securable has none"
+trace $CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes // {} | with_entries(select(.key | test("uc_securable")))'

acceptance/bundle/resources/apps/uc-securable-drift/test.toml

@@ -0,0 +1,8 @@
+# securable_kind drift is a direct-engine plan-classification concern, and the fake
+# server models the backend computing it on TABLE securables. A real workspace would
+# need the referenced UC table to exist, so this runs locally only.
+RecordRequests = false
+
+Ignore = [".databricks", "databricks.yml"]
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/resources/model_serving_endpoints/drift/write_only/output.txt

@@ -15,14 +15,14 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
 {
   "config.served_entities[0].burst_scaling_enabled": {
     "action": "skip",
-    "reason": "input_only",
+    "reason": "spec:input_only",
     "old": true,
     "new": true,
     "remote": false
   },
   "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": {
     "action": "skip",
-    "reason": "input_only",
+    "reason": "spec:input_only",
     "old": "sk-test-plaintext-key",
     "new": "sk-test-plaintext-key",
     "remote": ""

bundle/direct/dresources/resources.yml

@@ -317,39 +317,17 @@ resources:
       # reports empty. This is not a backend default, so suppress remote changes.
       - field: budget_policy_id
         reason: no_update_api
-      # Accepted on write but not returned by GET.
-      - field: config.served_entities[*].burst_scaling_enabled
-        reason: input_only
       # table_names creates a profile and is not returned; inference_table_config round-trips.
       # The spec annotated this input_only in the v0.171 spec but no longer does, so the rule
       # has to be manual again.
       - field: telemetry_config.table_names
         reason: input_only
       # Write-only secrets: the backend stores them and returns the reference field, not the plaintext.
-      - field: config.served_entities[*].external_model.ai21labs_config.ai21labs_api_key_plaintext
-        reason: input_only
-      - field: config.served_entities[*].external_model.amazon_bedrock_config.aws_access_key_id_plaintext
-        reason: input_only
-      - field: config.served_entities[*].external_model.amazon_bedrock_config.aws_secret_access_key_plaintext
-        reason: input_only
-      - field: config.served_entities[*].external_model.anthropic_config.anthropic_api_key_plaintext
-        reason: input_only
-      - field: config.served_entities[*].external_model.cohere_config.cohere_api_key_plaintext
-        reason: input_only
+      # custom_provider_config is not in the OpenAPI spec so the generator cannot reach it.
       - field: config.served_entities[*].external_model.custom_provider_config.api_key_auth.value_plaintext
         reason: input_only
       - field: config.served_entities[*].external_model.custom_provider_config.bearer_token_auth.token_plaintext
         reason: input_only
-      - field: config.served_entities[*].external_model.databricks_model_serving_config.databricks_api_token_plaintext
-        reason: input_only
-      - field: config.served_entities[*].external_model.google_cloud_vertex_ai_config.private_key_plaintext
-        reason: input_only
-      - field: config.served_entities[*].external_model.openai_config.openai_api_key_plaintext
-        reason: input_only
-      - field: config.served_entities[*].external_model.openai_config.microsoft_entra_client_secret_plaintext
-        reason: input_only
-      - field: config.served_entities[*].external_model.palm_config.palm_api_key_plaintext
-        reason: input_only
     ignore_local_changes:
       - field: budget_policy_id
         reason: no_update_api

bundle/direct/tools/generate_resources.py

@@ -33,30 +33,44 @@ def parse_apitypes(generated_path, override_path):
 
 
 def parse_out_fields(path):
-    """Parse out.fields.txt to extract STATE field names per resource."""
+    """Parse out.fields.txt to extract STATE field names per resource and array element types."""
     state_fields = {}
+    array_element_types = {}  # (resource, base_path) -> element_type
 
     for line in path.read_text().splitlines():
         parts = line.split("\t")
         if len(parts) < 3 or not parts[0].startswith("resources."):
             continue
 
-        field_path, flags = parts[0], parts[2:]
+        field_path, field_type, flags = parts[0], parts[1], parts[2:]
         if "STATE" not in flags and "ALL" not in flags:
             continue
 
         # Field line: resources.<name>.*.<field>
         match = re.match(r"resources\.([a-z_]+)\.\*\.(.+)", field_path)
-        if match and "[*]" not in match.group(2):
-            state_fields.setdefault(match.group(1), set()).add(match.group(2))
+        if not match:
+            continue
+        resource, rest = match.group(1), match.group(2)
+
+        # Array element type declarations (path ends with [*]): record the element type
+        # so the generator can traverse into array element schemas that lack a ref in the
+        # OpenAPI schema (e.g. App.resources[] -> AppResource).
+        if rest.endswith("[*]"):
+            base_path = rest[:-3]
+            elem_type = field_type.lstrip("*")
+            array_element_types[(resource, base_path)] = elem_type
 
-    return state_fields
+        state_fields.setdefault(resource, set()).add(rest)
 
+    return state_fields, array_element_types
 
-def get_field_behaviors(schemas, type_name):
+
+def get_field_behaviors(schemas, type_name, resource_name=None, array_element_types=None):
     """Extract field behaviors from a schema, propagating INPUT_ONLY/OUTPUT_ONLY from containers."""
     if type_name not in schemas:
         return {}
+    if array_element_types is None:
+        array_element_types = {}
 
     def extract(schema, prefix, visited, depth, inherited):
         if depth > 4:
@@ -76,6 +90,14 @@ def extract(schema, prefix, visited, depth, inherited):
                     visited.add(ref)
                     propagate = [b for b in behaviors if b in ("INPUT_ONLY", "OUTPUT_ONLY")]
                     results.update(extract(schemas[ref], path, visited, depth + 1, propagate))
+            elif resource_name is not None:
+                # For array fields with no ref in the schema, use the element type from
+                # out.fields.txt (e.g. App.resources[] -> AppResource).
+                elem_type = array_element_types.get((resource_name, path))
+                if elem_type and elem_type in schemas and elem_type not in visited:
+                    visited.add(elem_type)
+                    propagate = [b for b in behaviors if b in ("INPUT_ONLY", "OUTPUT_ONLY")]
+                    results.update(extract(schemas[elem_type], f"{path}[*]", visited, depth + 1, propagate))
         return results
 
     # Find INPUT_ONLY/OUTPUT_ONLY from container types that reference this type
@@ -177,14 +199,14 @@ def main():
     args = parser.parse_args()
 
     resource_types = parse_apitypes(args.apitypes, args.apitypes_override)
-    state_fields = parse_out_fields(args.out_fields)
+    state_fields, array_element_types = parse_out_fields(args.out_fields)
     schemas = json.loads(args.apischema.read_text())["schemas"]
 
     resource_behaviors = {}
     for resource, type_name in sorted(resource_types.items()):
         fields = state_fields.get(resource, set())
         print(f"\n{resource}: type={type_name}", file=sys.stderr)
-        all_behaviors = get_field_behaviors(schemas, type_name)
+        all_behaviors = get_field_behaviors(schemas, type_name, resource, array_element_types)
         if all_behaviors:
             print(f"  field behaviors from {type_name}:", file=sys.stderr)
             for field in sorted(all_behaviors):

libs/testserver/apps.go

@@ -16,6 +16,18 @@ const (
 	appStatusUnavailableMessage = "App status is unavailable."
 )
 
+// setUcSecurableKinds mimics the platform computing an output-only
+// securable_kind (e.g. TABLE_DELTA) for UC TABLE securables. The client never
+// sends it, but the backend returns it on every read; VOLUME and other
+// securable types get none. See https://github.com/databricks/cli/issues/6342
+func setUcSecurableKinds(app *apps.App) {
+	for _, res := range app.Resources {
+		if res.UcSecurable != nil && res.UcSecurable.SecurableType == apps.AppResourceUcSecurableUcSecurableTypeTable {
+			res.UcSecurable.SecurableKind = "TABLE_DELTA"
+		}
+	}
+}
+
 func (s *FakeWorkspace) AppsCreateUpdate(req Request, name string) Response {
 	var updateReq apps.AsyncUpdateAppRequest
 	if err := json.Unmarshal(req.Body, &updateReq); err != nil {
@@ -66,6 +78,7 @@ func (s *FakeWorkspace) AppsCreateUpdate(req Request, name string) Response {
 			return Response{Body: fmt.Sprintf("internal error: %s", err), StatusCode: http.StatusInternalServerError}
 		}
 	}
+	setUcSecurableKinds(&existing)
 	s.Apps[name] = existing
 
 	return Response{
@@ -356,6 +369,8 @@ func (s *FakeWorkspace) AppsUpsert(req Request, name string) Response {
 		})
 	}
 
+	setUcSecurableKinds(&app)
+
 	s.Apps[name] = app
 	return Response{
 		Body: app,