Report PII-free error templates in deploy telemetry

acceptance/bundle/migrate/auto-migrate-clean/output.txt

@@ -17,10 +17,10 @@ direct_drymigrate_warnings false
 >>> [CLI] bundle deploy
 Warn: Direct engine requested in bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
-Migrating state to direct deployment engine (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)...
-Migrated 1 resource to direct deployment engine.
 Files: 3 uploaded, 0 deleted
 Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
+Migrating state to direct deployment engine (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)...
+Migrated 1 resource to direct deployment engine.
 
 >>> print_migration_telemetry
 direct_migrated_via_config true

acceptance/bundle/migrate/auto-migrate-conversion-failure/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+  name: test-bundle
+
+resources:
+  secret_scopes:
+    my_scope:
+      name: my-scope
+      permissions:
+        # An unknown level: SecretScopeFixups rejects it when it prepares the
+        # config for the direct engine, so the state conversion fails while the
+        # terraform deploy itself succeeds.
+        - level: BOGUS
+          user_name: someone@example.com

acceptance/bundle/migrate/auto-migrate-conversion-failure/output.txt

@@ -0,0 +1,46 @@
+
+=== Not opted in: the conversion failure is reported as a failed dry run
+>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy
+Warning: invalid value "BOGUS" for enum field. Valid values are [READ WRITE MANAGE]
+  at resources.secret_scopes.my_scope.permissions[0].level
+
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Created secret_scopes.my_scope
+Created secret_scopes.my_scope.permissions
+Files: 5 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+Warn: post-deploy dry-run migration to direct: unknown permission level "BOGUS" for secret scope
+Warn: The warnings above are from a dry-run migration to the direct deployment engine (https://docs.databricks.com/aws/en/dev-tools/bundles/direct).
+Your deployment is not affected and works normally, but you may experience these issues when migrating to the direct deployment engine.
+Please forward these warnings to dabs-feedback@databricks.com
+
+>>> print_migration_telemetry
+direct_drymigrate_success false
+direct_drymigrate_warnings false
+direct_migrate_error_template unknown permission level %q for secret scope
+
+=== Opted in: the same failure stops the automatic migration
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy
+Warning: invalid value "BOGUS" for enum field. Valid values are [READ WRITE MANAGE]
+  at resources.secret_scopes.my_scope.permissions[0].level
+
+Warn: Direct engine requested in DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 2 unchanged
+Warn: post-deploy dry-run migration to direct: unknown permission level "BOGUS" for secret scope
+Warn: The warnings above are from a dry-run migration to the direct deployment engine (https://docs.databricks.com/aws/en/dev-tools/bundles/direct).
+Your deployment is not affected and works normally, but you may experience these issues when migrating to the direct deployment engine.
+Please forward these warnings to dabs-feedback@databricks.com
+Warn: Direct engine was requested but the dry-run migration reported issues; automatic migration to the direct deployment engine is stopped. Address the issues above or run "databricks bundle deployment migrate" manually.
+
+>>> print_migration_telemetry
+direct_migrate_error true
+direct_migrate_error_template unknown permission level %q for secret scope
+
+=== State is still terraform, so nothing was migrated
+
+>>> find .databricks/bundle -name resources.json -type f
+
+>>> find .databricks/bundle -name terraform.tfstate* -type f
+.databricks/bundle/default/terraform/terraform.tfstate

acceptance/bundle/migrate/auto-migrate-conversion-failure/script

@@ -0,0 +1,20 @@
+export DATABRICKS_BUNDLE_ENGINE=
+
+# An unknown permission level is only a warning for the terraform deploy, but
+# SecretScopeFixups rejects it while preparing the config for the direct engine.
+# That is a state conversion failure: no API call is involved, so unlike the
+# commit failures it cannot be produced by injecting a fault.
+
+title "Not opted in: the conversion failure is reported as a failed dry run"
+trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy
+trace print_migration_telemetry
+rm -f out.requests.txt
+
+title "Opted in: the same failure stops the automatic migration"
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy
+trace print_migration_telemetry
+rm -f out.requests.txt
+
+title "State is still terraform, so nothing was migrated\n"
+trace find .databricks/bundle -name "resources.json" -type f
+trace find .databricks/bundle -name "terraform.tfstate*" -type f

acceptance/bundle/migrate/auto-migrate-conversion-failure/test.toml

@@ -0,0 +1,3 @@
+# The unknown permission level is deliberate; the enum warning is part of what
+# this test asserts.
+Ignore = [".databricks"]

acceptance/bundle/migrate/auto-migrate-direct-only-envvar/output.txt

@@ -13,10 +13,10 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy
 Warn: Direct engine requested in DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
-Migrating state to direct deployment engine (opted in via DATABRICKS_BUNDLE_ENGINE environment variable)...
-Migrated 1 resource to direct deployment engine.
 Files: 3 uploaded, 0 deleted
 Resources: 0 created, 0 changed, 0 deleted, 2 unchanged
+Migrating state to direct deployment engine (opted in via DATABRICKS_BUNDLE_ENGINE environment variable)...
+Migrated 1 resource to direct deployment engine.
 
 >>> print_migration_telemetry
 direct_migrated_via_env true

acceptance/bundle/migrate/auto-migrate-direct-only/output.txt

@@ -23,10 +23,10 @@ json.plan.resources.instance_pools.pool.action = "skip";
 >>> [CLI] bundle deploy
 Warn: Direct engine requested in bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
-Migrating state to direct deployment engine (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)...
-Migrated 1 resource to direct deployment engine.
 Files: 3 uploaded, 0 deleted
 Resources: 0 created, 0 changed, 0 deleted, 2 unchanged
+Migrating state to direct deployment engine (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)...
+Migrated 1 resource to direct deployment engine.
 
 >>> print_migration_telemetry
 direct_migrated_via_config true

acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt

@@ -3,9 +3,9 @@
 >>> [CLI] bundle deploy
 Warn: Direct engine requested in bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
-Removing empty terraform state; direct engine will be used on the next deploy (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)...
 Files: 4 uploaded, 0 deleted
 Resources: 0 created, 0 changed, 0 deleted, 0 unchanged
+Removing empty terraform state; direct engine will be used on the next deploy (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)...
 
 === Terraform state is renamed to .backup; no resources.json (empty state, nothing to persist)
 

acceptance/bundle/migrate/auto-migrate-envvar/output.txt

@@ -14,10 +14,10 @@ direct_drymigrate_warnings false
 >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy
 Warn: Direct engine requested in DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
-Migrating state to direct deployment engine (opted in via DATABRICKS_BUNDLE_ENGINE environment variable)...
-Migrated 1 resource to direct deployment engine.
 Files: 2 uploaded, 0 deleted
 Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
+Migrating state to direct deployment engine (opted in via DATABRICKS_BUNDLE_ENGINE environment variable)...
+Migrated 1 resource to direct deployment engine.
 
 >>> print_migration_telemetry
 direct_migrated_via_env true

acceptance/bundle/migrate/auto-migrate-push-failure/output.txt

@@ -14,13 +14,14 @@ direct_drymigrate_warnings false
 >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy
 Warn: Direct engine requested in DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
-Migrating state to direct deployment engine (opted in via DATABRICKS_BUNDLE_ENGINE environment variable)...
-Warn: automatic migration to direct engine failed: pushing direct state to workspace: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json
 Files: 2 uploaded, 0 deleted
 Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
+Migrating state to direct deployment engine (opted in via DATABRICKS_BUNDLE_ENGINE environment variable)...
+Warn: automatic migration to direct engine failed: pushing direct state to workspace: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json
 
 >>> print_migration_telemetry
 direct_migrate_commit_error true
+direct_migrate_commit_error_template pushing direct state to workspace: access denied [403 INJECTED]
 
 === Local state was NOT rewritten (still terraform)
 
@@ -33,10 +34,10 @@ direct_migrate_commit_error true
 >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy
 Warn: Direct engine requested in DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
-Migrating state to direct deployment engine (opted in via DATABRICKS_BUNDLE_ENGINE environment variable)...
-Migrated 1 resource to direct deployment engine.
 Files: 2 uploaded, 0 deleted
 Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
+Migrating state to direct deployment engine (opted in via DATABRICKS_BUNDLE_ENGINE environment variable)...
+Migrated 1 resource to direct deployment engine.
 
 >>> print_migration_telemetry
 direct_migrated_via_env true

acceptance/bundle/migrate/auto-migrate-tfbackup-failure/output.txt

@@ -14,13 +14,14 @@ direct_drymigrate_warnings false
 >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy
 Warn: Direct engine requested in DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
-Migrating state to direct deployment engine (opted in via DATABRICKS_BUNDLE_ENGINE environment variable)...
-Warn: automatic migration to direct engine failed: pushing direct state to workspace: deleting remote terraform state: Fault injected by test.
 Files: 2 uploaded, 0 deleted
 Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
+Migrating state to direct deployment engine (opted in via DATABRICKS_BUNDLE_ENGINE environment variable)...
+Warn: automatic migration to direct engine failed: pushing direct state to workspace: deleting remote terraform state: Fault injected by test.
 
 >>> print_migration_telemetry
 direct_migrate_commit_error true
+direct_migrate_commit_error_template pushing direct state to workspace: deleting remote terraform state: %w [403 INJECTED]
 
 === Local state was NOT rewritten (still terraform)
 

acceptance/bundle/migrate/script.prepare

@@ -1,7 +1,12 @@
-# Filter print_telemetry_bool_values output to auto-migration keys
-# (direct_drymigrate_*, direct_migrate_*, direct_migrated_via_*). Callers
-# clear out.requests.txt themselves; some tests want to keep it for a
-# subsequent print_requests.py assertion.
+# Filter telemetry output to auto-migration keys (direct_drymigrate_*,
+# direct_migrate_*, direct_migrated_via_*), booleans first and then the
+# PII-free error template that says which failure a direct_migrate_error or
+# direct_drymigrate_success=false was.
+#
+# Call it right after the deploy whose telemetry you want, so a block covers
+# exactly one deploy. It does not clear out.requests.txt: callers do that
+# themselves, since some want it kept for a subsequent print_requests.py.
 print_migration_telemetry() {
     print_telemetry_bool_values | grep '^direct_' || true
+    print_telemetry_error_templates | grep '^direct_' || true
 }

acceptance/script.prepare

@@ -123,6 +123,12 @@ print_telemetry_bool_values() {
     jq -r 'select(.path? == "/telemetry-ext") | (.body.protoLogs // [])[] | fromjson | ( (.entry // .) | (.databricks_cli_log.bundle_deploy_event.experimental.bool_values // []) ) | map("\(.key) \(.value)") | .[]' out.requests.txt | grep -v '^engine_terraform_' | sort
 }
 
+# Print the PII-free error templates the deploy event carries, one "<field>
+# <template>" line each, skipping the ones this deploy did not set.
+print_telemetry_error_templates() {
+    jq -r 'select(.path? == "/telemetry-ext") | (.body.protoLogs // [])[] | fromjson | ( (.entry // .) | (.databricks_cli_log.bundle_deploy_event.experimental // {}) ) | to_entries | map(select((.key | endswith("_template")) and (.value // "") != "")) | map("\(.key) \(.value)") | .[]' out.requests.txt | sort
+}
+
 sethome() {
     local home="$1"
     mkdir -p "$home"

bundle/bundle.go

@@ -85,6 +85,12 @@ type Metrics struct {
 	ExecutionTimes              []protos.IntMapEntry
 	LocalCacheMeasurementsMs    []protos.IntMapEntry // Local cache measurements stored as milliseconds
 
+	// PII-free descriptions of the errors this deploy hit, reported without
+	// scrubbing. Each is a libs/safeerr message template; see the matching fields
+	// on protos.BundleDeployExperimental for what each one covers.
+	DirectMigrateErrorTemplate       string
+	DirectMigrateCommitErrorTemplate string
+
 	// StateEngine is the engine that ran the deploy, set in deployCore. Empty when
 	// telemetry is emitted without a deploy having run.
 	StateEngine engine.EngineType

bundle/config/mutator/resourcemutator/secret_scope_fixups.go

@@ -2,7 +2,6 @@ package resourcemutator
 
 import (
 	"context"
-	"fmt"
 	"slices"
 	"strings"
 
@@ -12,6 +11,7 @@ import (
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/iamutil"
+	"github.com/databricks/cli/libs/safeerr"
 	"github.com/databricks/databricks-sdk-go/service/iam"
 )
 
@@ -63,7 +63,7 @@ func collapsePermissions(scope *resources.SecretScope) error {
 	for _, perm := range scope.Permissions {
 		// Validate permission level
 		if _, ok := permissionRank[perm.Level]; !ok {
-			return fmt.Errorf("unknown permission level %q for secret scope", perm.Level)
+			return safeerr.Errorf("unknown permission level %q for secret scope", perm.Level)
 		}
 
 		// Add a prefix to retain the original principal type. In practice collisions here should
@@ -76,7 +76,7 @@ func collapsePermissions(scope *resources.SecretScope) error {
 		} else if perm.ServicePrincipalName != "" {
 			principal = "sp:" + perm.ServicePrincipalName
 		} else {
-			return fmt.Errorf("missing principal in permissions for secret scope %q", scope.Name)
+			return safeerr.Errorf("missing principal in permissions for secret scope %q", scope.Name)
 		}
 
 		existing, exists := principalPermissions[principal]
@@ -120,14 +120,18 @@ func collapsePermissions(scope *resources.SecretScope) error {
 	return nil
 }
 
-func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+// ApplySecretScopeFixups is the mutator's body, exported so a caller that wants
+// the failure as an error rather than a diagnostic can have it. The migration to
+// the direct engine is one: it reports the error's template to telemetry, which a
+// diagnostic's summary cannot carry. The returned key names the offending scope.
+func ApplySecretScopeFixups(b *bundle.Bundle, eng engine.EngineType) (string, error) {
 	// Secret scopes by default have the current user as a MANAGE ACL. We need to add it to the client ACL list
 	// to prevent a phantom persistent diff.
 	// We do not need to do this in terraform because terraform naively always applies the config during ACL
 	// creation without checking if the ACL already exists.
 	// https://github.com/databricks/terraform-provider-databricks/blob/5cb5d3fa46bc4843be1a4c4bce89296eaa2e14fc/secrets/resource_secret_acl.go#L43
-	if !m.engine.IsDirect() {
-		return nil
+	if !eng.IsDirect() {
+		return "", nil
 	}
 
 	// Secret scopes assigns the create MANAGE ACL on it by default. So we always add it to
@@ -140,19 +144,27 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di
 		currentUser := b.Config.Workspace.CurrentUser.User
 
 		addManageForCurrentUser(scope, currentUser)
-		err := collapsePermissions(scope)
-		if err != nil {
-			return diag.Diagnostics{
-				{
-					Severity:  diag.Error,
-					Summary:   "Failed to collapse permissions for secret scope",
-					Detail:    err.Error(),
-					Paths:     []dyn.Path{dyn.MustPathFromString("resources.secret_scopes." + key)},
-					Locations: []dyn.Location{b.Config.GetLocation("resources.secret_scopes." + key)},
-				},
-			}
+		if err := collapsePermissions(scope); err != nil {
+			return key, err
 		}
 	}
 
-	return nil
+	return "", nil
+}
+
+func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	key, err := ApplySecretScopeFixups(b, m.engine)
+	if err == nil {
+		return nil
+	}
+
+	return diag.Diagnostics{
+		{
+			Severity:  diag.Error,
+			Summary:   "Failed to collapse permissions for secret scope",
+			Detail:    err.Error(),
+			Paths:     []dyn.Path{dyn.MustPathFromString("resources.secret_scopes." + key)},
+			Locations: []dyn.Location{b.Config.GetLocation("resources.secret_scopes." + key)},
+		},
+	}
 }

bundle/config/resource_key.go

@@ -0,0 +1,40 @@
+package config
+
+import "strings"
+
+// ResourceKey wraps a resource key (e.g. "resources.jobs.my_job" or
+// "resources.jobs.my_job.permissions") for use as an error argument. It formats
+// as the full key, so error messages are unchanged, but it reports only its
+// resource type to telemetry — the resource name is user-authored and therefore
+// PII, while the type is a value the CLI itself defines.
+//
+// Pass it wherever a resource key is interpolated into a safeerr error:
+//
+//	safeerr.Errorf("%s: SaveState: %w", config.ResourceKey(node), err)
+//	  message:  resources.jobs.my_job: SaveState: ...
+//	  template: jobs.*: SaveState: %w
+type ResourceKey string
+
+func (k ResourceKey) String() string {
+	return string(k)
+}
+
+// SafeString implements safeerr.SafeStringer, standing in for the key with its
+// name replaced by "*". A key this package cannot parse reports nothing beyond
+// the redaction marker, since an unrecognized shape may be anything at all.
+func (k ResourceKey) SafeString() string {
+	resourceType := GetResourceTypeFromKey(string(k))
+	if resourceType == "" {
+		return "*"
+	}
+
+	// GetResourceTypeFromKey collapses a sub-resource into "<group>.<kind>"
+	// (e.g. "jobs.permissions"), but in the key itself the kind trails the
+	// name. Rebuild the key's own shape so the stand-in reads like the value.
+	// The "resources." prefix is dropped: every key carries it, so it is noise.
+	group, kind, hasKind := strings.Cut(resourceType, ".")
+	if hasKind {
+		return group + ".*." + kind
+	}
+	return group + ".*"
+}