add libraries field for Clusters

.nextchanges/bundles/cluster-libraries.md

@@ -0,0 +1 @@
+Add support for a `libraries` list on the `clusters` resource type in Declarative Automation Bundles. Libraries (whl, jar, pypi, maven, cran, egg, requirements) are installed on the all-purpose cluster via the Libraries API; local wheels/jars are built and uploaded automatically. Cluster libraries are only supported in direct deployment mode.

acceptance/bundle/refschema/out.fields.txt

@@ -460,6 +460,23 @@ resources.clusters.*.jdbc_port	int	REMOTE
 resources.clusters.*.kind	compute.Kind	ALL
 resources.clusters.*.last_restarted_time	int64	REMOTE
 resources.clusters.*.last_state_loss_time	int64	REMOTE
+resources.clusters.*.libraries	[]compute.Library	INPUT
+resources.clusters.*.libraries[*]	compute.Library	INPUT
+resources.clusters.*.libraries[*].cran	*compute.RCranLibrary	INPUT
+resources.clusters.*.libraries[*].cran.package	string	INPUT
+resources.clusters.*.libraries[*].cran.repo	string	INPUT
+resources.clusters.*.libraries[*].egg	string	INPUT
+resources.clusters.*.libraries[*].jar	string	INPUT
+resources.clusters.*.libraries[*].maven	*compute.MavenLibrary	INPUT
+resources.clusters.*.libraries[*].maven.coordinates	string	INPUT
+resources.clusters.*.libraries[*].maven.exclusions	[]string	INPUT
+resources.clusters.*.libraries[*].maven.exclusions[*]	string	INPUT
+resources.clusters.*.libraries[*].maven.repo	string	INPUT
+resources.clusters.*.libraries[*].pypi	*compute.PythonPyPiLibrary	INPUT
+resources.clusters.*.libraries[*].pypi.package	string	INPUT
+resources.clusters.*.libraries[*].pypi.repo	string	INPUT
+resources.clusters.*.libraries[*].requirements	string	INPUT
+resources.clusters.*.libraries[*].whl	string	INPUT
 resources.clusters.*.lifecycle	*dresources.StateLifecycle	REMOTE	STATE
 resources.clusters.*.lifecycle	*resources.LifecycleWithStarted	INPUT
 resources.clusters.*.lifecycle	resources.Lifecycle	INPUT
@@ -610,6 +627,23 @@ resources.clusters.*.workload_type	*compute.WorkloadType	ALL
 resources.clusters.*.workload_type.clients	compute.ClientsTypes	ALL
 resources.clusters.*.workload_type.clients.jobs	bool	ALL
 resources.clusters.*.workload_type.clients.notebooks	bool	ALL
+resources.clusters.*.libraries.cluster_id	string	ALL
+resources.clusters.*.libraries[*]	compute.Library	ALL
+resources.clusters.*.libraries[*].cran	*compute.RCranLibrary	ALL
+resources.clusters.*.libraries[*].cran.package	string	ALL
+resources.clusters.*.libraries[*].cran.repo	string	ALL
+resources.clusters.*.libraries[*].egg	string	ALL
+resources.clusters.*.libraries[*].jar	string	ALL
+resources.clusters.*.libraries[*].maven	*compute.MavenLibrary	ALL
+resources.clusters.*.libraries[*].maven.coordinates	string	ALL
+resources.clusters.*.libraries[*].maven.exclusions	[]string	ALL
+resources.clusters.*.libraries[*].maven.exclusions[*]	string	ALL
+resources.clusters.*.libraries[*].maven.repo	string	ALL
+resources.clusters.*.libraries[*].pypi	*compute.PythonPyPiLibrary	ALL
+resources.clusters.*.libraries[*].pypi.package	string	ALL
+resources.clusters.*.libraries[*].pypi.repo	string	ALL
+resources.clusters.*.libraries[*].requirements	string	ALL
+resources.clusters.*.libraries[*].whl	string	ALL
 resources.clusters.*.permissions.object_id	string	ALL
 resources.clusters.*.permissions[*]	dresources.StatePermission	ALL
 resources.clusters.*.permissions[*].group_name	string	ALL

acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl

@@ -0,0 +1,17 @@
+bundle:
+  name: cluster-libraries-drift-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  clusters:
+    mycluster:
+      cluster_name: mycluster-$UNIQUE_NAME
+      spark_version: $DEFAULT_SPARK_VERSION
+      node_type_id: $NODE_TYPE_ID
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
+      num_workers: 1
+      libraries:
+        - pypi:
+            package: requests

acceptance/bundle/resources/clusters/libraries-drift/output.txt

@@ -0,0 +1,12 @@
+
+=== Plan is a no-op immediately after deploy (no library drift)
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.clusters.mycluster
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 1 deleted

acceptance/bundle/resources/clusters/libraries-drift/script

@@ -0,0 +1,20 @@
+# A pypi library is used because its nested {package, repo} shape is the most
+# likely to drift (the status API echoing a repo we did not set); a workspace
+# wheel is covered by the local clusters/libraries test instead, since the
+# shared cloud test cluster rejects libraries from /Workspace paths.
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+# Cluster provisioning and library-install output is noisy and differs between
+# the fake and cloud, so route it to LOG and assert only the deterministic
+# drift signal below.
+$CLI bundle deploy &> LOG.deploy
+cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" > /dev/null
+
+title "Plan is a no-op immediately after deploy (no library drift)"
+trace $CLI bundle plan

acceptance/bundle/resources/clusters/libraries-drift/test.toml

@@ -0,0 +1,6 @@
+Cloud = true
+RecordRequests = false
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+Ignore = [".databricks", "databricks.yml"]

acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+  name: cluster-libraries-terraform-error
+
+resources:
+  clusters:
+    mycluster:
+      cluster_name: mycluster
+      spark_version: 15.4.x-scala2.12
+      node_type_id: i3.xlarge
+      num_workers: 1
+      libraries:
+        - pypi:
+            package: requests

acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt

@@ -0,0 +1,16 @@
+
+=== bundle plan fails with cluster libraries on terraform engine
+>>> errcode [CLI] bundle plan
+Error: cluster libraries are only supported in direct deployment mode
+  in databricks.yml:12:9
+
+
+Exit code: 1
+
+=== bundle deploy fails with cluster libraries on terraform engine
+>>> errcode [CLI] bundle deploy
+Error: cluster libraries are only supported in direct deployment mode
+  in databricks.yml:12:9
+
+
+Exit code: 1

acceptance/bundle/resources/clusters/libraries-terraform-error/script

@@ -0,0 +1,5 @@
+title "bundle plan fails with cluster libraries on terraform engine"
+trace errcode $CLI bundle plan
+
+title "bundle deploy fails with cluster libraries on terraform engine"
+trace errcode $CLI bundle deploy

acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml

@@ -0,0 +1,4 @@
+Cloud = false
+RecordRequests = false
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"]

acceptance/bundle/resources/clusters/libraries/databricks.yml

@@ -0,0 +1,14 @@
+bundle:
+  name: cluster-libraries
+
+resources:
+  clusters:
+    mycluster:
+      cluster_name: mycluster
+      spark_version: 15.4.x-scala2.12
+      node_type_id: i3.xlarge
+      num_workers: 1
+      libraries:
+        - pypi:
+            package: requests
+        - whl: ./dist/*.whl

acceptance/bundle/resources/clusters/libraries/output.txt

@@ -0,0 +1,58 @@
+
+=== Deploy a cluster with a pypi and a local wheel library
+>>> [CLI] bundle deploy
+Uploading dist/my_package-0.0.1-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files...
+Created clusters.mycluster
+Created clusters.mycluster.libraries
+Files: 6 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Libraries installed via the Libraries API (wheel rewritten to its uploaded path)
+>>> print_requests.py //libraries/install
+{
+  "method": "POST",
+  "path": "/api/2.0/libraries/install",
+  "body": {
+    "cluster_id": "[UUID]",
+    "libraries": [
+      {
+        "pypi": {
+          "package": "requests"
+        }
+      },
+      {
+        "whl": "/Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/artifacts/.internal/my_package-0.0.1-py3-none-any.whl"
+      }
+    ]
+  }
+}
+
+=== Removing the wheel and redeploying uninstalls it
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files...
+Updated clusters.mycluster.libraries
+Files: 3 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 1 unchanged
+
+>>> print_requests.py //libraries/uninstall
+{
+  "method": "POST",
+  "path": "/api/2.0/libraries/uninstall",
+  "body": {
+    "cluster_id": "[UUID]",
+    "libraries": [
+      {
+        "whl": "/Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/artifacts/.internal/my_package-0.0.1-py3-none-any.whl"
+      }
+    ]
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.clusters.mycluster
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default
+
+Destroy: 1 deleted

acceptance/bundle/resources/clusters/libraries/script

@@ -0,0 +1,16 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+title "Deploy a cluster with a pypi and a local wheel library"
+trace $CLI bundle deploy
+
+title "Libraries installed via the Libraries API (wheel rewritten to its uploaded path)"
+trace print_requests.py //libraries/install
+
+title "Removing the wheel and redeploying uninstalls it"
+update_file.py databricks.yml "        - whl: ./dist/*.whl" ""
+trace $CLI bundle deploy
+trace print_requests.py //libraries/uninstall

acceptance/bundle/resources/clusters/libraries/test.toml

@@ -0,0 +1,10 @@
+Cloud = false
+RecordRequests = true
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+Ignore = [".databricks"]
+
+[[Repls]]
+Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}"
+New = "[CLUSTER-ID]"

bundle/config/mutator/validate_cluster_libraries.go

@@ -0,0 +1,43 @@
+package mutator
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/engine"
+	"github.com/databricks/cli/libs/diag"
+)
+
+type validateClusterLibraries struct {
+	engine engine.EngineType
+}
+
+// ValidateClusterLibraries returns a mutator that errors when cluster libraries are used with
+// the terraform deployment engine. Cluster libraries are only supported in direct deployment mode.
+func ValidateClusterLibraries(e engine.EngineType) bundle.Mutator {
+	return &validateClusterLibraries{engine: e}
+}
+
+func (m *validateClusterLibraries) Name() string {
+	return "ValidateClusterLibraries"
+}
+
+func (m *validateClusterLibraries) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
+	if m.engine.IsDirect() {
+		return nil
+	}
+
+	var diags diag.Diagnostics
+	for key, cluster := range b.Config.Resources.Clusters {
+		if cluster == nil || len(cluster.Libraries) == 0 {
+			continue
+		}
+		path := "resources.clusters." + key + ".libraries"
+		diags = diags.Append(diag.Diagnostic{
+			Severity:  diag.Error,
+			Summary:   "cluster libraries are only supported in direct deployment mode",
+			Locations: b.Config.GetLocations(path),
+		})
+	}
+	return diags
+}

bundle/config/resources/clusters.go

@@ -18,6 +18,9 @@ type Cluster struct {
 	// Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started.
 	Lifecycle *LifecycleWithStarted `json:"lifecycle,omitempty"`
 
+	// Libraries are installed via the Libraries API, not the cluster spec.
+	Libraries []compute.Library `json:"libraries,omitempty"`
+
 	Permissions []ClusterPermission `json:"permissions,omitempty"`
 }
 

bundle/config/resources_types.go

@@ -43,6 +43,11 @@ var ResourcesTypes = func() map[string]reflect.Type {
 			if resourceField.Name == "Grants" {
 				grantsKey := name + ".grants"
 				res[grantsKey] = resourceField.Type
+				continue
+			}
+			if resourceField.Name == "Libraries" {
+				librariesKey := name + ".libraries"
+				res[librariesKey] = resourceField.Type
 			}
 		}
 	}

bundle/config/root.go

@@ -620,7 +620,7 @@ func GetNodeAndType(path dyn.Path) (dyn.Path, string) {
 	}
 
 	if len(path) >= 4 {
-		if path[3].Key() == "permissions" || path[3].Key() == "grants" {
+		if path[3].Key() == "permissions" || path[3].Key() == "grants" || path[3].Key() == "libraries" {
 			return path[:4], path[1].Key() + "." + path[3].Key()
 		}
 	}

bundle/direct/bundle_plan.go

@@ -713,7 +713,7 @@ func splitResourcePath(path *structpath.PathNode) (string, *structpath.PathNode)
 	// Check if the 4th component is "permissions" or "grants" (sub-resource)
 	if path.Len() > 4 {
 		first := path.SkipPrefix(3).Prefix(1)
-		if key, ok := first.StringKey(); ok && (key == "permissions" || key == "grants") {
+		if key, ok := first.StringKey(); ok && (key == "permissions" || key == "grants" || key == "libraries") {
 			return path.Prefix(4).String(), path.SkipPrefix(4)
 		}
 	}
@@ -930,6 +930,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root
 		dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()),
 		dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")),
 		dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")),
+		dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("libraries")),
 	}
 
 	// Walk?

bundle/direct/dresources/all.go

@@ -61,6 +61,9 @@ var SupportedResources = map[string]any{
 	"vector_search_endpoints.permissions": (*ResourcePermissions)(nil),
 	"instance_pools.permissions":          (*ResourcePermissions)(nil),
 
+	// Libraries
+	"clusters.libraries": (*ResourceLibraries)(nil),
+
 	// Grants
 	"catalogs.grants":              (*ResourceGrants)(nil),
 	"schemas.grants":               (*ResourceGrants)(nil),

bundle/direct/dresources/all_test.go

@@ -452,6 +452,24 @@ var testDeps = map[string]prepareWorkspace{
 		}, nil
 	},
 
+	"clusters.libraries": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) {
+		wait, err := client.Clusters.Create(ctx, compute.CreateCluster{
+			ClusterName:  "libraries-cluster",
+			SparkVersion: "13.3.x-scala2.12",
+			NodeTypeId:   "m5.large",
+			NumWorkers:   1,
+		})
+		if err != nil {
+			return nil, err
+		}
+		return &LibrariesState{
+			ClusterId: wait.ClusterId,
+			EmbeddedSlice: []compute.Library{
+				{Whl: "/Workspace/Users/test/lib.whl"},
+			},
+		}, nil
+	},
+
 	"cluster_policies.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) {
 		return &PermissionsState{
 			ObjectID: "/cluster-policies/cluster-policy-permissions",
@@ -1127,7 +1145,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W
 		require.NoError(t, err)
 	}
 
-	deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants")
+	deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") || strings.HasSuffix(group, "libraries")
 	// Apps DoDelete is fire-and-forget: the API returns success while the app
 	// sits in DELETING state for up to ~20 minutes before the record is removed.
 	// A GET on the DELETING app returns the app, not 404 -- the testserver