From e4e688e797ad174c864f21d26b97c1cf5fbca513 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 24 Aug 2026 13:20:37 +0000 Subject: [PATCH 01/14] add libraries field for Clusters --- bundle/config/resources/clusters.go | 3 +++ bundle/internal/schema/annotations.yml | 3 +++ bundle/schema/jsonschema.json | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/bundle/config/resources/clusters.go b/bundle/config/resources/clusters.go index 235ea6eee1a..cb69fc6b752 100644 --- a/bundle/config/resources/clusters.go +++ b/bundle/config/resources/clusters.go @@ -18,6 +18,9 @@ type Cluster struct { // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. Lifecycle *LifecycleWithStarted `json:"lifecycle,omitempty"` + // Libraries are installed via the Libraries API, not the cluster spec. + Libraries []compute.Library `json:"libraries,omitempty"` + Permissions []ClusterPermission `json:"permissions,omitempty"` } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index e47f89c505a..e01d21460d3 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -605,6 +605,9 @@ resources: notebook_path: "./src/my_notebook.py" ``` "$fields": + "libraries": + "description": |- + A list of libraries to install on the cluster. Installed via the Libraries API after the cluster is created. Only supported in direct deployment mode. "lifecycle": "description": |- Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 425956ab06d..f32d43532f0 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -488,6 +488,10 @@ "description": "The kind of compute described by this compute specification.\n\nDepending on `kind`, different validations and default values will be applied.\n\nClusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas clusters with no specified `kind` do not.\n* [is_single_node](/api/workspace/clusters/create#is_single_node)\n* [use_ml_runtime](/api/workspace/clusters/create#use_ml_runtime)\n\nBy using the [simple form](https://docs.databricks.com/compute/simple-form.html), your clusters are automatically using `kind = CLASSIC_PREVIEW`.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Kind" }, + "libraries": { + "description": "A list of libraries to install on the cluster. Installed via the Libraries API after the cluster is created. Only supported in direct deployment mode.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.Library" + }, "lifecycle": { "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted" From 04e21281d4ce7607422d49538b9e9a86da3a374a Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 13:57:59 +0000 Subject: [PATCH 02/14] Recognize clusters.libraries as a sub-resource node Wire the direct engine to treat resources.clusters.*.libraries as a child-resource node, the same way permissions and grants are handled: node/type resolution, reference splitting, and plan node discovery. Co-authored-by: Isaac --- bundle/config/resources_types.go | 5 +++++ bundle/config/root.go | 2 +- bundle/direct/bundle_plan.go | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bundle/config/resources_types.go b/bundle/config/resources_types.go index dcc91545f19..21d6d407468 100644 --- a/bundle/config/resources_types.go +++ b/bundle/config/resources_types.go @@ -43,6 +43,11 @@ var ResourcesTypes = func() map[string]reflect.Type { if resourceField.Name == "Grants" { grantsKey := name + ".grants" res[grantsKey] = resourceField.Type + continue + } + if resourceField.Name == "Libraries" { + librariesKey := name + ".libraries" + res[librariesKey] = resourceField.Type } } } diff --git a/bundle/config/root.go b/bundle/config/root.go index e13bf78bf15..0d7d03dadb3 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -620,7 +620,7 @@ func GetNodeAndType(path dyn.Path) (dyn.Path, string) { } if len(path) >= 4 { - if path[3].Key() == "permissions" || path[3].Key() == "grants" { + if path[3].Key() == "permissions" || path[3].Key() == "grants" || path[3].Key() == "libraries" { return path[:4], path[1].Key() + "." + path[3].Key() } } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 5b8829e3f59..4d37a21e090 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -713,7 +713,7 @@ func splitResourcePath(path *structpath.PathNode) (string, *structpath.PathNode) // Check if the 4th component is "permissions" or "grants" (sub-resource) if path.Len() > 4 { first := path.SkipPrefix(3).Prefix(1) - if key, ok := first.StringKey(); ok && (key == "permissions" || key == "grants") { + if key, ok := first.StringKey(); ok && (key == "permissions" || key == "grants" || key == "libraries") { return path.Prefix(4).String(), path.SkipPrefix(4) } } @@ -930,6 +930,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("libraries")), } // Walk? From a6473ddb79f77943d62228d8ea8e992334cccc01 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 13:58:10 +0000 Subject: [PATCH 03/14] Add clusters.libraries direct-engine child resource Implement ResourceLibraries: installs/uninstalls cluster libraries via the Libraries API, reconciling removed libraries on update and polling for install completion on a running cluster. Registered in all.go. Note: TestAll/clusters.libraries fails until the testserver models the libraries install/uninstall/cluster-status endpoints (next step). Co-authored-by: Isaac --- bundle/direct/dresources/all.go | 3 + bundle/direct/dresources/cluster_libraries.go | 253 ++++++++++++++++++ bundle/direct/dresources/type_test.go | 1 + 3 files changed, 257 insertions(+) create mode 100644 bundle/direct/dresources/cluster_libraries.go diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 391fb0684d2..4ffeed88ab2 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -61,6 +61,9 @@ var SupportedResources = map[string]any{ "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), "instance_pools.permissions": (*ResourcePermissions)(nil), + // Libraries + "clusters.libraries": (*ResourceLibraries)(nil), + // Grants "catalogs.grants": (*ResourceGrants)(nil), "schemas.grants": (*ResourceGrants)(nil), diff --git a/bundle/direct/dresources/cluster_libraries.go b/bundle/direct/dresources/cluster_libraries.go new file mode 100644 index 00000000000..a629dfe08ee --- /dev/null +++ b/bundle/direct/dresources/cluster_libraries.go @@ -0,0 +1,253 @@ +package dresources + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/retries" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +// Corresponds to the databricks_library terraform resource: +// https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/library + +// librariesWaitTimeout bounds how long we poll for libraries to finish installing. +const librariesWaitTimeout = 15 * time.Minute + +// LibrariesState is the state for a cluster's libraries sub-resource. Libraries are installed +// via the Libraries API against the parent cluster identified by ClusterId, not through the +// cluster spec. +type LibrariesState struct { + ClusterId string `json:"cluster_id"` + // By convention EmbeddedSlice fields have the __embed__ json tag, see permissions.go. + EmbeddedSlice []compute.Library `json:"__embed__,omitempty"` +} + +type ResourceLibraries struct { + client *databricks.WorkspaceClient +} + +func (*ResourceLibraries) New(client *databricks.WorkspaceClient) *ResourceLibraries { + return &ResourceLibraries{client: client} +} + +func (r *ResourceLibraries) PrepareInputConfig(inputConfig *[]compute.Library, resourceKey string) (*structvar.StructVar, error) { + baseNode, ok := strings.CutSuffix(resourceKey, ".libraries") + if !ok { + return nil, fmt.Errorf("internal error: node %q does not end with .libraries", resourceKey) + } + + return &structvar.StructVar{ + Value: &LibrariesState{ + ClusterId: "", // Always a reference, defined in Refs below. + EmbeddedSlice: *inputConfig, + }, + Refs: map[string]string{ + "cluster_id": "${" + baseNode + ".id}", + }, + }, nil +} + +func (*ResourceLibraries) PrepareState(state *LibrariesState) *LibrariesState { + return state +} + +// IsEmptyState reports an empty libraries list as no resource at all: nothing to install, and no +// state entry is persisted for it. +func (*ResourceLibraries) IsEmptyState(state *LibrariesState) bool { + return len(state.EmbeddedSlice) == 0 +} + +// libraryKey identifies a library by its type-specific field so slices compare by identity +// rather than by index (see KeyedSlices). +func libraryKey(l compute.Library) (string, string) { + switch { + case l.Whl != "": + return "whl", l.Whl + case l.Jar != "": + return "jar", l.Jar + case l.Egg != "": + return "egg", l.Egg + case l.Requirements != "": + return "requirements", l.Requirements + case l.Pypi != nil: + return "pypi", l.Pypi.Package + case l.Maven != nil: + return "maven", l.Maven.Coordinates + case l.Cran != nil: + return "cran", l.Cran.Package + } + return "", "" +} + +func (*ResourceLibraries) KeyedSlices() map[string]any { + // Empty key because EmbeddedSlice appears at the root path of LibrariesState. + return map[string]any{ + "": libraryKey, + } +} + +func (r *ResourceLibraries) DoRead(ctx context.Context, id string) (*LibrariesState, error) { + statuses, err := r.client.Libraries.ClusterStatusByClusterId(ctx, id) + if err != nil { + return nil, err + } + + state := &LibrariesState{ClusterId: id} + for _, s := range statuses.LibraryStatuses { + // Libraries set for all clusters via the UI are not managed by the bundle + // (following the permissions convention of ignoring inherited entries). + if s.Library == nil || s.IsLibraryForAllClusters { + continue + } + // A library pending uninstall on restart is on its way out; don't report it as present. + if s.Status == compute.LibraryInstallStatusUninstallOnRestart { + continue + } + state.EmbeddedSlice = append(state.EmbeddedSlice, *s.Library) + } + return state, nil +} + +// DoCreate installs the libraries on the cluster. +// https://docs.databricks.com/api/workspace/libraries/install +func (r *ResourceLibraries) DoCreate(ctx context.Context, state *LibrariesState) (string, *LibrariesState, error) { + err := r.client.Libraries.Install(ctx, compute.InstallLibraries{ + ClusterId: state.ClusterId, + Libraries: state.EmbeddedSlice, + }) + if err != nil { + // Install is idempotent (installing an already-installed library is a no-op), + // so retrying on transient errors is safe. + return "", nil, retrySafe(err) + } + return state.ClusterId, nil, nil +} + +// DoUpdate uninstalls libraries removed from config and installs the desired set. This is two API +// calls because the Libraries API exposes install and uninstall as separate endpoints, unlike the +// single-call model most resources follow. +func (r *ResourceLibraries) DoUpdate(ctx context.Context, id string, state *LibrariesState, entry *PlanEntry) (*LibrariesState, error) { + removed := removedLibraries(state.EmbeddedSlice, entry) + if len(removed) > 0 { + err := r.client.Libraries.Uninstall(ctx, compute.UninstallLibraries{ + ClusterId: id, + Libraries: removed, + }) + if err != nil { + return nil, err + } + } + + if len(state.EmbeddedSlice) > 0 { + err := r.client.Libraries.Install(ctx, compute.InstallLibraries{ + ClusterId: id, + Libraries: state.EmbeddedSlice, + }) + if err != nil { + return nil, err + } + } + return nil, nil +} + +// DoDelete is a no-op: removing individual libraries is handled by DoUpdate's uninstall diff, and +// DoDelete only fires when the parent cluster is deleted, at which point uninstalling is moot. +func (r *ResourceLibraries) DoDelete(ctx context.Context, id string, _ *LibrariesState) error { + return nil +} + +// removedLibraries returns libraries present in the remote state but absent from the desired set. +func removedLibraries(desired []compute.Library, entry *PlanEntry) []compute.Library { + if entry == nil { + return nil + } + remote, ok := entry.RemoteState.(*LibrariesState) + if !ok || remote == nil { + return nil + } + + desiredKeys := make(map[string]struct{}, len(desired)) + for _, l := range desired { + desiredKeys[libraryMapKey(l)] = struct{}{} + } + + var result []compute.Library + for _, l := range remote.EmbeddedSlice { + if _, ok := desiredKeys[libraryMapKey(l)]; !ok { + result = append(result, l) + } + } + return result +} + +// libraryMapKey flattens libraryKey into a single string for map lookups. +func libraryMapKey(l compute.Library) string { + f, v := libraryKey(l) + return f + "=" + v +} + +func (r *ResourceLibraries) WaitAfterCreate(ctx context.Context, id string, state *LibrariesState) (*LibrariesState, error) { + return nil, r.waitForInstall(ctx, id, state.EmbeddedSlice) +} + +func (r *ResourceLibraries) WaitAfterUpdate(ctx context.Context, id string, state *LibrariesState) (*LibrariesState, error) { + return nil, r.waitForInstall(ctx, id, state.EmbeddedSlice) +} + +// waitForInstall polls until every desired library reaches a terminal installed state. It returns +// early without waiting when the cluster is not running: installs only progress on a running +// cluster and are queued until it next starts. +func (r *ResourceLibraries) waitForInstall(ctx context.Context, id string, desired []compute.Library) error { + if len(desired) == 0 { + return nil + } + + details, err := r.client.Clusters.GetByClusterId(ctx, id) + if err != nil { + return err + } + if details.State != compute.StateRunning { + log.Debugf(ctx, "cluster %s is not running (%s); skipping wait for library installation", id, details.State) + return nil + } + + desiredKeys := make(map[string]struct{}, len(desired)) + for _, l := range desired { + desiredKeys[libraryMapKey(l)] = struct{}{} + } + + _, err = retries.Poll(ctx, librariesWaitTimeout, func() (*struct{}, *retries.Err) { + statuses, err := r.client.Libraries.ClusterStatusByClusterId(ctx, id) + if err != nil { + return nil, retries.Halt(err) + } + + pending := len(desiredKeys) + for _, s := range statuses.LibraryStatuses { + if s.Library == nil { + continue + } + if _, ok := desiredKeys[libraryMapKey(*s.Library)]; !ok { + continue + } + switch s.Status { + case compute.LibraryInstallStatusFailed: + return nil, retries.Halt(fmt.Errorf("library %s failed to install: %s", libraryMapKey(*s.Library), strings.Join(s.Messages, "; "))) + case compute.LibraryInstallStatusInstalled, compute.LibraryInstallStatusSkipped, compute.LibraryInstallStatusRestored: + pending-- + } + } + + if pending > 0 { + return nil, retries.Continues(fmt.Sprintf("waiting for %d librar(ies) to install on cluster %s", pending, id)) + } + return &struct{}{}, nil + }) + return err +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 2d5516d59c7..e471d6b2b6e 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -66,6 +66,7 @@ var knownMissingInRemoteType = map[string][]string{ // These are bundle-specific fields that exist in InputType but not in StateType. var commonMissingInStateType = []string{ "grants", + "libraries", "lifecycle", "permissions", } From be83d49729c216952932a28ca7cffc74935f7ca2 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:07:36 +0000 Subject: [PATCH 04/14] Model cluster libraries endpoints in testserver Add stateful fakes for the Libraries API (install, uninstall, cluster-status) so clusters.libraries runs against the in-process server. Add the TestAll fixture and classify libraries as a no-op delete alongside permissions/grants; this greens TestAll/clusters.libraries. Co-authored-by: Isaac --- bundle/direct/dresources/all_test.go | 20 ++++++- libs/testserver/fake_workspace.go | 4 +- libs/testserver/handlers.go | 13 +++++ libs/testserver/libraries.go | 87 ++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 libs/testserver/libraries.go diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 0c0d3d05ada..7ec1e9b3b72 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -452,6 +452,24 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "clusters.libraries": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + wait, err := client.Clusters.Create(ctx, compute.CreateCluster{ + ClusterName: "libraries-cluster", + SparkVersion: "13.3.x-scala2.12", + NodeTypeId: "m5.large", + NumWorkers: 1, + }) + if err != nil { + return nil, err + } + return &LibrariesState{ + ClusterId: wait.ClusterId, + EmbeddedSlice: []compute.Library{ + {Whl: "/Workspace/Users/test/lib.whl"}, + }, + }, nil + }, + "cluster_policies.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { return &PermissionsState{ ObjectID: "/cluster-policies/cluster-policy-permissions", @@ -1127,7 +1145,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NoError(t, err) } - deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") + deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") || strings.HasSuffix(group, "libraries") // Apps DoDelete is fire-and-forget: the API returns success while the app // sits in DELETING state for up to ~20 minutes before the record is removed. // A GET on the DELETING app returns the app, not 404 -- the testserver diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 25178a9e75f..02865c15443 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -196,6 +196,7 @@ type FakeWorkspace struct { ModelRegistryModels map[string]ml.Model ModelRegistryModelIDs map[string]string // model name -> numeric ID Clusters map[string]compute.ClusterDetails + ClusterLibraries map[string][]compute.Library // cluster id -> installed libraries InstancePools map[string]compute.GetInstancePool ClusterPolicies map[string]compute.Policy Catalogs map[string]catalog.CatalogInfo @@ -430,7 +431,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { SingleUserName: TestUser.UserName, }, }, - InstancePools: map[string]compute.GetInstancePool{}, + InstancePools: map[string]compute.GetInstancePool{}, + ClusterLibraries: map[string][]compute.Library{}, ClusterPolicies: map[string]compute.Policy{ // Seeded so the stateful list keeps backing the variable-lookup tests // (e.g. acceptance/bundle/variables/env_overrides resolves these by name). diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8c44d97eae7..e1ea0baa471 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -927,6 +927,19 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.ClustersPermanentDelete(req) }) + // Cluster libraries: + server.Handle("POST", "/api/2.0/libraries/install", func(req Request) any { + return req.Workspace.LibrariesInstall(req) + }) + + server.Handle("POST", "/api/2.0/libraries/uninstall", func(req Request) any { + return req.Workspace.LibrariesUninstall(req) + }) + + server.Handle("GET", "/api/2.0/libraries/cluster-status", func(req Request) any { + return req.Workspace.LibrariesClusterStatus(req, req.URL.Query().Get("cluster_id")) + }) + // MLflow Experiments: server.Handle("GET", "/api/2.0/mlflow/experiments/get", func(req Request) any { experimentId := req.URL.Query().Get("experiment_id") diff --git a/libs/testserver/libraries.go b/libs/testserver/libraries.go new file mode 100644 index 00000000000..63042ca8e66 --- /dev/null +++ b/libs/testserver/libraries.go @@ -0,0 +1,87 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "net/http" + "reflect" + + "github.com/databricks/databricks-sdk-go/service/compute" +) + +func (s *FakeWorkspace) LibrariesInstall(req Request) any { + var request compute.InstallLibraries + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: http.StatusBadRequest, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + if _, ok := s.Clusters[request.ClusterId]; !ok { + return Response{StatusCode: http.StatusNotFound} + } + + // Install is additive and idempotent: installing an already-present library is a no-op. + installed := s.ClusterLibraries[request.ClusterId] + for _, lib := range request.Libraries { + if !containsLibrary(installed, lib) { + installed = append(installed, lib) + } + } + s.ClusterLibraries[request.ClusterId] = installed + + return Response{} +} + +func (s *FakeWorkspace) LibrariesUninstall(req Request) any { + var request compute.UninstallLibraries + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: http.StatusBadRequest, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + installed := s.ClusterLibraries[request.ClusterId] + remaining := make([]compute.Library, 0, len(installed)) + for _, lib := range installed { + if !containsLibrary(request.Libraries, lib) { + remaining = append(remaining, lib) + } + } + s.ClusterLibraries[request.ClusterId] = remaining + + return Response{} +} + +func (s *FakeWorkspace) LibrariesClusterStatus(req Request, clusterId string) any { + defer s.LockUnlock()() + + if _, ok := s.Clusters[clusterId]; !ok { + return Response{StatusCode: http.StatusNotFound} + } + + installed := s.ClusterLibraries[clusterId] + statuses := make([]compute.LibraryFullStatus, 0, len(installed)) + for i := range installed { + statuses = append(statuses, compute.LibraryFullStatus{ + Library: &installed[i], + Status: compute.LibraryInstallStatusInstalled, + }) + } + + return Response{ + Body: compute.ClusterLibraryStatuses{ + ClusterId: clusterId, + LibraryStatuses: statuses, + }, + } +} + +func containsLibrary(libs []compute.Library, target compute.Library) bool { + for _, l := range libs { + if reflect.DeepEqual(l, target) { + return true + } + } + return false +} From 822baaa397821dc00e3a53c02f6b993e9f2d8e8f Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:16:18 +0000 Subject: [PATCH 05/14] Wire cluster libraries into the wheel build/upload pipeline Add clusterLibrariesPattern (resources.clusters.*.libraries) to glob expansion, local-library collection/upload, duplicate-name checking, and patched-wheel swapping, mirroring the job task library wiring. Local whl/jar globs now build, upload to artifact_path/.internal, and rewrite to absolute workspace paths; pypi/maven entries pass through unchanged. Co-authored-by: Isaac --- bundle/libraries/expand_glob_references.go | 11 +++++++++++ bundle/libraries/remote_path.go | 2 ++ bundle/libraries/same_name_libraries.go | 2 ++ bundle/libraries/switch_to_patched_wheels.go | 16 ++++++++++++++++ 4 files changed, 31 insertions(+) diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index 720142fe6d7..ab1da3df68d 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -198,6 +198,13 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) +var clusterLibrariesPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("clusters"), + dyn.AnyKey(), + dyn.Key("libraries"), +) + func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { expanders := []expandPattern{ { @@ -216,6 +223,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { pattern: pipelineEnvDepsPattern, fn: expandEnvironmentDeps, }, + { + pattern: clusterLibrariesPattern, + fn: expandLibraries, + }, } var diags diag.Diagnostics diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index 02a1172f36d..e40de7460df 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -68,6 +68,8 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), // The AI Runtime task's code_source_path is a local archive (typically an diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 49776fbd8c9..8fb140d7aa4 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -17,6 +17,8 @@ var patterns = []dyn.Pattern{ taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), } diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 56250d713a7..3c5ff184a97 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -79,6 +79,22 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag } } + // Update resources.clusters.*.libraries[*].whl + for clusterName, clusterRef := range b.Config.Resources.Clusters { + if clusterRef == nil { + continue + } + for libInd, lib := range clusterRef.Libraries { + repl := replacements[lib.Whl] + if repl != "" { + log.Debugf(ctx, "Updating resources.clusters.%s.libraries[%d].whl from %s to %s", clusterName, libInd, lib.Whl, repl) + clusterRef.Libraries[libInd].Whl = repl + } else { + log.Debugf(ctx, "Not updating resources.clusters.%s.libraries[%d].whl from %s. Available replacements: %v", clusterName, libInd, lib.Whl, slices.Sorted(maps.Keys(replacements))) + } + } + } + return nil } From d8f8afa8f72442baf6d5e6bf7c5f42023684eebc Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:21:34 +0000 Subject: [PATCH 06/14] Reject cluster libraries under the terraform engine Add ValidateClusterLibraries and register it in PreDeployChecks so a libraries block on a cluster errors under the terraform engine instead of being silently dropped. Cluster libraries are direct-only. Mirrors the existing lifecycle.started guard. Co-authored-by: Isaac --- .../mutator/validate_cluster_libraries.go | 43 +++++++++++++++++++ bundle/phases/plan.go | 1 + 2 files changed, 44 insertions(+) create mode 100644 bundle/config/mutator/validate_cluster_libraries.go diff --git a/bundle/config/mutator/validate_cluster_libraries.go b/bundle/config/mutator/validate_cluster_libraries.go new file mode 100644 index 00000000000..f13cd3be373 --- /dev/null +++ b/bundle/config/mutator/validate_cluster_libraries.go @@ -0,0 +1,43 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/diag" +) + +type validateClusterLibraries struct { + engine engine.EngineType +} + +// ValidateClusterLibraries returns a mutator that errors when cluster libraries are used with +// the terraform deployment engine. Cluster libraries are only supported in direct deployment mode. +func ValidateClusterLibraries(e engine.EngineType) bundle.Mutator { + return &validateClusterLibraries{engine: e} +} + +func (m *validateClusterLibraries) Name() string { + return "ValidateClusterLibraries" +} + +func (m *validateClusterLibraries) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + if m.engine.IsDirect() { + return nil + } + + var diags diag.Diagnostics + for key, cluster := range b.Config.Resources.Clusters { + if cluster == nil || len(cluster.Libraries) == 0 { + continue + } + path := "resources.clusters." + key + ".libraries" + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "cluster libraries are only supported in direct deployment mode", + Locations: b.Config.GetLocations(path), + }) + } + return diags +} diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 3db0864c2c4..208582a2d9d 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -28,6 +28,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), mutator.ValidateLifecycleStarted(engine), + mutator.ValidateClusterLibraries(engine), mutator.ValidateCascadeOnDestroy(engine), mutator.ValidateJobRunTriggers(), statemgmt.CheckRunningResource(engine), From 54f4ae322bdd73546f215c7f4e96cbb195c3545b Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:30:29 +0000 Subject: [PATCH 07/14] Regenerate terraform<->DABs field map for cluster libraries Adding the clusters libraries field makes DABs libraries map to the terraform databricks_cluster.library field; regenerate the mapping so reference translation and the tf-only field audit stay correct. Co-authored-by: Isaac --- bundle/terraform_dabs_map/generated.go | 32 ++++++++------------------ 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/bundle/terraform_dabs_map/generated.go b/bundle/terraform_dabs_map/generated.go index d0a783f35ac..f23c0bfa9eb 100644 --- a/bundle/terraform_dabs_map/generated.go +++ b/bundle/terraform_dabs_map/generated.go @@ -6,7 +6,8 @@ package terraform_dabs_map // alerts / databricks_alert_v2: 3 tf-only // apps / databricks_app: 6 dabs-only // apps / databricks_app: 1 tf-only -// clusters / databricks_cluster: 26 tf-only +// clusters / databricks_cluster: 1 renames +// clusters / databricks_cluster: 11 tf-only // dashboards / databricks_dashboard: 2 tf-only // database_instances / databricks_database_instance: 1 tf-only // experiments / databricks_mlflow_experiment: 1 tf-only @@ -34,6 +35,9 @@ package terraform_dabs_map // TerraformToDABsFieldMap maps DABs group name → nested TF segments → DABs segment name. // Navigate using TF field name segments; DABs is the corresponding DABs name when it differs. var TerraformToDABsFieldMap = map[string]RenameTree{ + "clusters": { + "library": {NewName: "libraries"}, + }, "jobs": { "environment": {NewName: "environments"}, "git_source": {Children: RenameTree{ @@ -168,27 +172,8 @@ var TerraformOnlyFields = map[string]FieldSet{ }, "idempotency_token": {}, "is_pinned": {}, - "library": { - "cran": { - "package": {}, // databricks_cluster.*.library.cran.package - "repo": {}, // databricks_cluster.*.library.cran.repo - }, - "egg": {}, // databricks_cluster.*.library.egg - "jar": {}, // databricks_cluster.*.library.jar - "maven": { - "coordinates": {}, // databricks_cluster.*.library.maven.coordinates - "exclusions": {}, // databricks_cluster.*.library.maven.exclusions - "repo": {}, // databricks_cluster.*.library.maven.repo - }, - "pypi": { - "package": {}, // databricks_cluster.*.library.pypi.package - "repo": {}, // databricks_cluster.*.library.pypi.repo - }, - "requirements": {}, // databricks_cluster.*.library.requirements - "whl": {}, // databricks_cluster.*.library.whl - }, - "no_wait": {}, - "url": {}, + "no_wait": {}, + "url": {}, }, "dashboards": { "dashboard_change_detected": {}, @@ -569,6 +554,9 @@ var TerraformOnlyFields = map[string]FieldSet{ // DABsToTerraformRenameMap maps DABs group name → nested DABs segments → TF segment name. // Navigate using DABs field name segments; NewName is the TF name when it differs. var DABsToTerraformRenameMap = map[string]RenameTree{ + "clusters": { + "libraries": {NewName: "library"}, + }, "jobs": { "environments": {NewName: "environment"}, "git_source": {Children: RenameTree{ From 18a1d0723f921bccb94da8e1053ffb99c7070654 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:30:30 +0000 Subject: [PATCH 08/14] Add acceptance tests and changelog for cluster libraries - clusters/libraries: direct-engine deploy installs pypi + local wheel (rewritten to its uploaded path), and removing the wheel and redeploying uninstalls it. - clusters/libraries-terraform-error: bundle plan/deploy reject cluster libraries under the terraform engine. Co-authored-by: Isaac --- .nextchanges/bundles/cluster-libraries.md | 1 + .../libraries-terraform-error/databricks.yml | 13 +++++ .../libraries-terraform-error/out.test.toml | 2 + .../libraries-terraform-error/output.txt | 16 +++++ .../clusters/libraries-terraform-error/script | 5 ++ .../libraries-terraform-error/test.toml | 4 ++ .../clusters/libraries/databricks.yml | 14 +++++ .../clusters/libraries/out.test.toml | 2 + .../resources/clusters/libraries/output.txt | 58 +++++++++++++++++++ .../resources/clusters/libraries/script | 16 +++++ .../resources/clusters/libraries/test.toml | 10 ++++ 11 files changed, 141 insertions(+) create mode 100644 .nextchanges/bundles/cluster-libraries.md create mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml create mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt create mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/script create mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries/databricks.yml create mode 100644 acceptance/bundle/resources/clusters/libraries/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries/output.txt create mode 100644 acceptance/bundle/resources/clusters/libraries/script create mode 100644 acceptance/bundle/resources/clusters/libraries/test.toml diff --git a/.nextchanges/bundles/cluster-libraries.md b/.nextchanges/bundles/cluster-libraries.md new file mode 100644 index 00000000000..4c9b1a52006 --- /dev/null +++ b/.nextchanges/bundles/cluster-libraries.md @@ -0,0 +1 @@ +Add support for a `libraries` list on the `clusters` resource type in Declarative Automation Bundles. Libraries (whl, jar, pypi, maven, cran, egg, requirements) are installed on the all-purpose cluster via the Libraries API; local wheels/jars are built and uploaded automatically. Cluster libraries are only supported in direct deployment mode. diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml b/acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml new file mode 100644 index 00000000000..e2645b5cc5e --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: cluster-libraries-terraform-error + +resources: + clusters: + mycluster: + cluster_name: mycluster + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml b/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml new file mode 100644 index 00000000000..d2059b4b5d7 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt b/acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt new file mode 100644 index 00000000000..aa0a6711e7e --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt @@ -0,0 +1,16 @@ + +=== bundle plan fails with cluster libraries on terraform engine +>>> errcode [CLI] bundle plan +Error: cluster libraries are only supported in direct deployment mode + in databricks.yml:12:9 + + +Exit code: 1 + +=== bundle deploy fails with cluster libraries on terraform engine +>>> errcode [CLI] bundle deploy +Error: cluster libraries are only supported in direct deployment mode + in databricks.yml:12:9 + + +Exit code: 1 diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/script b/acceptance/bundle/resources/clusters/libraries-terraform-error/script new file mode 100644 index 00000000000..93db6e06309 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/script @@ -0,0 +1,5 @@ +title "bundle plan fails with cluster libraries on terraform engine" +trace errcode $CLI bundle plan + +title "bundle deploy fails with cluster libraries on terraform engine" +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml b/acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml new file mode 100644 index 00000000000..e4a0f1c6301 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml @@ -0,0 +1,4 @@ +Cloud = false +RecordRequests = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/clusters/libraries/databricks.yml b/acceptance/bundle/resources/clusters/libraries/databricks.yml new file mode 100644 index 00000000000..922f719fda4 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: cluster-libraries + +resources: + clusters: + mycluster: + cluster_name: mycluster + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + libraries: + - pypi: + package: requests + - whl: ./dist/*.whl diff --git a/acceptance/bundle/resources/clusters/libraries/out.test.toml b/acceptance/bundle/resources/clusters/libraries/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries/output.txt b/acceptance/bundle/resources/clusters/libraries/output.txt new file mode 100644 index 00000000000..74c60339a00 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/output.txt @@ -0,0 +1,58 @@ + +=== Deploy a cluster with a pypi and a local wheel library +>>> [CLI] bundle deploy +Uploading dist/my_package-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files... +Created clusters.mycluster +Created clusters.mycluster.libraries +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Libraries installed via the Libraries API (wheel rewritten to its uploaded path) +>>> print_requests.py //libraries/install +{ + "method": "POST", + "path": "/api/2.0/libraries/install", + "body": { + "cluster_id": "[UUID]", + "libraries": [ + { + "pypi": { + "package": "requests" + } + }, + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/artifacts/.internal/my_package-0.0.1-py3-none-any.whl" + } + ] + } +} + +=== Removing the wheel and redeploying uninstalls it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files... +Updated clusters.mycluster.libraries +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //libraries/uninstall +{ + "method": "POST", + "path": "/api/2.0/libraries/uninstall", + "body": { + "cluster_id": "[UUID]", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/artifacts/.internal/my_package-0.0.1-py3-none-any.whl" + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries/script b/acceptance/bundle/resources/clusters/libraries/script new file mode 100644 index 00000000000..33af4f7c5d5 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/script @@ -0,0 +1,16 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy a cluster with a pypi and a local wheel library" +trace $CLI bundle deploy + +title "Libraries installed via the Libraries API (wheel rewritten to its uploaded path)" +trace print_requests.py //libraries/install + +title "Removing the wheel and redeploying uninstalls it" +update_file.py databricks.yml " - whl: ./dist/*.whl" "" +trace $CLI bundle deploy +trace print_requests.py //libraries/uninstall diff --git a/acceptance/bundle/resources/clusters/libraries/test.toml b/acceptance/bundle/resources/clusters/libraries/test.toml new file mode 100644 index 00000000000..93cb0543245 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/test.toml @@ -0,0 +1,10 @@ +Cloud = false +RecordRequests = true + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [".databricks"] + +[[Repls]] +Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" +New = "[CLUSTER-ID]" From 1d5923a546ae7e00bdcc2b146ecedc857daca24b Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:41:41 +0000 Subject: [PATCH 09/14] Fix lint in cluster_libraries (exhaustive switch, exhaustruct) Enumerate the remaining LibraryInstallStatus cases in the install-wait poll and set EmbeddedSlice explicitly in DoRead's state literal. Co-authored-by: Isaac --- bundle/direct/dresources/cluster_libraries.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bundle/direct/dresources/cluster_libraries.go b/bundle/direct/dresources/cluster_libraries.go index a629dfe08ee..a3458da761d 100644 --- a/bundle/direct/dresources/cluster_libraries.go +++ b/bundle/direct/dresources/cluster_libraries.go @@ -98,7 +98,7 @@ func (r *ResourceLibraries) DoRead(ctx context.Context, id string) (*LibrariesSt return nil, err } - state := &LibrariesState{ClusterId: id} + state := &LibrariesState{ClusterId: id, EmbeddedSlice: nil} for _, s := range statuses.LibraryStatuses { // Libraries set for all clusters via the UI are not managed by the bundle // (following the permissions convention of ignoring inherited entries). @@ -241,6 +241,8 @@ func (r *ResourceLibraries) waitForInstall(ctx context.Context, id string, desir return nil, retries.Halt(fmt.Errorf("library %s failed to install: %s", libraryMapKey(*s.Library), strings.Join(s.Messages, "; "))) case compute.LibraryInstallStatusInstalled, compute.LibraryInstallStatusSkipped, compute.LibraryInstallStatusRestored: pending-- + case compute.LibraryInstallStatusPending, compute.LibraryInstallStatusResolving, compute.LibraryInstallStatusInstalling, compute.LibraryInstallStatusUninstallOnRestart: + // Still in progress (or being removed); keep polling. } } From a5e6afdccd7da473910ec18b241ffb62664d6156 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:48:54 +0000 Subject: [PATCH 10/14] Add cloud drift check for cluster libraries clusters/libraries-drift (Cloud = true, direct engine): deploy a cluster with a pypi library, then assert the immediate re-plan is a no-op (0 to change). Verified on a real AWS workspace: the Libraries status API round-trips the library without drift, so no normalization is needed. Co-authored-by: Isaac --- .../libraries-drift/databricks.yml.tmpl | 17 ++++++++++++++++ .../clusters/libraries-drift/out.test.toml | 2 ++ .../clusters/libraries-drift/output.txt | 12 +++++++++++ .../resources/clusters/libraries-drift/script | 20 +++++++++++++++++++ .../clusters/libraries-drift/test.toml | 6 ++++++ 5 files changed, 57 insertions(+) create mode 100644 acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/clusters/libraries-drift/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries-drift/output.txt create mode 100644 acceptance/bundle/resources/clusters/libraries-drift/script create mode 100644 acceptance/bundle/resources/clusters/libraries-drift/test.toml diff --git a/acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl new file mode 100644 index 00000000000..b3bf2463741 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: cluster-libraries-drift-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + clusters: + mycluster: + cluster_name: mycluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID + num_workers: 1 + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-drift/out.test.toml b/acceptance/bundle/resources/clusters/libraries-drift/out.test.toml new file mode 100644 index 00000000000..c502b28221b --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries-drift/output.txt b/acceptance/bundle/resources/clusters/libraries-drift/output.txt new file mode 100644 index 00000000000..4a180dfbf0c --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/output.txt @@ -0,0 +1,12 @@ + +=== Plan is a no-op immediately after deploy (no library drift) +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-drift/script b/acceptance/bundle/resources/clusters/libraries-drift/script new file mode 100644 index 00000000000..b1fef7ee702 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/script @@ -0,0 +1,20 @@ +# A pypi library is used because its nested {package, repo} shape is the most +# likely to drift (the status API echoing a repo we did not set); a workspace +# wheel is covered by the local clusters/libraries test instead, since the +# shared cloud test cluster rejects libraries from /Workspace paths. +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# Cluster provisioning and library-install output is noisy and differs between +# the fake and cloud, so route it to LOG and assert only the deterministic +# drift signal below. +$CLI bundle deploy &> LOG.deploy +cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" > /dev/null + +title "Plan is a no-op immediately after deploy (no library drift)" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/clusters/libraries-drift/test.toml b/acceptance/bundle/resources/clusters/libraries-drift/test.toml new file mode 100644 index 00000000000..63a8c8a332c --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/test.toml @@ -0,0 +1,6 @@ +Cloud = true +RecordRequests = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [".databricks", "databricks.yml"] From 546938b6ee84f854743aa829d011336b59933bb5 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:53:23 +0000 Subject: [PATCH 11/14] Regenerate refschema and required_fields for cluster libraries Co-authored-by: Isaac --- acceptance/bundle/refschema/out.fields.txt | 34 +++++++++++++++++++ .../validation/generated/required_fields.go | 3 ++ 2 files changed, 37 insertions(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index de252dc4162..5e00e1b1f7e 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -460,6 +460,23 @@ resources.clusters.*.jdbc_port int REMOTE resources.clusters.*.kind compute.Kind ALL resources.clusters.*.last_restarted_time int64 REMOTE resources.clusters.*.last_state_loss_time int64 REMOTE +resources.clusters.*.libraries []compute.Library INPUT +resources.clusters.*.libraries[*] compute.Library INPUT +resources.clusters.*.libraries[*].cran *compute.RCranLibrary INPUT +resources.clusters.*.libraries[*].cran.package string INPUT +resources.clusters.*.libraries[*].cran.repo string INPUT +resources.clusters.*.libraries[*].egg string INPUT +resources.clusters.*.libraries[*].jar string INPUT +resources.clusters.*.libraries[*].maven *compute.MavenLibrary INPUT +resources.clusters.*.libraries[*].maven.coordinates string INPUT +resources.clusters.*.libraries[*].maven.exclusions []string INPUT +resources.clusters.*.libraries[*].maven.exclusions[*] string INPUT +resources.clusters.*.libraries[*].maven.repo string INPUT +resources.clusters.*.libraries[*].pypi *compute.PythonPyPiLibrary INPUT +resources.clusters.*.libraries[*].pypi.package string INPUT +resources.clusters.*.libraries[*].pypi.repo string INPUT +resources.clusters.*.libraries[*].requirements string INPUT +resources.clusters.*.libraries[*].whl string INPUT resources.clusters.*.lifecycle *dresources.StateLifecycle REMOTE STATE resources.clusters.*.lifecycle *resources.LifecycleWithStarted INPUT resources.clusters.*.lifecycle resources.Lifecycle INPUT @@ -610,6 +627,23 @@ resources.clusters.*.workload_type *compute.WorkloadType ALL resources.clusters.*.workload_type.clients compute.ClientsTypes ALL resources.clusters.*.workload_type.clients.jobs bool ALL resources.clusters.*.workload_type.clients.notebooks bool ALL +resources.clusters.*.libraries.cluster_id string ALL +resources.clusters.*.libraries[*] compute.Library ALL +resources.clusters.*.libraries[*].cran *compute.RCranLibrary ALL +resources.clusters.*.libraries[*].cran.package string ALL +resources.clusters.*.libraries[*].cran.repo string ALL +resources.clusters.*.libraries[*].egg string ALL +resources.clusters.*.libraries[*].jar string ALL +resources.clusters.*.libraries[*].maven *compute.MavenLibrary ALL +resources.clusters.*.libraries[*].maven.coordinates string ALL +resources.clusters.*.libraries[*].maven.exclusions []string ALL +resources.clusters.*.libraries[*].maven.exclusions[*] string ALL +resources.clusters.*.libraries[*].maven.repo string ALL +resources.clusters.*.libraries[*].pypi *compute.PythonPyPiLibrary ALL +resources.clusters.*.libraries[*].pypi.package string ALL +resources.clusters.*.libraries[*].pypi.repo string ALL +resources.clusters.*.libraries[*].requirements string ALL +resources.clusters.*.libraries[*].whl string ALL resources.clusters.*.permissions.object_id string ALL resources.clusters.*.permissions[*] dresources.StatePermission ALL resources.clusters.*.permissions[*].group_name string ALL diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 617f5a65ecc..d0937851098 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -56,6 +56,9 @@ var RequiredFields = map[string][]string{ "resources.clusters.*.init_scripts[*].s3": {"destination"}, "resources.clusters.*.init_scripts[*].volumes": {"destination"}, "resources.clusters.*.init_scripts[*].workspace": {"destination"}, + "resources.clusters.*.libraries[*].cran": {"package"}, + "resources.clusters.*.libraries[*].maven": {"coordinates"}, + "resources.clusters.*.libraries[*].pypi": {"package"}, "resources.clusters.*.permissions[*]": {"level"}, "resources.clusters.*.workload_type": {"clients"}, From ac8acd16f3a34849e24380b3994950513085a8ed Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 15:05:40 +0000 Subject: [PATCH 12/14] Scope libraries sub-resource to clusters only Pipelines have a native libraries field that is a plain field, not a child resource. The sub-resource wiring matched resources.*.*.libraries for every resource type, so the direct engine tried to plan pipelines.libraries as a resource and failed with 'unsupported resource type: pipelines.libraries'. Scope both GetNodeAndType and the plan pattern to clusters. Co-authored-by: Isaac --- bundle/config/root.go | 11 +++++++++-- bundle/direct/bundle_plan.go | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 0d7d03dadb3..c05851c3d86 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -620,8 +620,15 @@ func GetNodeAndType(path dyn.Path) (dyn.Path, string) { } if len(path) >= 4 { - if path[3].Key() == "permissions" || path[3].Key() == "grants" || path[3].Key() == "libraries" { - return path[:4], path[1].Key() + "." + path[3].Key() + sub := path[3].Key() + if sub == "permissions" || sub == "grants" { + return path[:4], path[1].Key() + "." + sub + } + // libraries is a sub-resource only for clusters. Other resource types + // (e.g. pipelines) have a native libraries field that is a plain field, + // not a child resource. + if sub == "libraries" && path[1].Key() == "clusters" { + return path[:4], path[1].Key() + "." + sub } } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 4d37a21e090..c9e08fe798e 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -930,7 +930,8 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("libraries")), + // libraries is a sub-resource only for clusters; other resource types have a native libraries field. + dyn.NewPattern(dyn.Key("resources"), dyn.Key("clusters"), dyn.AnyKey(), dyn.Key("libraries")), } // Walk? From 5e4c795024955ba12a7e2fbef024961e9e645bb2 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 15:09:02 +0000 Subject: [PATCH 13/14] Scope libraries entry in ResourcesTypes to clusters Consistency follow-up to the GetNodeAndType/plan-pattern scoping: ResourcesTypes registered a .libraries key for every resource type with a Libraries field, spuriously adding pipelines.libraries and cluster_policies.libraries. Those keys are unreachable now that GetNodeAndType is scoped, but scope this branch too so the map stays consistent and the entries don't mislead future callers. Co-authored-by: Isaac --- bundle/config/resources_types.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bundle/config/resources_types.go b/bundle/config/resources_types.go index 21d6d407468..20dbee0222a 100644 --- a/bundle/config/resources_types.go +++ b/bundle/config/resources_types.go @@ -45,7 +45,10 @@ var ResourcesTypes = func() map[string]reflect.Type { res[grantsKey] = resourceField.Type continue } - if resourceField.Name == "Libraries" { + // libraries is a child resource only for clusters. Pipelines and + // cluster_policies have a native Libraries field that is a plain + // field, not a child resource. + if resourceField.Name == "Libraries" && name == "clusters" { librariesKey := name + ".libraries" res[librariesKey] = resourceField.Type } From 6b15637e2ee53655c12fdd5be822d8e245585bb6 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 15:27:36 +0000 Subject: [PATCH 14/14] Cover clusters.libraries in invariant suite; commit wheel fixture Two CI failures in the local acceptance suite: 1. bundle/resources/clusters/libraries deployed a prebuilt wheel from ./dist/*.whl, but dist/ is gitignored so the fixture was never committed. CI's clean checkout hit 'no files match pattern: ./dist/*.whl'. Force-add the dummy wheel as a committed test input. 2. TestInvariantConfigsCoverage requires every config.ResourcesTypes key to be covered. clusters.libraries had no coverage: the scanner only understood .permissions/.grants sub-resources. Teach it .libraries too, add a pypi-only cluster_libraries invariant config, and wire it into INPUT_CONFIG. Cluster libraries are direct-only, so exclude the config from the terraform-seeded migrate subtest like the other direct-only resources. Co-authored-by: Isaac --- .../configs/cluster_libraries.yml.tmpl | 14 ++++++++++ .../invariant/continue_293/out.test.toml | 1 + .../invariant/delete_idempotent/out.test.toml | 1 + .../destroy_idempotent/out.test.toml | 1 + acceptance/bundle/invariant/migrate/test.toml | 2 ++ .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 1 + .../dist/my_package-0.0.1-py3-none-any.whl | 1 + acceptance/invariant_test.go | 28 +++++++++++++------ 9 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl create mode 100644 acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl b/acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl new file mode 100644 index 00000000000..507730ec287 --- /dev/null +++ b/acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + clusters: + foo: + cluster_name: test-cluster-$UNIQUE_NAME + spark_version: 13.3.x-scala2.12 + node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID + num_workers: 1 + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index f5883975fcd..f041da59ad0 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -7,6 +7,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 8ed109d4820..2e4027b3b43 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -7,6 +7,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 8ed109d4820..2e4027b3b43 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -7,6 +7,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index aa24bf58eed..c9a21f879a4 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -16,6 +16,8 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] # Cluster policies are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"] +# Cluster libraries are direct-only; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 8ed109d4820..2e4027b3b43 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -7,6 +7,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index ac4584e034d..968afe44f3e 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -25,6 +25,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", diff --git a/acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl b/acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl new file mode 100644 index 00000000000..99c37c880a6 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl @@ -0,0 +1 @@ +dummy wheel contents \ No newline at end of file diff --git a/acceptance/invariant_test.go b/acceptance/invariant_test.go index 1c8204055ed..3450728ea6f 100644 --- a/acceptance/invariant_test.go +++ b/acceptance/invariant_test.go @@ -19,8 +19,9 @@ const invariantConfigsDir = "bundle/invariant/configs" // LackingInvariantTest lists keys from config.ResourcesTypes that knowingly lack // a covering config in invariantConfigsDir. Keys match the ResourcesTypes // form: "" for the resource itself, ".permissions" / ".grants" -// for permissions/grants coverage. Add a config and remove the entry to close a gap; -// the test fails if an entry here is actually covered, so the list only shrinks. +// / ".libraries" for sub-resource coverage. Add a config and remove the entry +// to close a gap; the test fails if an entry here is actually covered, so the list +// only shrinks. var LackingInvariantTest = map[string]bool{ "quality_monitors": true, } @@ -30,10 +31,11 @@ var LackingInvariantTest = map[string]bool{ // types supporting permissions or grants have at least one config exercising them. // // config.ResourcesTypes is the source of truth: it maps each resource group -// (e.g. "jobs") to its Go type and, where the resource struct has a Permissions -// or Grants field, adds derived keys ".permissions" and ".grants". +// (e.g. "jobs") to its Go type and adds derived keys ".permissions", +// ".grants", and ".libraries" where the resource has the +// corresponding sub-resource. func TestInvariantConfigsCoverage(t *testing.T) { - present, withPermissions, withGrants := scanInvariantConfigs(t) + present, withPermissions, withGrants, withLibraries := scanInvariantConfigs(t) keys := make([]string, 0, len(config.ResourcesTypes)) for key := range config.ResourcesTypes { @@ -53,6 +55,10 @@ func TestInvariantConfigsCoverage(t *testing.T) { group := strings.TrimSuffix(key, ".grants") covered = withGrants[group] hint = "attaches grants to a " + group + " resource" + case strings.HasSuffix(key, ".libraries"): + group := strings.TrimSuffix(key, ".libraries") + covered = withLibraries[group] + hint = "attaches libraries to a " + group + " resource" default: covered = present[key] hint = "defines a " + key + " resource" @@ -69,12 +75,13 @@ func TestInvariantConfigsCoverage(t *testing.T) { } // scanInvariantConfigs parses every config in the invariant configs directory and -// returns the set of resource groups present, the groups with at least one resource -// carrying permissions, and the groups with at least one resource carrying grants. -func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants map[string]bool) { +// returns the set of resource groups present, and the groups with at least one +// resource carrying permissions, grants, or libraries. +func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants, withLibraries map[string]bool) { present = map[string]bool{} withPermissions = map[string]bool{} withGrants = map[string]bool{} + withLibraries = map[string]bool{} entries, err := os.ReadDir(invariantConfigsDir) require.NoError(t, err) @@ -114,9 +121,12 @@ func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants ma if cfg.Get("grants").Kind() != dyn.KindInvalid { withGrants[groupName] = true } + if cfg.Get("libraries").Kind() != dyn.KindInvalid { + withLibraries[groupName] = true + } } } } - return present, withPermissions, withGrants + return present, withPermissions, withGrants, withLibraries }