diff --git a/.nextchanges/bundles/cluster-libraries.md b/.nextchanges/bundles/cluster-libraries.md new file mode 100644 index 0000000000..4c9b1a5200 --- /dev/null +++ b/.nextchanges/bundles/cluster-libraries.md @@ -0,0 +1 @@ +Add support for a `libraries` list on the `clusters` resource type in Declarative Automation Bundles. Libraries (whl, jar, pypi, maven, cran, egg, requirements) are installed on the all-purpose cluster via the Libraries API; local wheels/jars are built and uploaded automatically. Cluster libraries are only supported in direct deployment mode. diff --git a/acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl b/acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl new file mode 100644 index 0000000000..507730ec28 --- /dev/null +++ b/acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + clusters: + foo: + cluster_name: test-cluster-$UNIQUE_NAME + spark_version: 13.3.x-scala2.12 + node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID + num_workers: 1 + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index f5883975fc..f041da59ad 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -7,6 +7,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 8ed109d482..2e4027b3b4 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -7,6 +7,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 8ed109d482..2e4027b3b4 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -7,6 +7,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index aa24bf58ee..c9a21f879a 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -16,6 +16,8 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] # Cluster policies are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"] +# Cluster libraries are direct-only; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 8ed109d482..2e4027b3b4 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -7,6 +7,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index ac4584e034..968afe44f3 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -25,6 +25,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index de252dc416..5e00e1b1f7 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -460,6 +460,23 @@ resources.clusters.*.jdbc_port int REMOTE resources.clusters.*.kind compute.Kind ALL resources.clusters.*.last_restarted_time int64 REMOTE resources.clusters.*.last_state_loss_time int64 REMOTE +resources.clusters.*.libraries []compute.Library INPUT +resources.clusters.*.libraries[*] compute.Library INPUT +resources.clusters.*.libraries[*].cran *compute.RCranLibrary INPUT +resources.clusters.*.libraries[*].cran.package string INPUT +resources.clusters.*.libraries[*].cran.repo string INPUT +resources.clusters.*.libraries[*].egg string INPUT +resources.clusters.*.libraries[*].jar string INPUT +resources.clusters.*.libraries[*].maven *compute.MavenLibrary INPUT +resources.clusters.*.libraries[*].maven.coordinates string INPUT +resources.clusters.*.libraries[*].maven.exclusions []string INPUT +resources.clusters.*.libraries[*].maven.exclusions[*] string INPUT +resources.clusters.*.libraries[*].maven.repo string INPUT +resources.clusters.*.libraries[*].pypi *compute.PythonPyPiLibrary INPUT +resources.clusters.*.libraries[*].pypi.package string INPUT +resources.clusters.*.libraries[*].pypi.repo string INPUT +resources.clusters.*.libraries[*].requirements string INPUT +resources.clusters.*.libraries[*].whl string INPUT resources.clusters.*.lifecycle *dresources.StateLifecycle REMOTE STATE resources.clusters.*.lifecycle *resources.LifecycleWithStarted INPUT resources.clusters.*.lifecycle resources.Lifecycle INPUT @@ -610,6 +627,23 @@ resources.clusters.*.workload_type *compute.WorkloadType ALL resources.clusters.*.workload_type.clients compute.ClientsTypes ALL resources.clusters.*.workload_type.clients.jobs bool ALL resources.clusters.*.workload_type.clients.notebooks bool ALL +resources.clusters.*.libraries.cluster_id string ALL +resources.clusters.*.libraries[*] compute.Library ALL +resources.clusters.*.libraries[*].cran *compute.RCranLibrary ALL +resources.clusters.*.libraries[*].cran.package string ALL +resources.clusters.*.libraries[*].cran.repo string ALL +resources.clusters.*.libraries[*].egg string ALL +resources.clusters.*.libraries[*].jar string ALL +resources.clusters.*.libraries[*].maven *compute.MavenLibrary ALL +resources.clusters.*.libraries[*].maven.coordinates string ALL +resources.clusters.*.libraries[*].maven.exclusions []string ALL +resources.clusters.*.libraries[*].maven.exclusions[*] string ALL +resources.clusters.*.libraries[*].maven.repo string ALL +resources.clusters.*.libraries[*].pypi *compute.PythonPyPiLibrary ALL +resources.clusters.*.libraries[*].pypi.package string ALL +resources.clusters.*.libraries[*].pypi.repo string ALL +resources.clusters.*.libraries[*].requirements string ALL +resources.clusters.*.libraries[*].whl string ALL resources.clusters.*.permissions.object_id string ALL resources.clusters.*.permissions[*] dresources.StatePermission ALL resources.clusters.*.permissions[*].group_name string ALL diff --git a/acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl new file mode 100644 index 0000000000..b3bf246374 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: cluster-libraries-drift-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + clusters: + mycluster: + cluster_name: mycluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID + num_workers: 1 + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-drift/out.test.toml b/acceptance/bundle/resources/clusters/libraries-drift/out.test.toml new file mode 100644 index 0000000000..c502b28221 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries-drift/output.txt b/acceptance/bundle/resources/clusters/libraries-drift/output.txt new file mode 100644 index 0000000000..4a180dfbf0 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/output.txt @@ -0,0 +1,12 @@ + +=== Plan is a no-op immediately after deploy (no library drift) +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-drift/script b/acceptance/bundle/resources/clusters/libraries-drift/script new file mode 100644 index 0000000000..b1fef7ee70 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/script @@ -0,0 +1,20 @@ +# A pypi library is used because its nested {package, repo} shape is the most +# likely to drift (the status API echoing a repo we did not set); a workspace +# wheel is covered by the local clusters/libraries test instead, since the +# shared cloud test cluster rejects libraries from /Workspace paths. +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# Cluster provisioning and library-install output is noisy and differs between +# the fake and cloud, so route it to LOG and assert only the deterministic +# drift signal below. +$CLI bundle deploy &> LOG.deploy +cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" > /dev/null + +title "Plan is a no-op immediately after deploy (no library drift)" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/clusters/libraries-drift/test.toml b/acceptance/bundle/resources/clusters/libraries-drift/test.toml new file mode 100644 index 0000000000..63a8c8a332 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/test.toml @@ -0,0 +1,6 @@ +Cloud = true +RecordRequests = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [".databricks", "databricks.yml"] diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml b/acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml new file mode 100644 index 0000000000..e2645b5cc5 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: cluster-libraries-terraform-error + +resources: + clusters: + mycluster: + cluster_name: mycluster + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml b/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml new file mode 100644 index 0000000000..d2059b4b5d --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt b/acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt new file mode 100644 index 0000000000..aa0a6711e7 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt @@ -0,0 +1,16 @@ + +=== bundle plan fails with cluster libraries on terraform engine +>>> errcode [CLI] bundle plan +Error: cluster libraries are only supported in direct deployment mode + in databricks.yml:12:9 + + +Exit code: 1 + +=== bundle deploy fails with cluster libraries on terraform engine +>>> errcode [CLI] bundle deploy +Error: cluster libraries are only supported in direct deployment mode + in databricks.yml:12:9 + + +Exit code: 1 diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/script b/acceptance/bundle/resources/clusters/libraries-terraform-error/script new file mode 100644 index 0000000000..93db6e0630 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/script @@ -0,0 +1,5 @@ +title "bundle plan fails with cluster libraries on terraform engine" +trace errcode $CLI bundle plan + +title "bundle deploy fails with cluster libraries on terraform engine" +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml b/acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml new file mode 100644 index 0000000000..e4a0f1c630 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml @@ -0,0 +1,4 @@ +Cloud = false +RecordRequests = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/clusters/libraries/databricks.yml b/acceptance/bundle/resources/clusters/libraries/databricks.yml new file mode 100644 index 0000000000..922f719fda --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: cluster-libraries + +resources: + clusters: + mycluster: + cluster_name: mycluster + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + libraries: + - pypi: + package: requests + - whl: ./dist/*.whl diff --git a/acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl b/acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl new file mode 100644 index 0000000000..99c37c880a --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl @@ -0,0 +1 @@ +dummy wheel contents \ No newline at end of file diff --git a/acceptance/bundle/resources/clusters/libraries/out.test.toml b/acceptance/bundle/resources/clusters/libraries/out.test.toml new file mode 100644 index 0000000000..0938e67898 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries/output.txt b/acceptance/bundle/resources/clusters/libraries/output.txt new file mode 100644 index 0000000000..74c60339a0 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/output.txt @@ -0,0 +1,58 @@ + +=== Deploy a cluster with a pypi and a local wheel library +>>> [CLI] bundle deploy +Uploading dist/my_package-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files... +Created clusters.mycluster +Created clusters.mycluster.libraries +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Libraries installed via the Libraries API (wheel rewritten to its uploaded path) +>>> print_requests.py //libraries/install +{ + "method": "POST", + "path": "/api/2.0/libraries/install", + "body": { + "cluster_id": "[UUID]", + "libraries": [ + { + "pypi": { + "package": "requests" + } + }, + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/artifacts/.internal/my_package-0.0.1-py3-none-any.whl" + } + ] + } +} + +=== Removing the wheel and redeploying uninstalls it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files... +Updated clusters.mycluster.libraries +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //libraries/uninstall +{ + "method": "POST", + "path": "/api/2.0/libraries/uninstall", + "body": { + "cluster_id": "[UUID]", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/artifacts/.internal/my_package-0.0.1-py3-none-any.whl" + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries/script b/acceptance/bundle/resources/clusters/libraries/script new file mode 100644 index 0000000000..33af4f7c5d --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/script @@ -0,0 +1,16 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy a cluster with a pypi and a local wheel library" +trace $CLI bundle deploy + +title "Libraries installed via the Libraries API (wheel rewritten to its uploaded path)" +trace print_requests.py //libraries/install + +title "Removing the wheel and redeploying uninstalls it" +update_file.py databricks.yml " - whl: ./dist/*.whl" "" +trace $CLI bundle deploy +trace print_requests.py //libraries/uninstall diff --git a/acceptance/bundle/resources/clusters/libraries/test.toml b/acceptance/bundle/resources/clusters/libraries/test.toml new file mode 100644 index 0000000000..93cb054324 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/test.toml @@ -0,0 +1,10 @@ +Cloud = false +RecordRequests = true + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [".databricks"] + +[[Repls]] +Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" +New = "[CLUSTER-ID]" diff --git a/acceptance/invariant_test.go b/acceptance/invariant_test.go index 1c8204055e..3450728ea6 100644 --- a/acceptance/invariant_test.go +++ b/acceptance/invariant_test.go @@ -19,8 +19,9 @@ const invariantConfigsDir = "bundle/invariant/configs" // LackingInvariantTest lists keys from config.ResourcesTypes that knowingly lack // a covering config in invariantConfigsDir. Keys match the ResourcesTypes // form: "" for the resource itself, ".permissions" / ".grants" -// for permissions/grants coverage. Add a config and remove the entry to close a gap; -// the test fails if an entry here is actually covered, so the list only shrinks. +// / ".libraries" for sub-resource coverage. Add a config and remove the entry +// to close a gap; the test fails if an entry here is actually covered, so the list +// only shrinks. var LackingInvariantTest = map[string]bool{ "quality_monitors": true, } @@ -30,10 +31,11 @@ var LackingInvariantTest = map[string]bool{ // types supporting permissions or grants have at least one config exercising them. // // config.ResourcesTypes is the source of truth: it maps each resource group -// (e.g. "jobs") to its Go type and, where the resource struct has a Permissions -// or Grants field, adds derived keys ".permissions" and ".grants". +// (e.g. "jobs") to its Go type and adds derived keys ".permissions", +// ".grants", and ".libraries" where the resource has the +// corresponding sub-resource. func TestInvariantConfigsCoverage(t *testing.T) { - present, withPermissions, withGrants := scanInvariantConfigs(t) + present, withPermissions, withGrants, withLibraries := scanInvariantConfigs(t) keys := make([]string, 0, len(config.ResourcesTypes)) for key := range config.ResourcesTypes { @@ -53,6 +55,10 @@ func TestInvariantConfigsCoverage(t *testing.T) { group := strings.TrimSuffix(key, ".grants") covered = withGrants[group] hint = "attaches grants to a " + group + " resource" + case strings.HasSuffix(key, ".libraries"): + group := strings.TrimSuffix(key, ".libraries") + covered = withLibraries[group] + hint = "attaches libraries to a " + group + " resource" default: covered = present[key] hint = "defines a " + key + " resource" @@ -69,12 +75,13 @@ func TestInvariantConfigsCoverage(t *testing.T) { } // scanInvariantConfigs parses every config in the invariant configs directory and -// returns the set of resource groups present, the groups with at least one resource -// carrying permissions, and the groups with at least one resource carrying grants. -func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants map[string]bool) { +// returns the set of resource groups present, and the groups with at least one +// resource carrying permissions, grants, or libraries. +func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants, withLibraries map[string]bool) { present = map[string]bool{} withPermissions = map[string]bool{} withGrants = map[string]bool{} + withLibraries = map[string]bool{} entries, err := os.ReadDir(invariantConfigsDir) require.NoError(t, err) @@ -114,9 +121,12 @@ func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants ma if cfg.Get("grants").Kind() != dyn.KindInvalid { withGrants[groupName] = true } + if cfg.Get("libraries").Kind() != dyn.KindInvalid { + withLibraries[groupName] = true + } } } } - return present, withPermissions, withGrants + return present, withPermissions, withGrants, withLibraries } diff --git a/bundle/config/mutator/validate_cluster_libraries.go b/bundle/config/mutator/validate_cluster_libraries.go new file mode 100644 index 0000000000..f13cd3be37 --- /dev/null +++ b/bundle/config/mutator/validate_cluster_libraries.go @@ -0,0 +1,43 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/diag" +) + +type validateClusterLibraries struct { + engine engine.EngineType +} + +// ValidateClusterLibraries returns a mutator that errors when cluster libraries are used with +// the terraform deployment engine. Cluster libraries are only supported in direct deployment mode. +func ValidateClusterLibraries(e engine.EngineType) bundle.Mutator { + return &validateClusterLibraries{engine: e} +} + +func (m *validateClusterLibraries) Name() string { + return "ValidateClusterLibraries" +} + +func (m *validateClusterLibraries) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + if m.engine.IsDirect() { + return nil + } + + var diags diag.Diagnostics + for key, cluster := range b.Config.Resources.Clusters { + if cluster == nil || len(cluster.Libraries) == 0 { + continue + } + path := "resources.clusters." + key + ".libraries" + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "cluster libraries are only supported in direct deployment mode", + Locations: b.Config.GetLocations(path), + }) + } + return diags +} diff --git a/bundle/config/resources/clusters.go b/bundle/config/resources/clusters.go index 235ea6eee1..cb69fc6b75 100644 --- a/bundle/config/resources/clusters.go +++ b/bundle/config/resources/clusters.go @@ -18,6 +18,9 @@ type Cluster struct { // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. Lifecycle *LifecycleWithStarted `json:"lifecycle,omitempty"` + // Libraries are installed via the Libraries API, not the cluster spec. + Libraries []compute.Library `json:"libraries,omitempty"` + Permissions []ClusterPermission `json:"permissions,omitempty"` } diff --git a/bundle/config/resources_types.go b/bundle/config/resources_types.go index dcc91545f1..20dbee0222 100644 --- a/bundle/config/resources_types.go +++ b/bundle/config/resources_types.go @@ -43,6 +43,14 @@ var ResourcesTypes = func() map[string]reflect.Type { if resourceField.Name == "Grants" { grantsKey := name + ".grants" res[grantsKey] = resourceField.Type + continue + } + // libraries is a child resource only for clusters. Pipelines and + // cluster_policies have a native Libraries field that is a plain + // field, not a child resource. + if resourceField.Name == "Libraries" && name == "clusters" { + librariesKey := name + ".libraries" + res[librariesKey] = resourceField.Type } } } diff --git a/bundle/config/root.go b/bundle/config/root.go index e13bf78bf1..c05851c3d8 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -620,8 +620,15 @@ func GetNodeAndType(path dyn.Path) (dyn.Path, string) { } if len(path) >= 4 { - if path[3].Key() == "permissions" || path[3].Key() == "grants" { - return path[:4], path[1].Key() + "." + path[3].Key() + sub := path[3].Key() + if sub == "permissions" || sub == "grants" { + return path[:4], path[1].Key() + "." + sub + } + // libraries is a sub-resource only for clusters. Other resource types + // (e.g. pipelines) have a native libraries field that is a plain field, + // not a child resource. + if sub == "libraries" && path[1].Key() == "clusters" { + return path[:4], path[1].Key() + "." + sub } } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 5b8829e3f5..c9e08fe798 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -713,7 +713,7 @@ func splitResourcePath(path *structpath.PathNode) (string, *structpath.PathNode) // Check if the 4th component is "permissions" or "grants" (sub-resource) if path.Len() > 4 { first := path.SkipPrefix(3).Prefix(1) - if key, ok := first.StringKey(); ok && (key == "permissions" || key == "grants") { + if key, ok := first.StringKey(); ok && (key == "permissions" || key == "grants" || key == "libraries") { return path.Prefix(4).String(), path.SkipPrefix(4) } } @@ -930,6 +930,8 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + // libraries is a sub-resource only for clusters; other resource types have a native libraries field. + dyn.NewPattern(dyn.Key("resources"), dyn.Key("clusters"), dyn.AnyKey(), dyn.Key("libraries")), } // Walk? diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 391fb0684d..4ffeed88ab 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -61,6 +61,9 @@ var SupportedResources = map[string]any{ "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), "instance_pools.permissions": (*ResourcePermissions)(nil), + // Libraries + "clusters.libraries": (*ResourceLibraries)(nil), + // Grants "catalogs.grants": (*ResourceGrants)(nil), "schemas.grants": (*ResourceGrants)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 0c0d3d05ad..7ec1e9b3b7 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -452,6 +452,24 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "clusters.libraries": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + wait, err := client.Clusters.Create(ctx, compute.CreateCluster{ + ClusterName: "libraries-cluster", + SparkVersion: "13.3.x-scala2.12", + NodeTypeId: "m5.large", + NumWorkers: 1, + }) + if err != nil { + return nil, err + } + return &LibrariesState{ + ClusterId: wait.ClusterId, + EmbeddedSlice: []compute.Library{ + {Whl: "/Workspace/Users/test/lib.whl"}, + }, + }, nil + }, + "cluster_policies.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { return &PermissionsState{ ObjectID: "/cluster-policies/cluster-policy-permissions", @@ -1127,7 +1145,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NoError(t, err) } - deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") + deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") || strings.HasSuffix(group, "libraries") // Apps DoDelete is fire-and-forget: the API returns success while the app // sits in DELETING state for up to ~20 minutes before the record is removed. // A GET on the DELETING app returns the app, not 404 -- the testserver diff --git a/bundle/direct/dresources/cluster_libraries.go b/bundle/direct/dresources/cluster_libraries.go new file mode 100644 index 0000000000..a3458da761 --- /dev/null +++ b/bundle/direct/dresources/cluster_libraries.go @@ -0,0 +1,255 @@ +package dresources + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/retries" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +// Corresponds to the databricks_library terraform resource: +// https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/library + +// librariesWaitTimeout bounds how long we poll for libraries to finish installing. +const librariesWaitTimeout = 15 * time.Minute + +// LibrariesState is the state for a cluster's libraries sub-resource. Libraries are installed +// via the Libraries API against the parent cluster identified by ClusterId, not through the +// cluster spec. +type LibrariesState struct { + ClusterId string `json:"cluster_id"` + // By convention EmbeddedSlice fields have the __embed__ json tag, see permissions.go. + EmbeddedSlice []compute.Library `json:"__embed__,omitempty"` +} + +type ResourceLibraries struct { + client *databricks.WorkspaceClient +} + +func (*ResourceLibraries) New(client *databricks.WorkspaceClient) *ResourceLibraries { + return &ResourceLibraries{client: client} +} + +func (r *ResourceLibraries) PrepareInputConfig(inputConfig *[]compute.Library, resourceKey string) (*structvar.StructVar, error) { + baseNode, ok := strings.CutSuffix(resourceKey, ".libraries") + if !ok { + return nil, fmt.Errorf("internal error: node %q does not end with .libraries", resourceKey) + } + + return &structvar.StructVar{ + Value: &LibrariesState{ + ClusterId: "", // Always a reference, defined in Refs below. + EmbeddedSlice: *inputConfig, + }, + Refs: map[string]string{ + "cluster_id": "${" + baseNode + ".id}", + }, + }, nil +} + +func (*ResourceLibraries) PrepareState(state *LibrariesState) *LibrariesState { + return state +} + +// IsEmptyState reports an empty libraries list as no resource at all: nothing to install, and no +// state entry is persisted for it. +func (*ResourceLibraries) IsEmptyState(state *LibrariesState) bool { + return len(state.EmbeddedSlice) == 0 +} + +// libraryKey identifies a library by its type-specific field so slices compare by identity +// rather than by index (see KeyedSlices). +func libraryKey(l compute.Library) (string, string) { + switch { + case l.Whl != "": + return "whl", l.Whl + case l.Jar != "": + return "jar", l.Jar + case l.Egg != "": + return "egg", l.Egg + case l.Requirements != "": + return "requirements", l.Requirements + case l.Pypi != nil: + return "pypi", l.Pypi.Package + case l.Maven != nil: + return "maven", l.Maven.Coordinates + case l.Cran != nil: + return "cran", l.Cran.Package + } + return "", "" +} + +func (*ResourceLibraries) KeyedSlices() map[string]any { + // Empty key because EmbeddedSlice appears at the root path of LibrariesState. + return map[string]any{ + "": libraryKey, + } +} + +func (r *ResourceLibraries) DoRead(ctx context.Context, id string) (*LibrariesState, error) { + statuses, err := r.client.Libraries.ClusterStatusByClusterId(ctx, id) + if err != nil { + return nil, err + } + + state := &LibrariesState{ClusterId: id, EmbeddedSlice: nil} + for _, s := range statuses.LibraryStatuses { + // Libraries set for all clusters via the UI are not managed by the bundle + // (following the permissions convention of ignoring inherited entries). + if s.Library == nil || s.IsLibraryForAllClusters { + continue + } + // A library pending uninstall on restart is on its way out; don't report it as present. + if s.Status == compute.LibraryInstallStatusUninstallOnRestart { + continue + } + state.EmbeddedSlice = append(state.EmbeddedSlice, *s.Library) + } + return state, nil +} + +// DoCreate installs the libraries on the cluster. +// https://docs.databricks.com/api/workspace/libraries/install +func (r *ResourceLibraries) DoCreate(ctx context.Context, state *LibrariesState) (string, *LibrariesState, error) { + err := r.client.Libraries.Install(ctx, compute.InstallLibraries{ + ClusterId: state.ClusterId, + Libraries: state.EmbeddedSlice, + }) + if err != nil { + // Install is idempotent (installing an already-installed library is a no-op), + // so retrying on transient errors is safe. + return "", nil, retrySafe(err) + } + return state.ClusterId, nil, nil +} + +// DoUpdate uninstalls libraries removed from config and installs the desired set. This is two API +// calls because the Libraries API exposes install and uninstall as separate endpoints, unlike the +// single-call model most resources follow. +func (r *ResourceLibraries) DoUpdate(ctx context.Context, id string, state *LibrariesState, entry *PlanEntry) (*LibrariesState, error) { + removed := removedLibraries(state.EmbeddedSlice, entry) + if len(removed) > 0 { + err := r.client.Libraries.Uninstall(ctx, compute.UninstallLibraries{ + ClusterId: id, + Libraries: removed, + }) + if err != nil { + return nil, err + } + } + + if len(state.EmbeddedSlice) > 0 { + err := r.client.Libraries.Install(ctx, compute.InstallLibraries{ + ClusterId: id, + Libraries: state.EmbeddedSlice, + }) + if err != nil { + return nil, err + } + } + return nil, nil +} + +// DoDelete is a no-op: removing individual libraries is handled by DoUpdate's uninstall diff, and +// DoDelete only fires when the parent cluster is deleted, at which point uninstalling is moot. +func (r *ResourceLibraries) DoDelete(ctx context.Context, id string, _ *LibrariesState) error { + return nil +} + +// removedLibraries returns libraries present in the remote state but absent from the desired set. +func removedLibraries(desired []compute.Library, entry *PlanEntry) []compute.Library { + if entry == nil { + return nil + } + remote, ok := entry.RemoteState.(*LibrariesState) + if !ok || remote == nil { + return nil + } + + desiredKeys := make(map[string]struct{}, len(desired)) + for _, l := range desired { + desiredKeys[libraryMapKey(l)] = struct{}{} + } + + var result []compute.Library + for _, l := range remote.EmbeddedSlice { + if _, ok := desiredKeys[libraryMapKey(l)]; !ok { + result = append(result, l) + } + } + return result +} + +// libraryMapKey flattens libraryKey into a single string for map lookups. +func libraryMapKey(l compute.Library) string { + f, v := libraryKey(l) + return f + "=" + v +} + +func (r *ResourceLibraries) WaitAfterCreate(ctx context.Context, id string, state *LibrariesState) (*LibrariesState, error) { + return nil, r.waitForInstall(ctx, id, state.EmbeddedSlice) +} + +func (r *ResourceLibraries) WaitAfterUpdate(ctx context.Context, id string, state *LibrariesState) (*LibrariesState, error) { + return nil, r.waitForInstall(ctx, id, state.EmbeddedSlice) +} + +// waitForInstall polls until every desired library reaches a terminal installed state. It returns +// early without waiting when the cluster is not running: installs only progress on a running +// cluster and are queued until it next starts. +func (r *ResourceLibraries) waitForInstall(ctx context.Context, id string, desired []compute.Library) error { + if len(desired) == 0 { + return nil + } + + details, err := r.client.Clusters.GetByClusterId(ctx, id) + if err != nil { + return err + } + if details.State != compute.StateRunning { + log.Debugf(ctx, "cluster %s is not running (%s); skipping wait for library installation", id, details.State) + return nil + } + + desiredKeys := make(map[string]struct{}, len(desired)) + for _, l := range desired { + desiredKeys[libraryMapKey(l)] = struct{}{} + } + + _, err = retries.Poll(ctx, librariesWaitTimeout, func() (*struct{}, *retries.Err) { + statuses, err := r.client.Libraries.ClusterStatusByClusterId(ctx, id) + if err != nil { + return nil, retries.Halt(err) + } + + pending := len(desiredKeys) + for _, s := range statuses.LibraryStatuses { + if s.Library == nil { + continue + } + if _, ok := desiredKeys[libraryMapKey(*s.Library)]; !ok { + continue + } + switch s.Status { + case compute.LibraryInstallStatusFailed: + return nil, retries.Halt(fmt.Errorf("library %s failed to install: %s", libraryMapKey(*s.Library), strings.Join(s.Messages, "; "))) + case compute.LibraryInstallStatusInstalled, compute.LibraryInstallStatusSkipped, compute.LibraryInstallStatusRestored: + pending-- + case compute.LibraryInstallStatusPending, compute.LibraryInstallStatusResolving, compute.LibraryInstallStatusInstalling, compute.LibraryInstallStatusUninstallOnRestart: + // Still in progress (or being removed); keep polling. + } + } + + if pending > 0 { + return nil, retries.Continues(fmt.Sprintf("waiting for %d librar(ies) to install on cluster %s", pending, id)) + } + return &struct{}{}, nil + }) + return err +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 2d5516d59c..e471d6b2b6 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -66,6 +66,7 @@ var knownMissingInRemoteType = map[string][]string{ // These are bundle-specific fields that exist in InputType but not in StateType. var commonMissingInStateType = []string{ "grants", + "libraries", "lifecycle", "permissions", } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index e47f89c505..e01d21460d 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -605,6 +605,9 @@ resources: notebook_path: "./src/my_notebook.py" ``` "$fields": + "libraries": + "description": |- + A list of libraries to install on the cluster. Installed via the Libraries API after the cluster is created. Only supported in direct deployment mode. "lifecycle": "description": |- Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 617f5a65ec..d093785109 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -56,6 +56,9 @@ var RequiredFields = map[string][]string{ "resources.clusters.*.init_scripts[*].s3": {"destination"}, "resources.clusters.*.init_scripts[*].volumes": {"destination"}, "resources.clusters.*.init_scripts[*].workspace": {"destination"}, + "resources.clusters.*.libraries[*].cran": {"package"}, + "resources.clusters.*.libraries[*].maven": {"coordinates"}, + "resources.clusters.*.libraries[*].pypi": {"package"}, "resources.clusters.*.permissions[*]": {"level"}, "resources.clusters.*.workload_type": {"clients"}, diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index 720142fe6d..ab1da3df68 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -198,6 +198,13 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) +var clusterLibrariesPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("clusters"), + dyn.AnyKey(), + dyn.Key("libraries"), +) + func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { expanders := []expandPattern{ { @@ -216,6 +223,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { pattern: pipelineEnvDepsPattern, fn: expandEnvironmentDeps, }, + { + pattern: clusterLibrariesPattern, + fn: expandLibraries, + }, } var diags diag.Diagnostics diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index 02a1172f36..e40de7460d 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -68,6 +68,8 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), // The AI Runtime task's code_source_path is a local archive (typically an diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 49776fbd8c..8fb140d7aa 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -17,6 +17,8 @@ var patterns = []dyn.Pattern{ taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), } diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 56250d713a..3c5ff184a9 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -79,6 +79,22 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag } } + // Update resources.clusters.*.libraries[*].whl + for clusterName, clusterRef := range b.Config.Resources.Clusters { + if clusterRef == nil { + continue + } + for libInd, lib := range clusterRef.Libraries { + repl := replacements[lib.Whl] + if repl != "" { + log.Debugf(ctx, "Updating resources.clusters.%s.libraries[%d].whl from %s to %s", clusterName, libInd, lib.Whl, repl) + clusterRef.Libraries[libInd].Whl = repl + } else { + log.Debugf(ctx, "Not updating resources.clusters.%s.libraries[%d].whl from %s. Available replacements: %v", clusterName, libInd, lib.Whl, slices.Sorted(maps.Keys(replacements))) + } + } + } + return nil } diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 3db0864c2c..208582a2d9 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -28,6 +28,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), mutator.ValidateLifecycleStarted(engine), + mutator.ValidateClusterLibraries(engine), mutator.ValidateCascadeOnDestroy(engine), mutator.ValidateJobRunTriggers(), statemgmt.CheckRunningResource(engine), diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 425956ab06..f32d43532f 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -488,6 +488,10 @@ "description": "The kind of compute described by this compute specification.\n\nDepending on `kind`, different validations and default values will be applied.\n\nClusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas clusters with no specified `kind` do not.\n* [is_single_node](/api/workspace/clusters/create#is_single_node)\n* [use_ml_runtime](/api/workspace/clusters/create#use_ml_runtime)\n\nBy using the [simple form](https://docs.databricks.com/compute/simple-form.html), your clusters are automatically using `kind = CLASSIC_PREVIEW`.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Kind" }, + "libraries": { + "description": "A list of libraries to install on the cluster. Installed via the Libraries API after the cluster is created. Only supported in direct deployment mode.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.Library" + }, "lifecycle": { "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted" diff --git a/bundle/terraform_dabs_map/generated.go b/bundle/terraform_dabs_map/generated.go index d0a783f35a..f23c0bfa9e 100644 --- a/bundle/terraform_dabs_map/generated.go +++ b/bundle/terraform_dabs_map/generated.go @@ -6,7 +6,8 @@ package terraform_dabs_map // alerts / databricks_alert_v2: 3 tf-only // apps / databricks_app: 6 dabs-only // apps / databricks_app: 1 tf-only -// clusters / databricks_cluster: 26 tf-only +// clusters / databricks_cluster: 1 renames +// clusters / databricks_cluster: 11 tf-only // dashboards / databricks_dashboard: 2 tf-only // database_instances / databricks_database_instance: 1 tf-only // experiments / databricks_mlflow_experiment: 1 tf-only @@ -34,6 +35,9 @@ package terraform_dabs_map // TerraformToDABsFieldMap maps DABs group name → nested TF segments → DABs segment name. // Navigate using TF field name segments; DABs is the corresponding DABs name when it differs. var TerraformToDABsFieldMap = map[string]RenameTree{ + "clusters": { + "library": {NewName: "libraries"}, + }, "jobs": { "environment": {NewName: "environments"}, "git_source": {Children: RenameTree{ @@ -168,27 +172,8 @@ var TerraformOnlyFields = map[string]FieldSet{ }, "idempotency_token": {}, "is_pinned": {}, - "library": { - "cran": { - "package": {}, // databricks_cluster.*.library.cran.package - "repo": {}, // databricks_cluster.*.library.cran.repo - }, - "egg": {}, // databricks_cluster.*.library.egg - "jar": {}, // databricks_cluster.*.library.jar - "maven": { - "coordinates": {}, // databricks_cluster.*.library.maven.coordinates - "exclusions": {}, // databricks_cluster.*.library.maven.exclusions - "repo": {}, // databricks_cluster.*.library.maven.repo - }, - "pypi": { - "package": {}, // databricks_cluster.*.library.pypi.package - "repo": {}, // databricks_cluster.*.library.pypi.repo - }, - "requirements": {}, // databricks_cluster.*.library.requirements - "whl": {}, // databricks_cluster.*.library.whl - }, - "no_wait": {}, - "url": {}, + "no_wait": {}, + "url": {}, }, "dashboards": { "dashboard_change_detected": {}, @@ -569,6 +554,9 @@ var TerraformOnlyFields = map[string]FieldSet{ // DABsToTerraformRenameMap maps DABs group name → nested DABs segments → TF segment name. // Navigate using DABs field name segments; NewName is the TF name when it differs. var DABsToTerraformRenameMap = map[string]RenameTree{ + "clusters": { + "libraries": {NewName: "library"}, + }, "jobs": { "environments": {NewName: "environment"}, "git_source": {Children: RenameTree{ diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 25178a9e75..02865c1544 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -196,6 +196,7 @@ type FakeWorkspace struct { ModelRegistryModels map[string]ml.Model ModelRegistryModelIDs map[string]string // model name -> numeric ID Clusters map[string]compute.ClusterDetails + ClusterLibraries map[string][]compute.Library // cluster id -> installed libraries InstancePools map[string]compute.GetInstancePool ClusterPolicies map[string]compute.Policy Catalogs map[string]catalog.CatalogInfo @@ -430,7 +431,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { SingleUserName: TestUser.UserName, }, }, - InstancePools: map[string]compute.GetInstancePool{}, + InstancePools: map[string]compute.GetInstancePool{}, + ClusterLibraries: map[string][]compute.Library{}, ClusterPolicies: map[string]compute.Policy{ // Seeded so the stateful list keeps backing the variable-lookup tests // (e.g. acceptance/bundle/variables/env_overrides resolves these by name). diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8c44d97eae..e1ea0baa47 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -927,6 +927,19 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.ClustersPermanentDelete(req) }) + // Cluster libraries: + server.Handle("POST", "/api/2.0/libraries/install", func(req Request) any { + return req.Workspace.LibrariesInstall(req) + }) + + server.Handle("POST", "/api/2.0/libraries/uninstall", func(req Request) any { + return req.Workspace.LibrariesUninstall(req) + }) + + server.Handle("GET", "/api/2.0/libraries/cluster-status", func(req Request) any { + return req.Workspace.LibrariesClusterStatus(req, req.URL.Query().Get("cluster_id")) + }) + // MLflow Experiments: server.Handle("GET", "/api/2.0/mlflow/experiments/get", func(req Request) any { experimentId := req.URL.Query().Get("experiment_id") diff --git a/libs/testserver/libraries.go b/libs/testserver/libraries.go new file mode 100644 index 0000000000..63042ca8e6 --- /dev/null +++ b/libs/testserver/libraries.go @@ -0,0 +1,87 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "net/http" + "reflect" + + "github.com/databricks/databricks-sdk-go/service/compute" +) + +func (s *FakeWorkspace) LibrariesInstall(req Request) any { + var request compute.InstallLibraries + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: http.StatusBadRequest, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + if _, ok := s.Clusters[request.ClusterId]; !ok { + return Response{StatusCode: http.StatusNotFound} + } + + // Install is additive and idempotent: installing an already-present library is a no-op. + installed := s.ClusterLibraries[request.ClusterId] + for _, lib := range request.Libraries { + if !containsLibrary(installed, lib) { + installed = append(installed, lib) + } + } + s.ClusterLibraries[request.ClusterId] = installed + + return Response{} +} + +func (s *FakeWorkspace) LibrariesUninstall(req Request) any { + var request compute.UninstallLibraries + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: http.StatusBadRequest, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + installed := s.ClusterLibraries[request.ClusterId] + remaining := make([]compute.Library, 0, len(installed)) + for _, lib := range installed { + if !containsLibrary(request.Libraries, lib) { + remaining = append(remaining, lib) + } + } + s.ClusterLibraries[request.ClusterId] = remaining + + return Response{} +} + +func (s *FakeWorkspace) LibrariesClusterStatus(req Request, clusterId string) any { + defer s.LockUnlock()() + + if _, ok := s.Clusters[clusterId]; !ok { + return Response{StatusCode: http.StatusNotFound} + } + + installed := s.ClusterLibraries[clusterId] + statuses := make([]compute.LibraryFullStatus, 0, len(installed)) + for i := range installed { + statuses = append(statuses, compute.LibraryFullStatus{ + Library: &installed[i], + Status: compute.LibraryInstallStatusInstalled, + }) + } + + return Response{ + Body: compute.ClusterLibraryStatuses{ + ClusterId: clusterId, + LibraryStatuses: statuses, + }, + } +} + +func containsLibrary(libs []compute.Library, target compute.Library) bool { + for _, l := range libs { + if reflect.DeepEqual(l, target) { + return true + } + } + return false +}