diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/databricks.yml b/acceptance/bundle/resources/jobs/clear_instance_profile/databricks.yml new file mode 100644 index 0000000000..eb9060e93e --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/databricks.yml @@ -0,0 +1,32 @@ +bundle: + name: test-bundle + +resources: + jobs: + with_pool: + name: with_pool + tasks: + - task_key: key + notebook_task: + notebook_path: /Workspace/nb + new_cluster: + spark_version: 13.3.x-scala2.12 + num_workers: 1 + instance_pool_id: pool-id + aws_attributes: + availability: SPOT + instance_profile_arn: arn:aws:iam::123456789012:instance-profile/original + + without_pool: + name: without_pool + tasks: + - task_key: key + notebook_task: + notebook_path: /Workspace/nb + new_cluster: + spark_version: 13.3.x-scala2.12 + num_workers: 1 + node_type_id: m5.large + aws_attributes: + availability: SPOT + instance_profile_arn: arn:aws:iam::123456789012:instance-profile/original diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/out.create.requests.direct.json b/acceptance/bundle/resources/jobs/clear_instance_profile/out.create.requests.direct.json new file mode 100644 index 0000000000..962f509994 --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/out.create.requests.direct.json @@ -0,0 +1,67 @@ +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "with_pool", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "aws_attributes": { + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/original" + }, + "instance_pool_id": "pool-id", + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/Workspace/nb" + }, + "task_key": "key" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "without_pool", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "aws_attributes": { + "availability": "SPOT", + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/original" + }, + "node_type_id": "m5.large", + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/Workspace/nb" + }, + "task_key": "key" + } + ] + } +} diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/out.create.requests.terraform.json b/acceptance/bundle/resources/jobs/clear_instance_profile/out.create.requests.terraform.json new file mode 100644 index 0000000000..2352e7a05a --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/out.create.requests.terraform.json @@ -0,0 +1,69 @@ +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "with_pool", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "aws_attributes": { + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/original" + }, + "instance_pool_id": "pool-id", + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/Workspace/nb", + "source": "WORKSPACE" + }, + "task_key": "key" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "without_pool", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "aws_attributes": { + "availability": "SPOT", + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/original" + }, + "node_type_id": "m5.large", + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/Workspace/nb", + "source": "WORKSPACE" + }, + "task_key": "key" + } + ] + } +} diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/out.remove.direct.txt b/acceptance/bundle/resources/jobs/clear_instance_profile/out.remove.direct.txt new file mode 100644 index 0000000000..8cee48a409 --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/out.remove.direct.txt @@ -0,0 +1,110 @@ + +>>> local_aws_attributes +Warning: There are no files to sync, please check your .gitignore + +{ + "with_pool": {}, + "without_pool": { + "availability": "SPOT" + } +} + +>>> [CLI] bundle plan +update jobs.with_pool +update jobs.without_pool + +Plan: 0 to add, 2 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> job_requests +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [WITH_POOL_ID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "with_pool", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "aws_attributes": {}, + "instance_pool_id": "pool-id", + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/Workspace/nb" + }, + "task_key": "key" + } + ] + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [WITHOUT_POOL_ID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "without_pool", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "aws_attributes": { + "availability": "SPOT" + }, + "node_type_id": "m5.large", + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/Workspace/nb" + }, + "task_key": "key" + } + ] + } + } +} + +>>> remote_aws_attributes +with_pool: +{ + "instance_pool_id": "pool-id", + "node_type_id": null, + "aws_attributes": {} +} +without_pool: +{ + "instance_pool_id": null, + "node_type_id": "m5.large", + "aws_attributes": { + "availability": "SPOT" + } +} diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/out.remove.terraform.txt b/acceptance/bundle/resources/jobs/clear_instance_profile/out.remove.terraform.txt new file mode 100644 index 0000000000..5a3d5d7054 --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/out.remove.terraform.txt @@ -0,0 +1,40 @@ + +>>> local_aws_attributes +Warning: There are no files to sync, please check your .gitignore + +{ + "with_pool": {}, + "without_pool": { + "availability": "SPOT" + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> job_requests + +>>> remote_aws_attributes +with_pool: +{ + "instance_pool_id": "pool-id", + "node_type_id": null, + "aws_attributes": { + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/original" + } +} +without_pool: +{ + "instance_pool_id": null, + "node_type_id": "m5.large", + "aws_attributes": { + "availability": "SPOT", + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/original" + } +} diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/out.replace.direct.txt b/acceptance/bundle/resources/jobs/clear_instance_profile/out.replace.direct.txt new file mode 100644 index 0000000000..2f724b61b7 --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/out.replace.direct.txt @@ -0,0 +1,119 @@ + +>>> local_aws_attributes +Warning: There are no files to sync, please check your .gitignore + +{ + "with_pool": { + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/replacement" + }, + "without_pool": { + "availability": "SPOT", + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/replacement" + } +} + +>>> [CLI] bundle plan +update jobs.with_pool +update jobs.without_pool + +Plan: 0 to add, 2 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> job_requests +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [WITH_POOL_ID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "with_pool", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "aws_attributes": { + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/replacement" + }, + "instance_pool_id": "pool-id", + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/Workspace/nb" + }, + "task_key": "key" + } + ] + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [WITHOUT_POOL_ID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "without_pool", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "aws_attributes": { + "availability": "SPOT", + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/replacement" + }, + "node_type_id": "m5.large", + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/Workspace/nb" + }, + "task_key": "key" + } + ] + } + } +} + +>>> remote_aws_attributes +with_pool: +{ + "instance_pool_id": "pool-id", + "node_type_id": null, + "aws_attributes": { + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/replacement" + } +} +without_pool: +{ + "instance_pool_id": null, + "node_type_id": "m5.large", + "aws_attributes": { + "availability": "SPOT", + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/replacement" + } +} diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/out.replace.terraform.txt b/acceptance/bundle/resources/jobs/clear_instance_profile/out.replace.terraform.txt new file mode 100644 index 0000000000..0beecaf217 --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/out.replace.terraform.txt @@ -0,0 +1,137 @@ + +>>> local_aws_attributes +Warning: There are no files to sync, please check your .gitignore + +{ + "with_pool": { + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/replacement" + }, + "without_pool": { + "availability": "SPOT", + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/replacement" + } +} + +>>> [CLI] bundle plan +update jobs.with_pool +update jobs.without_pool + +Plan: 0 to add, 2 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> job_requests +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [WITH_POOL_ID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "with_pool", + "queue": { + "enabled": true + }, + "run_as": { + "user_name": "[USERNAME]" + }, + "tasks": [ + { + "email_notifications": {}, + "new_cluster": { + "aws_attributes": { + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/replacement" + }, + "data_security_mode": "SINGLE_USER", + "instance_pool_id": "pool-id", + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/Workspace/nb", + "source": "WORKSPACE" + }, + "run_if": "ALL_SUCCESS", + "task_key": "key" + } + ], + "webhook_notifications": {} + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [WITHOUT_POOL_ID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "without_pool", + "queue": { + "enabled": true + }, + "run_as": { + "user_name": "[USERNAME]" + }, + "tasks": [ + { + "email_notifications": {}, + "new_cluster": { + "aws_attributes": { + "availability": "SPOT", + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/replacement" + }, + "data_security_mode": "SINGLE_USER", + "node_type_id": "m5.large", + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/Workspace/nb", + "source": "WORKSPACE" + }, + "run_if": "ALL_SUCCESS", + "task_key": "key" + } + ], + "webhook_notifications": {} + } + } +} + +>>> remote_aws_attributes +with_pool: +{ + "instance_pool_id": "pool-id", + "node_type_id": null, + "aws_attributes": { + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/replacement" + } +} +without_pool: +{ + "instance_pool_id": null, + "node_type_id": "m5.large", + "aws_attributes": { + "availability": "SPOT", + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/replacement" + } +} diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/out.test.toml b/acceptance/bundle/resources/jobs/clear_instance_profile/out.test.toml new file mode 100644 index 0000000000..f784a18325 --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/out.workaround.direct.txt b/acceptance/bundle/resources/jobs/clear_instance_profile/out.workaround.direct.txt new file mode 100644 index 0000000000..6b3a3c85f0 --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/out.workaround.direct.txt @@ -0,0 +1,37 @@ + +>>> local_aws_attributes +Warning: There are no files to sync, please check your .gitignore + +{ + "with_pool": {}, + "without_pool": { + "availability": "SPOT" + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> job_requests + +>>> remote_aws_attributes +with_pool: +{ + "instance_pool_id": "pool-id", + "node_type_id": null, + "aws_attributes": {} +} +without_pool: +{ + "instance_pool_id": null, + "node_type_id": "m5.large", + "aws_attributes": { + "availability": "SPOT" + } +} diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/out.workaround.terraform.txt b/acceptance/bundle/resources/jobs/clear_instance_profile/out.workaround.terraform.txt new file mode 100644 index 0000000000..5a3d5d7054 --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/out.workaround.terraform.txt @@ -0,0 +1,40 @@ + +>>> local_aws_attributes +Warning: There are no files to sync, please check your .gitignore + +{ + "with_pool": {}, + "without_pool": { + "availability": "SPOT" + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> job_requests + +>>> remote_aws_attributes +with_pool: +{ + "instance_pool_id": "pool-id", + "node_type_id": null, + "aws_attributes": { + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/original" + } +} +without_pool: +{ + "instance_pool_id": null, + "node_type_id": "m5.large", + "aws_attributes": { + "availability": "SPOT", + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/original" + } +} diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/output.txt b/acceptance/bundle/resources/jobs/clear_instance_profile/output.txt new file mode 100644 index 0000000000..2b3e84cf83 --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/output.txt @@ -0,0 +1,32 @@ + +=== Create both jobs with aws_attributes.instance_profile_arn set +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> job_requests + +>>> remote_aws_attributes +with_pool: +{ + "instance_pool_id": "pool-id", + "node_type_id": null, + "aws_attributes": { + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/original" + } +} +without_pool: +{ + "instance_pool_id": null, + "node_type_id": "m5.large", + "aws_attributes": { + "availability": "SPOT", + "instance_profile_arn": "arn:aws:iam::[NUMID]:instance-profile/original" + } +} + +=== Case 1: drop the instance_profile_arn lines, keep the aws_attributes blocks +=== Case 2: the suggested workaround, instance_profile_arn: "" +=== Reference: replacing the ARN with a different one is detected \ No newline at end of file diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/script b/acceptance/bundle/resources/jobs/clear_instance_profile/script new file mode 100644 index 0000000000..efc8e10748 --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/script @@ -0,0 +1,39 @@ +# Under the terraform engine, a job whose new_cluster runs on an +# instance pool cannot have aws_attributes.instance_profile_arn cleared. Neither +# dropping the field nor the suggested `instance_profile_arn: ""` workaround +# reaches the backend; replacing the ARN with a different one does. Each phase +# records a per-engine file because the direct engine does clear it. +# +# with_pool vs without_pool isolates the instance-pool-specific part: for +# without_pool the empty string survives the mutator pipeline (and is then +# dropped by DropEmptyStrings), while for with_pool it is already gone from the +# resolved config that `bundle validate` prints. + +echo "*" > .gitignore + +ARN_LINE=" instance_profile_arn: arn:aws:iam::123456789012:instance-profile/original" + +title "Create both jobs with aws_attributes.instance_profile_arn set" +trace $CLI bundle deploy +trace job_requests > out.create.requests.$DATABRICKS_BUNDLE_ENGINE.json + +declare -A JOB_IDS +for name in with_pool without_pool; do + JOB_IDS[$name]=`read_id.py $name` +done +trace remote_aws_attributes + +title "Case 1: drop the instance_profile_arn lines, keep the aws_attributes blocks" +update_file.py databricks.yml "$ARN_LINE" " # instance_profile_arn removed" +phase remove + +title 'Case 2: the suggested workaround, instance_profile_arn: ""' +update_file.py databricks.yml " # instance_profile_arn removed" ' instance_profile_arn: ""' +phase workaround + +title "Reference: replacing the ARN with a different one is detected" +update_file.py databricks.yml ' instance_profile_arn: ""' " instance_profile_arn: arn:aws:iam::123456789012:instance-profile/replacement" +phase replace + +# The last `jobs get` of the last phase leaves a recorded GET behind. +rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/script.prepare b/acceptance/bundle/resources/jobs/clear_instance_profile/script.prepare new file mode 100644 index 0000000000..523242ea1a --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/script.prepare @@ -0,0 +1,34 @@ +# Project each job's cluster spec down to the fields this test asserts, so a new +# field in the Jobs API response cannot break the goldens. +remote_aws_attributes() { + for name in with_pool without_pool; do + echo "$name:" + $CLI jobs get "${JOB_IDS[$name]}" | jq '.settings.tasks[0].new_cluster | {instance_pool_id, node_type_id, aws_attributes}' + done +} + +# The resolved config the engine actually deploys: aws_attributes as it survives +# the mutator pipeline. This is where the empty string is lost, so record it +# alongside the request and the remote state. +local_aws_attributes() { + $CLI bundle validate -o json | jq '.resources.jobs | map_values(.tasks[0].new_cluster.aws_attributes)' +} + +# The two jobs are deployed concurrently, so order by job name rather than +# print_requests.py --sort: that sorts on the rendered request, whose job_id +# differs per run. +job_requests() { + print_requests.py //jobs | jq -s 'sort_by(.body.new_settings.name // .body.name) | .[]' +} + +# Plan, deploy, and read back. The engines disagree on whether the ARN is +# cleared, so every phase records into its own per-engine file. +phase() { + { + trace local_aws_attributes + trace $CLI bundle plan + trace $CLI bundle deploy + trace job_requests + trace remote_aws_attributes + } &> "out.$1.$DATABRICKS_BUNDLE_ENGINE.txt" +} diff --git a/acceptance/bundle/resources/jobs/clear_instance_profile/test.toml b/acceptance/bundle/resources/jobs/clear_instance_profile/test.toml new file mode 100644 index 0000000000..6f5264d8ea --- /dev/null +++ b/acceptance/bundle/resources/jobs/clear_instance_profile/test.toml @@ -0,0 +1,3 @@ +RecordRequests = true + +Badness = "Under terraform, aws_attributes.instance_profile_arn cannot be cleared on an instance-pool cluster. Two independent drops: ModifyRequestOnInstancePool rebuilds AwsAttributes without ForceSendFields (losing the explicit \"\"), and DropEmptyStrings removes the empty string for every cluster. Neither the removal nor the documented workaround reaches the backend. The direct engine clears the field in both cases."