Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/deploy-progressive-resource-output.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`bundle deploy` on the direct engine now reports each resource as soon as it is deployed, instead of listing them all after the deployment finishes. A deploy that fails part way through now reports the resources it did apply.
32 changes: 29 additions & 3 deletions acceptance/acceptance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1020,14 +1020,15 @@ func runTest(t *testing.T,
printedRepls := false

pathFilter := preparePathFilter(config, customEnv)
sortLines := compileSortLines(t, config)

// Compare expected outputs
for relPath := range outputs {
if shouldSkip(pathFilter, relPath) {
continue
}

doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
doComparison(t, repls, sortLines, dir, tmpDir, relPath, &printedRepls)
}

// Make sure there are not unaccounted for new files
Expand Down Expand Up @@ -1058,7 +1059,7 @@ func runTest(t *testing.T,
if strings.HasPrefix(relPath, "out") {
// We have a new file starting with "out"
// Show the contents & support overwrite mode for it:
doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
doComparison(t, repls, sortLines, dir, tmpDir, relPath, &printedRepls)
}
}

Expand Down Expand Up @@ -1131,7 +1132,24 @@ func addEnvVar(t *testing.T, env []string, repls *testdiff.ReplacementsContext,
return append(env, key+"="+newValue)
}

func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) {
// compileSortLines compiles the enabled SortLines patterns from the test config.
// Patterns are returned in name order so the result does not depend on map iteration.
func compileSortLines(t *testing.T, config internal.TestConfig) []*regexp.Regexp {
result := make([]*regexp.Regexp, 0, len(config.SortLines))
for _, name := range slices.Sorted(maps.Keys(config.SortLines)) {
if on, ok := config.SortLinesOn[name]; ok && !on {
continue
}
re, err := regexp.Compile(config.SortLines[name])
if err != nil {
t.Fatalf("Invalid SortLines pattern %s = %#v: %s", name, config.SortLines[name], err)
}
result = append(result, re)
}
return result
}

func doComparison(t *testing.T, repls testdiff.ReplacementsContext, sortLines []*regexp.Regexp, dirRef, dirNew, relPath string, printedRepls *bool) {
pathRef := filepath.Join(dirRef, relPath)
pathNew := filepath.Join(dirNew, relPath)
bufRef, okRef := tryReading(t, pathRef)
Expand All @@ -1150,6 +1168,14 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
valueNew = repls.Replace(valueNew)
}

// Canonicalize line runs whose order the command does not guarantee. Applied to
// the reference too so a hand-edited golden compares the same way; sorting is
// idempotent, so a stored reference is unaffected.
for _, re := range sortLines {
valueRef = testdiff.SortLineRuns(valueRef, re)
valueNew = testdiff.SortLineRuns(valueNew, re)
}

// In update mode, regenerating the reference files is the goal: each branch below
// writes or removes the reference and returns without failing the test. Genuine
// problems still fail — read errors (via tryReading above), write errors (via
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@

>>> errcode [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created jobs.my_job
Error: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json (403 INJECTED)

Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fresources.json?overwrite=true
Expand Down
2 changes: 2 additions & 0 deletions acceptance/bundle/deploy/wal/chain-3-jobs/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

>>> errcode [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/wal-chain-test/default/files...
Created jobs.job_01
Created jobs.job_02
[PROCESS_KILLED]

Exit code: [KILLED]
Expand Down
1 change: 1 addition & 0 deletions acceptance/bundle/deploy/wal/crash-after-create/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

>>> errcode [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/wal-crash-test/default/files...
Created jobs.job_a
[PROCESS_KILLED]

Exit code: [KILLED]
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
bundle:
name: test-bundle
engine: direct

experimental:
scripts:
postdeploy: "python3 ./myscript.py $POSTDEPLOY_EXITCODE postdeploy"
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import sys

exitcode, name = sys.argv[1], sys.argv[2]
print(f"from {name}: hello")
sys.exit(int(exitcode))

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@

=== postdeploy script output comes before the migration notice
>>> [CLI] bundle deploy
Warn: Direct engine requested in bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Executing 'postdeploy' script
from postdeploy: hello
Removing empty terraform state; direct engine will be used on the next deploy (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)...
Files: 6 uploaded, 0 deleted
Resources: 0 created, 0 changed, 0 deleted, 0 unchanged

=== Same when the postdeploy script fails: the migration still runs
>>> errcode [CLI] bundle deploy
Warn: Direct engine requested in bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Executing 'postdeploy' script
from postdeploy: hello
Error: failed to execute script: exit status 1

Removing empty terraform state; direct engine will be used on the next deploy (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)...
Files: 6 uploaded, 0 deleted
Resources: 0 created, 0 changed, 0 deleted, 0 unchanged

Exit code: 1
22 changes: 22 additions & 0 deletions acceptance/bundle/migrate/auto-migrate-postdeploy-script/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# The deploy runs first, then the postdeploy script, then the migration. The migration is
# not gated on the script: the resources are deployed either way, so the state describes
# the same deployment whether or not the script succeeded.

seed_empty_tfstate() {
mkdir -p .databricks/bundle/default/terraform
cat > .databricks/bundle/default/terraform/terraform.tfstate <<'JSON'
{"version": 4, "serial": 1, "lineage": "test-lineage", "resources": []}
JSON
}

seed_empty_tfstate
title "postdeploy script output comes before the migration notice"
trace $CLI bundle deploy

title "Same when the postdeploy script fails: the migration still runs"
rm -rf .databricks
seed_empty_tfstate
export POSTDEPLOY_EXITCODE=1
trace errcode $CLI bundle deploy

rm -f out.requests.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Env.POSTDEPLOY_EXITCODE = "0"
4 changes: 2 additions & 2 deletions acceptance/bundle/resource_deps/create_error/script
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
echo "*" > .gitignore
trace $CLI bundle plan 2>&1
musterr $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt
musterr $CLI bundle deploy -q &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt # -q: independent job races the failure
trace print_requests.py --sort //jobs

trace $CLI bundle summary
Expand All @@ -12,7 +12,7 @@ title "Plan should still contain foo and bar"
trace $CLI bundle plan

rm out.requests.txt
musterr $CLI bundle deploy &> out.deploy2.$DATABRICKS_BUNDLE_ENGINE.txt
musterr $CLI bundle deploy -q &> out.deploy2.$DATABRICKS_BUNDLE_ENGINE.txt
title "Expecting no difference in the output between first and second deploy"
diff.py out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt out.deploy2.$DATABRICKS_BUNDLE_ENGINE.txt
rm out.deploy2.$DATABRICKS_BUNDLE_ENGINE.txt
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created pipelines.foo
Error: cannot create resources.pipelines.bar: cannot resolve "${resources.pipelines.foo.ingestion_definition.connection_name}": ingestion_definition: cannot access nil value

Files: 7 uploaded, 0 deleted
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/state/default/files...
Created jobs.test
Error: cannot create resources.jobs.bar: cannot resolve "${resources.jobs.test.tasks[0].new_cluster.custom_tags.missing_tag}": tasks[0].new_cluster.custom_tags.missing_tag: key "missing_tag" not found in map

Files: 7 uploaded, 0 deleted
2 changes: 1 addition & 1 deletion acceptance/bundle/resources/job_runs/basic/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,10 @@ Resources:

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files...
Created jobs.my_job
Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]
Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS
Created job_runs.my_run
Created jobs.my_job
Files: 4 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

Expand Down
1 change: 1 addition & 0 deletions acceptance/bundle/resources/job_runs/failed_run/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
=== a run that finishes FAILED fails the deploy
>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
Created jobs.my_job
Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [RUN_URL]
Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details.
task "main": RuntimeError: intentional failure
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
=== the deploy stops waiting before the run finishes
>>> errcode [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-interrupted-run/default/files...
Created jobs.my_job
Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: Fault injected by test. (403 INJECTED)

Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/runs/get?run_id=[MY_RUN_ID]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
=== deploy triggers the run with only the overridden parameter
>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default/files...
Created jobs.my_job
Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]
Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS
Created job_runs.my_run
Created jobs.my_job
Files: 4 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
=== first deploy triggers a run
>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-bundle-deploy/default/files...
Created jobs.my_job
Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]
Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS
Created job_runs.my_run
Created jobs.my_job
Files: 5 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/resources/job_runs/redeploy/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
=== initial deploy triggers the first run
>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files...
Created jobs.my_job
Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]
Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS
Created job_runs.my_run
Created jobs.my_job
Files: 4 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
=== the deploy succeeds, though its run-now was answered with a 503
>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-retried-run-now/default/files...
Created jobs.my_job
Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]
Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS
Created job_runs.my_run
Created jobs.my_job
Files: 5 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/resources/job_runs/wait/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
=== the deploy waits for the run to finish
>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
Created jobs.my_job
Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [RUN_URL]
Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS
Created job_runs.my_run
Created jobs.downstream_job
Created jobs.my_job
Files: 6 uploaded, 0 deleted
Resources: 3 created, 0 changed, 0 deleted, 0 unchanged

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.pipelines.foo.permissions: retrying after 504 Gateway Timeout from PUT /api/2.0/permissions/pipelines/[UUID]
Created pipelines.foo
Warn: deploying resources.pipelines.foo.permissions: retrying after 504 Gateway Timeout from PUT /api/2.0/permissions/pipelines/[UUID]
Created pipelines.foo.permissions
Files: 5 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ Warning: required field "branch_id" is not set
in databricks.yml:21:7

Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-branch-no-id-[UNIQUE_NAME]/default/files...
Created postgres_projects.my_project
Error: cannot create resources.postgres_branches.main: Field 'branch_id' is required, expected non-default value (not "")! (400 INVALID_PARAMETER_VALUE)

Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/bad-database-id-[UNIQUE_NAME]/default/files...
Created postgres_branches.main
Created postgres_projects.my_project
Created postgres_roles.owner
Error: cannot create resources.postgres_databases.my_database: Field database_id must match pattern ^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$, got 'Invalid_DB_ID'. (400 INVALID_PARAMETER_VALUE)

Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases?database_id=Invalid_DB_ID
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/bad-role-ref-[UNIQUE_NAME]/default/files...
Created postgres_branches.main
Created postgres_projects.my_project
Error: cannot create resources.postgres_databases.my_database: role not found; role_id:"does-not-exist" [TraceId: [TRACE_ID]] (404 NOT_FOUND)

Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases?database_id=my-database
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ Warning: required field "endpoint_id" is not set
in databricks.yml:27:7

Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-endpoint-no-id-[UNIQUE_NAME]/default/files...
Created postgres_branches.main
Created postgres_projects.my_project
Error: cannot create resources.postgres_endpoints.primary: Field 'endpoint_id' is required, expected non-default value (not "")! (400 INVALID_PARAMETER_VALUE)

Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@

>>> musterr [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/files...
Created schemas.myschema
Error: cannot create resources.volumes.volume1: CreateVolume storage_location can not be provided. (400 INVALID_PARAMETER_VALUE)

Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/volumes
Expand Down
11 changes: 11 additions & 0 deletions acceptance/internal/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,17 @@ type TestConfig struct {
// List of request headers to include when recording requests.
IncludeRequestHeaders []string

// Map of name -> regexp. Each run of consecutive output lines matching one of the
// regexps is sorted before comparison. Use for output whose line order is not
// deterministic, e.g. "bundle deploy" reporting resources as they are applied in
// parallel. Entries are keyed by name so an inner test.toml can replace an
// inherited pattern by reusing its name, or switch it off via SortLinesOn.
SortLines map[string]string

// Map of SortLines name -> whether to apply it. If a name is not listed, defaults
// to true; set it to false to drop an inherited pattern for this test.
SortLinesOn map[string]bool

// List of gitignore patterns to ignore when checking output files
Ignore []string

Expand Down
2 changes: 2 additions & 0 deletions acceptance/selftest/sortlines/out.test.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions acceptance/selftest/sortlines/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
Fruit: quince
Fruit: zucchini
Not a fruit: rock
Fruit: apple
Fruit: banana
Not a fruit: spoon
Fruit: melon
Created jobs.z
Created jobs.a
2 changes: 2 additions & 0 deletions acceptance/selftest/sortlines/override/out.test.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions acceptance/selftest/sortlines/override/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fruit: pear
Fruit: apple
Veg: carrot
Veg: turnip
4 changes: 4 additions & 0 deletions acceptance/selftest/sortlines/override/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
echo "Fruit: pear"
echo "Fruit: apple"
echo "Veg: turnip"
echo "Veg: carrot"
3 changes: 3 additions & 0 deletions acceptance/selftest/sortlines/override/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Reusing the inherited name replaces the parent's pattern rather than adding to it:
# "Fruit:" lines are left alone from here on, and "Veg:" lines are sorted instead.
SortLines.fruit = '^Veg: '
15 changes: 15 additions & 0 deletions acceptance/selftest/sortlines/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Each run of consecutive "Fruit:" lines is sorted on its own, and a non-matching line ends
# the run. The values are picked so that this is visible in the output: the first block
# sorts late in the alphabet and the second sorts early, so if the two blocks were sorted
# together, "apple" and "banana" would move up past "Not a fruit: rock".
echo "Fruit: zucchini"
echo "Fruit: quince"
echo "Not a fruit: rock"
echo "Fruit: banana"
echo "Fruit: apple"
echo "Not a fruit: spoon"
echo "Fruit: melon"

# deploy_actions is disabled here, so these keep the order they were printed in.
echo "Created jobs.z"
echo "Created jobs.a"
Loading
Loading