diff --git a/.nextchanges/bundles/job-runs-on-bundle-deploy.md b/.nextchanges/bundles/job-runs-on-bundle-deploy.md new file mode 100644 index 00000000000..126b403edc3 --- /dev/null +++ b/.nextchanges/bundles/job-runs-on-bundle-deploy.md @@ -0,0 +1 @@ +direct: `resources.job_runs` can set `lifecycle.triggers.on_bundle_deploy: true` to re-fire the run on every bundle deploy. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 6c2b033fa97..49418540acf 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -845,8 +845,15 @@ resources.job_runs.*.jar_params[*] string ALL resources.job_runs.*.job_id int64 ALL resources.job_runs.*.job_parameters map[string]string ALL resources.job_runs.*.job_parameters.* string ALL +resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState STATE +resources.job_runs.*.lifecycle *resources.JobRunLifecycle INPUT resources.job_runs.*.lifecycle resources.Lifecycle INPUT resources.job_runs.*.lifecycle.prevent_destroy bool INPUT +resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE +resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT +resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE +resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT +resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.modified_status string INPUT resources.job_runs.*.notebook_params map[string]string ALL resources.job_runs.*.notebook_params.* string ALL diff --git a/acceptance/bundle/resources/job_runs/basic/output.txt b/acceptance/bundle/resources/job_runs/basic/output.txt index eb1b1461eda..95624f43a11 100644 --- a/acceptance/bundle/resources/job_runs/basic/output.txt +++ b/acceptance/bundle/resources/job_runs/basic/output.txt @@ -37,7 +37,7 @@ job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID] job run [MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 5 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy diff --git a/acceptance/bundle/resources/job_runs/failed_run/test.toml b/acceptance/bundle/resources/job_runs/failed_run/test.toml index 14707b0e51d..561602d5295 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/test.toml +++ b/acceptance/bundle/resources/job_runs/failed_run/test.toml @@ -1,7 +1,3 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - # Runs the failing job for real, so the message the deploy names the task with is # one a workspace reported. Serverless needs Unity Catalog. Cloud = true diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/test.toml b/acceptance/bundle/resources/job_runs/interrupted_run/test.toml index 5e1d7da088b..dc6f4cafee7 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/test.toml +++ b/acceptance/bundle/resources/job_runs/interrupted_run/test.toml @@ -1,8 +1,3 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - # The interruption is staged by injecting a fault into the testserver, so this # stays off cloud. -RecordRequests = true Ignore = ["tmp.plan.json"] diff --git a/acceptance/bundle/resources/job_runs/job_parameters/output.txt b/acceptance/bundle/resources/job_runs/job_parameters/output.txt index 64016f7356b..ac37228ada8 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/output.txt +++ b/acceptance/bundle/resources/job_runs/job_parameters/output.txt @@ -6,7 +6,7 @@ job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID] job run [MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 5 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/job_parameters/test.toml b/acceptance/bundle/resources/job_runs/job_parameters/test.toml deleted file mode 100644 index 4b94d8b58e9..00000000000 --- a/acceptance/bundle/resources/job_runs/job_parameters/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -RecordRequests = true diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/databricks.yml b/acceptance/bundle/resources/job_runs/on_bundle_deploy/databricks.yml new file mode 100644 index 00000000000..0ca892109c3 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-bundle-deploy + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_bundle_deploy: true diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/out.test.toml b/acceptance/bundle/resources/job_runs/on_bundle_deploy/out.test.toml new file mode 100644 index 00000000000..57b0f616850 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt new file mode 100644 index 00000000000..2ed8e3bf01a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -0,0 +1,145 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-bundle-deploy/default/files... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== redeploy re-fires with unchanged config +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_bundle_deploy": "[UUID]" + } + }, + "new": { + "triggers": { + "on_bundle_deploy": "[UUID]" + } + } + }, + "lifecycle.triggers.on_bundle_deploy": { + "action": "recreate", + "reason": "immutable", + "old": "[UUID]", + "new": "[UUID]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-bundle-deploy/default/files... +job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +job run [MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> [CLI] bundle summary +Name: job-runs-on-bundle-deploy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-bundle-deploy/default +Resources: + Job Runs: + my_run: + Name: + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?w=[NUMID] + Jobs: + my_job: + Name: my-job + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[NUMID] + +=== second run-now after recreate +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== removing on_bundle_deploy rewrites state without a run +>>> [CLI] bundle plan +update job_runs.my_run + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "update", + "reason": "trigger removed", + "old": { + "triggers": { + "on_bundle_deploy": "[UUID]" + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-bundle-deploy/default/files... +Updated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-bundle-deploy/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/script b/acceptance/bundle/resources/job_runs/on_bundle_deploy/script new file mode 100644 index 00000000000..e24cf944cd2 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/script @@ -0,0 +1,41 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +# Name the first run so the second becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "redeploy re-fires with unchanged config" +trace $CLI bundle plan +# Save the plan so the READPLAN=1 variant deploys the fingerprint minted here. +# The deploys are not traced: readplanarg makes the command line differ per variant. +$CLI bundle plan -o json > tmp.plan.json +trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +read_id.py my_run > /dev/null +trace $CLI bundle summary + +title "second run-now after recreate" +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "removing on_bundle_deploy rewrites state without a run" +# Drop the lifecycle block so the next plan compares a nil fingerprint against +# the UUID left in state: a state-only update that clears it, never a recreate. +update_file.py databricks.yml " lifecycle: + triggers: + - on_bundle_deploy: true +" "" +trace $CLI bundle plan +$CLI bundle plan -o json > tmp.plan.json +trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/test.toml b/acceptance/bundle/resources/job_runs/on_bundle_deploy/test.toml new file mode 100644 index 00000000000..663d9510c10 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/test.toml @@ -0,0 +1,6 @@ +# Deploy both by re-planning and from a plan saved on disk, so the fingerprint +# minted during planning is exercised across plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +# The saved plan embeds a fresh fingerprint on every run. +Ignore = ["tmp.plan.json"] diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 7f3c1fcf664..e5d381769d0 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -6,7 +6,7 @@ job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID] job run [MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 5 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle summary diff --git a/acceptance/bundle/resources/job_runs/redeploy/test.toml b/acceptance/bundle/resources/job_runs/redeploy/test.toml deleted file mode 100644 index 4b94d8b58e9..00000000000 --- a/acceptance/bundle/resources/job_runs/redeploy/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -RecordRequests = true diff --git a/acceptance/bundle/resources/job_runs/retried_run_now/test.toml b/acceptance/bundle/resources/job_runs/retried_run_now/test.toml index 3930a076f57..fce68392ea1 100644 --- a/acceptance/bundle/resources/job_runs/retried_run_now/test.toml +++ b/acceptance/bundle/resources/job_runs/retried_run_now/test.toml @@ -1,10 +1,4 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - # Local only: the lost response is staged by injecting a fault into the testserver. -RecordRequests = true - Ignore = [".databricks"] # Number each distinct token so a reused token shows as [0] on both requests. diff --git a/acceptance/bundle/resources/job_runs/basic/test.toml b/acceptance/bundle/resources/job_runs/test.toml similarity index 88% rename from acceptance/bundle/resources/job_runs/basic/test.toml rename to acceptance/bundle/resources/job_runs/test.toml index 4b94d8b58e9..b056e544027 100644 --- a/acceptance/bundle/resources/job_runs/basic/test.toml +++ b/acceptance/bundle/resources/job_runs/test.toml @@ -1,4 +1,3 @@ # job_runs is a direct-engine-only resource; the Terraform provider has no # equivalent, so restrict the matrix to direct. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -RecordRequests = true diff --git a/acceptance/bundle/resources/job_runs/wait/test.toml b/acceptance/bundle/resources/job_runs/wait/test.toml index 5c84ac24640..989d6b2052d 100644 --- a/acceptance/bundle/resources/job_runs/wait/test.toml +++ b/acceptance/bundle/resources/job_runs/wait/test.toml @@ -1,7 +1,3 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - # Runs the job for real on cloud. Serverless needs Unity Catalog. Cloud = true RequiresUnityCatalog = true diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go new file mode 100644 index 00000000000..a7c5ada925e --- /dev/null +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -0,0 +1,56 @@ +package mutator + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type validateJobRunTriggers struct{} + +// ValidateJobRunTriggers rejects invalid lifecycle.triggers on job_runs. +func ValidateJobRunTriggers() bundle.Mutator { + return &validateJobRunTriggers{} +} + +func (*validateJobRunTriggers) Name() string { + return "ValidateJobRunTriggers" +} + +func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + for name, jr := range b.Config.Resources.JobRuns { + if jr == nil || jr.Lifecycle == nil { + continue + } + // Recreate-every-deploy cannot coexist with prevent_destroy. + if jr.HasOnBundleDeploy() && jr.Lifecycle.PreventDestroy { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), + }) + } + for i, t := range jr.Lifecycle.Triggers { + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) + if t.OnBundleDeploy == nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + Locations: b.Config.GetLocations(path), + }) + continue + } + if !*t.OnBundleDeploy { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_bundle_deploy must be true when set", + Locations: b.Config.GetLocations(path + ".on_bundle_deploy"), + }) + } + } + } + return diags +} diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go new file mode 100644 index 00000000000..9c225457471 --- /dev/null +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -0,0 +1,82 @@ +package mutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" +) + +func TestValidateJobRunTriggers(t *testing.T) { + trueVal := true + falseVal := false + + tests := []struct { + name string + triggers []resources.JobRunTrigger + preventDestroy bool + summary string + }{ + { + name: "on_bundle_deploy true", + triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &trueVal}, + }, + }, + { + name: "empty entry", + triggers: []resources.JobRunTrigger{ + {}, + }, + summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + }, + { + name: "on_bundle_deploy false", + triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &falseVal}, + }, + summary: "lifecycle.triggers.on_bundle_deploy must be true when set", + }, + { + name: "on_bundle_deploy with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &trueVal}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + }, + { + name: "prevent_destroy alone", + preventDestroy: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Lifecycle: resources.Lifecycle{PreventDestroy: tt.preventDestroy}, + Triggers: tt.triggers, + }, + }, + }, + }, + }, + } + diags := bundle.Apply(t.Context(), b, mutator.ValidateJobRunTriggers()) + if tt.summary == "" { + assert.Empty(t, diags) + return + } + assert.True(t, diags.HasError()) + assert.Equal(t, tt.summary, diags[0].Summary) + }) + } +} diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 8db6ced76e1..56f3c39a356 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -14,18 +14,34 @@ import ( ) // JobRun is the bundle config for a triggered job run, described by the same -// fields as the Jobs RunNow request (embedded). It re-triggers only when its own -// config changes, not when the targeted job (stable job_id) changes. +// fields as the Jobs RunNow request (embedded). By default it re-fires when its +// own configuration changes; lifecycle.triggers can add further conditions. type JobRun struct { BaseResource jobs.RunNow + // Lifecycle shadows BaseResource.Lifecycle so job_runs can set triggers. + Lifecycle *JobRunLifecycle `json:"lifecycle,omitempty"` + // ResolvedJobID holds the run's job_id loaded from state, used only to build // the run URL. Keeping it separate from RunNow.JobId (a ${resources.jobs.*.id} // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` } +// HasOnBundleDeploy reports whether any trigger re-fires on every deploy. +func (r *JobRun) HasOnBundleDeploy() bool { + if r.Lifecycle == nil { + return false + } + for _, t := range r.Lifecycle.Triggers { + if t.OnBundleDeploy != nil && *t.OnBundleDeploy { + return true + } + } + return false +} + func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index db2b130d313..88d130239ee 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -25,3 +25,16 @@ type LifecycleWithStarted struct { // Supported only for apps, clusters, and sql_warehouses. Started *bool `json:"started,omitempty"` } + +// JobRunLifecycle extends Lifecycle with run-fire triggers. +type JobRunLifecycle struct { + Lifecycle + + // Triggers that cause the run to re-fire (in addition to config changes). + Triggers []JobRunTrigger `json:"triggers,omitempty"` +} + +// JobRunTrigger is one lifecycle.triggers entry. +type JobRunTrigger struct { + OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` +} diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index ac549b876cd..c7487aa0810 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,12 +26,27 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour +// JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. +type JobRunTriggersState struct { + // Fresh UUID each plan while armed so Old!=New forces recreate. + OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` +} + +// JobRunLifecycleState holds local-only lifecycle fields persisted in state. +type JobRunLifecycleState struct { + Triggers *JobRunTriggersState `json:"triggers,omitempty"` +} + // JobRunState is the RunNow request plus the outcome required for planning. type JobRunState struct { jobs.RunNow // Always SUCCESS during planning and cleared before persistence. ResultState jobs.RunResultState `json:"result_state,omitempty"` + + // Local-only; listed in knownMissingInRemoteType. Nested under lifecycle to + // mirror config and avoid colliding with a future Jobs API field. + Lifecycle *JobRunLifecycleState `json:"lifecycle,omitempty"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { @@ -79,10 +94,17 @@ func (*ResourceJobRun) New(client *databricks.WorkspaceClient) *ResourceJobRun { } func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { - return &JobRunState{ + state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, + Lifecycle: nil, + } + if input.HasOnBundleDeploy() { + state.Lifecycle = &JobRunLifecycleState{ + Triggers: &JobRunTriggersState{OnBundleDeploy: uuid.NewString()}, + } } + return state } // makeJobRunRemote maps the GetRun response into the RunNow-shaped remote: GET @@ -158,7 +180,12 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, // RemapState extracts the fields used for diffing: the RunNow request and the // outcome the run reached. func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { - return &JobRunState{RunNow: remote.RunNow, ResultState: remote.ResultState} + return &JobRunState{ + RunNow: remote.RunNow, + ResultState: remote.ResultState, + // Local-only lifecycle fingerprints stay unset on the remapped remote. + Lifecycle: nil, + } } func (r *ResourceJobRun) DoCreate(ctx context.Context, config *JobRunState) (string, *JobRunRemote, error) { @@ -336,8 +363,15 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { } } -// DoUpdate finishes the wait an interrupted deploy abandoned. -func (r *ResourceJobRun) DoUpdate(ctx context.Context, id string, config *JobRunState, _ *PlanEntry) (*JobRunRemote, error) { +// DoUpdate rewrites state after a cleared trigger (no API call) or finishes the +// wait an interrupted deploy abandoned. +func (r *ResourceJobRun) DoUpdate(ctx context.Context, id string, config *JobRunState, entry *PlanEntry) (*JobRunRemote, error) { + // Clearing a trigger only drops its local-only fingerprint from state; wait on + // the run only when some other field changed. + if !entry.Changes.HasChangeExcept("lifecycle", "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy") { + config.ResultState = "" + return nil, nil + } remote, err := r.waitForRun(ctx, id) config.ResultState = "" return remote, err @@ -347,14 +381,29 @@ func (r *ResourceJobRun) DoUpdate(ctx context.Context, id string, config *JobRun // still going, so a run that may yet succeed is adopted and waited on. A run that // stopped without succeeding keeps its recreate. A SKIPPED run reports no // result_state either, so the lifecycle state is what tells the two apart. +// Clearing a trigger downgrades the recreate to a state-only update so the +// fingerprint is dropped from state without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - // The planner passes no remote state when the run could not be read. - if path.String() != "result_state" || remote == nil || runIsTerminal(remote.State.LifeCycleState) { + switch path.String() { + case "lifecycle", "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy": + // A cleared trigger sets New empty; structdiff may report it at lifecycle, + // lifecycle.triggers, or the leaf. DoUpdate treats these paths as no-ops. + if change.New == nil || change.New == "" { + change.Action = deployplan.Update + change.Reason = "trigger removed" + } + return nil + case "result_state": + // The planner passes no remote state when the run could not be read. + if remote == nil || runIsTerminal(remote.State.LifeCycleState) { + return nil + } + change.Action = deployplan.Update + change.Reason = "run in progress" + return nil + default: return nil } - change.Action = deployplan.Update - change.Reason = "run in progress" - return nil } // DoDelete deletes the run via jobs/runs/delete, on both destroy and the diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 62a22a9745f..587ee594948 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -342,6 +343,65 @@ func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { assert.Equal(t, jobs.RunResultStateSuccess, state.ResultState) } +func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { + t.Run("unset", func(t *testing.T) { + state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{}) + assert.Nil(t, state.Lifecycle) + }) + + t.Run("armed", func(t *testing.T) { + on := true + input := &resources.JobRun{ + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, + }, + } + first := (&ResourceJobRun{}).PrepareState(input) + require.NotNil(t, first.Lifecycle) + require.NotNil(t, first.Lifecycle.Triggers) + assert.NotEmpty(t, first.Lifecycle.Triggers.OnBundleDeploy) + + second := (&ResourceJobRun{}).PrepareState(input) + assert.NotEqual(t, first.Lifecycle.Triggers.OnBundleDeploy, second.Lifecycle.Triggers.OnBundleDeploy) + }) +} + +func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { + r := &ResourceJobRun{} + + t.Run("clearing lifecycle downgrades to update", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: &JobRunLifecycleState{Triggers: &JobRunTriggersState{OnBundleDeploy: "old"}}, + New: nil, + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle"), change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + assert.Equal(t, "trigger removed", change.Reason) + }) + + t.Run("clearing on_bundle_deploy leaf downgrades to update", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: "old", + New: "", + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy"), change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + assert.Equal(t, "trigger removed", change.Reason) + }) + + t.Run("fresh fingerprint still recreates", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: "old", + New: "new", + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy"), change, nil)) + assert.Equal(t, deployplan.Recreate, change.Action) + }) +} + // The planner diffs RemapState(remote) against PrepareState(config), so a run // that did not end in SUCCESS has to surface as a difference on result_state. func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 109b98c3a86..2d5516d59c7 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -56,6 +56,10 @@ var knownMissingInRemoteType = map[string][]string{ "vector_search_endpoints": { "usage_policy_id", }, + "job_runs": { + // Local-only trigger fingerprints under lifecycle. + "lifecycle", + }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. @@ -82,6 +86,11 @@ var knownMissingInStateType = map[string][]string{ "sql_warehouses": { "lifecycle.prevent_destroy", }, + "job_runs": { + // State stores trigger fingerprints, not the config trigger list / prevent_destroy. + "lifecycle.prevent_destroy", + "lifecycle.triggers[*]", + }, "dashboards": { "file_path", }, diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 60bedb38d16..a47e8151681 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -978,7 +978,18 @@ resources: "$fields": "lifecycle": "description": |- - Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed and when the run re-fires. + "$fields": + "prevent_destroy": + "description": |- + Lifecycle setting to prevent the resource from being destroyed. + "triggers": + "description": |- + Conditions that re-fire this job run (in addition to configuration changes). + "$fields": + "on_bundle_deploy": + "description": |- + If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index a130da820d6..b15e1c30df6 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -188,6 +188,9 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Reject configured job_runs.idempotency_token; the CLI sets it on run-now. validate.ValidateJobRunIdempotencyToken(), + // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). + mutator.ValidateJobRunTriggers(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index aee481b2b14..3db0864c2c4 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -29,6 +29,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateDirectOnlyResources(engine), mutator.ValidateLifecycleStarted(engine), mutator.ValidateCascadeOnDestroy(engine), + mutator.ValidateJobRunTriggers(), statemgmt.CheckRunningResource(engine), ) } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index b4084979746..e0c710428a3 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1199,8 +1199,8 @@ "$ref": "#/$defs/map/string" }, "lifecycle": { - "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed and when the run re-fires.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.JobRunLifecycle" }, "notebook_params": { "description": "[Private Preview] A map from keys to values for jobs with notebook task, for example `\"notebook_params\": {\"name\": \"john doe\", \"age\": \"35\"}`.\nThe map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function.\n\nIf not specified upon `run-now`, the triggered run uses the job’s base parameters.\n\nnotebook_params cannot be specified in conjunction with jar_params.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.\n\nThe JSON representation of this field (for example `{\"notebook_params\":{\"name\":\"john doe\",\"age\":\"35\"}}`) cannot exceed 10,000 bytes.", @@ -1270,6 +1270,46 @@ } ] }, + "resources.JobRunLifecycle": { + "oneOf": [ + { + "type": "object", + "properties": { + "prevent_destroy": { + "description": "Lifecycle setting to prevent the resource from being destroyed.", + "$ref": "#/$defs/bool" + }, + "triggers": { + "description": "Conditions that re-fire this job run (in addition to configuration changes).", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.JobRunTrigger" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.JobRunTrigger": { + "oneOf": [ + { + "type": "object", + "properties": { + "on_bundle_deploy": { + "description": "If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Lifecycle": { "oneOf": [ { @@ -15159,6 +15199,20 @@ } ] }, + "resources.JobRunTrigger": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.JobRunTrigger" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.MlflowExperimentPermission": { "oneOf": [ {