Add job_runs lifecycle.triggers.on_value_change

.nextchanges/bundles/job-runs-on-value-change.md

@@ -0,0 +1 @@
+direct: `resources.job_runs` can set `lifecycle.triggers.on_value_change` to an interpolated expression (for example `${resources.jobs.foo.id}`) to re-fire the run when that value changes. Removing the trigger does not recreate the existing run.

acceptance/bundle/refschema/out.fields.txt

@@ -874,9 +874,12 @@ resources.job_runs.*.lifecycle.triggers	dresources.JobRunTriggersState	STATE
 resources.job_runs.*.lifecycle.triggers.on_bundle_deploy	string	STATE
 resources.job_runs.*.lifecycle.triggers.on_file_change	map[string]string	STATE
 resources.job_runs.*.lifecycle.triggers.on_file_change.*	string	STATE
+resources.job_runs.*.lifecycle.triggers.on_value_change	map[string]string	STATE
+resources.job_runs.*.lifecycle.triggers.on_value_change.*	string	STATE
 resources.job_runs.*.lifecycle.triggers[*]	resources.JobRunTrigger	INPUT
 resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy	*bool	INPUT
 resources.job_runs.*.lifecycle.triggers[*].on_file_change	*string	INPUT
+resources.job_runs.*.lifecycle.triggers[*].on_value_change	*string	INPUT
 resources.job_runs.*.modified_status	string	INPUT
 resources.job_runs.*.notebook_params	map[string]string	ALL
 resources.job_runs.*.notebook_params.*	string	ALL

acceptance/bundle/resources/job_runs/on_value_change/databricks.yml

@@ -0,0 +1,25 @@
+bundle:
+  name: job-runs-on-value-change
+
+variables:
+  stamp:
+    default: v1
+  other:
+    default: a1
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+  job_runs:
+    my_run:
+      job_id: ${resources.jobs.my_job.id}
+      lifecycle:
+        triggers:
+          - on_value_change: ${var.stamp}
+          - on_value_change: ${var.other}

acceptance/bundle/resources/job_runs/on_value_change/output.txt

@@ -0,0 +1,119 @@
+
+=== first deploy triggers a run
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files...
+Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]
+Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS
+Created job_runs.my_run
+Created jobs.my_job
+Files: 4 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> read_id.py my_job
+[MY_JOB_ID]
+
+>>> print_requests.py //jobs/run-now
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "idempotency_token": "[UUID]",
+    "job_id": [MY_JOB_ID]
+  }
+}
+
+=== redeploy with unchanged values plans nothing
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files...
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 2 unchanged
+
+>>> print_requests.py //jobs/run-now
+
+=== changing one value re-fires
+>>> [CLI] bundle plan
+recreate job_runs.my_run
+
+Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged
+
+>>> changes
+{
+  "lifecycle.triggers.on_value_change": {
+    "action": "recreate",
+    "reason": "immutable",
+    "old": {
+      "a1": "a1",
+      "v1": "v1"
+    },
+    "new": {
+      "a1": "a1",
+      "v2": "v2"
+    }
+  },
+  "lifecycle.triggers.on_value_change['v1']": {
+    "action": "recreate",
+    "reason": "immutable",
+    "old": "v1"
+  },
+  "lifecycle.triggers.on_value_change['v2']": {
+    "action": "recreate",
+    "reason": "immutable",
+    "new": "v2"
+  },
+  "result_state": {
+    "action": "skip",
+    "reason": "remote_already_set",
+    "new": "SUCCESS",
+    "remote": "SUCCESS"
+  }
+}
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files...
+Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID]
+Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS
+Recreated job_runs.my_run
+Files: 3 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 1 deleted, 1 unchanged
+
+>>> print_requests.py --keep //jobs/runs/delete
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/runs/delete",
+  "body": {
+    "run_id": [MY_RUN_ID]
+  }
+}
+
+>>> print_requests.py //jobs/run-now
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "idempotency_token": "[UUID]",
+    "job_id": [MY_JOB_ID]
+  }
+}
+
+=== removing on_value_change is unchanged and does not run
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files...
+Files: 3 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 2 unchanged
+
+>>> print_requests.py //jobs/run-now
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.job_runs.my_run
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default
+
+Destroy: 2 deleted

acceptance/bundle/resources/job_runs/on_value_change/script

@@ -0,0 +1,39 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+changes() {
+    $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes'
+}
+
+title "first deploy triggers a run"
+trace $CLI bundle deploy
+trace read_id.py my_job
+read_id.py my_run > /dev/null
+trace print_requests.py //jobs/run-now
+
+title "redeploy with unchanged values plans nothing"
+trace $CLI bundle plan
+trace $CLI bundle deploy
+trace print_requests.py //jobs/run-now
+
+title "changing one value re-fires"
+update_file.py databricks.yml "default: v1" "default: v2"
+trace $CLI bundle plan
+trace changes
+trace $CLI bundle deploy
+read_id.py my_run > /dev/null
+trace print_requests.py --keep //jobs/runs/delete
+trace print_requests.py //jobs/run-now
+
+title "removing on_value_change is unchanged and does not run"
+update_file.py databricks.yml '      lifecycle:
+        triggers:
+          - on_value_change: ${var.stamp}
+          - on_value_change: ${var.other}
+' ''
+trace $CLI bundle plan
+trace $CLI bundle deploy
+trace print_requests.py //jobs/run-now

acceptance/bundle/resources/job_runs/on_value_change_ref/databricks.yml

@@ -0,0 +1,32 @@
+bundle:
+  name: job-runs-on-value-change-ref
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+    other:
+      name: other-job
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+    extra:
+      name: extra-job
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+  job_runs:
+    my_run:
+      job_id: ${resources.jobs.my_job.id}
+      lifecycle:
+        triggers:
+          - on_value_change: ${resources.jobs.other.id}-${resources.jobs.extra.id}

acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt

@@ -0,0 +1,76 @@
+
+=== plan depends on both jobs in the expression
+>>> jq .plan["resources.job_runs.my_run"].depends_on tmp.plan.json
+[
+  {
+    "node": "resources.jobs.extra",
+    "label": "${resources.jobs.extra.id}"
+  },
+  {
+    "node": "resources.jobs.my_job",
+    "label": "${resources.jobs.my_job.id}"
+  },
+  {
+    "node": "resources.jobs.other",
+    "label": "${resources.jobs.other.id}"
+  }
+]
+
+=== first deploy triggers a run
+=== bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default/files...
+Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]
+Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS
+Created job_runs.my_run
+Created jobs.extra
+Created jobs.my_job
+Created jobs.other
+Files: 6 uploaded, 0 deleted
+Resources: 4 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> read_id.py my_job
+[MY_JOB_ID]
+
+>>> read_id.py other
+[OTHER_ID]
+
+>>> read_id.py extra
+[EXTRA_ID]
+
+>>> print_requests.py //jobs/run-now
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "idempotency_token": "[UUID]",
+    "job_id": [MY_JOB_ID]
+  }
+}
+
+=== state records the concatenated resolved ids
+>>> print_state.py
+{
+  "${resources.jobs.other.id}-${resources.jobs.extra.id}": "[OTHER_ID]-[EXTRA_ID]"
+}
+
+=== redeploy with unchanged ids plans nothing
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default/files...
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 4 unchanged
+
+>>> print_requests.py //jobs/run-now
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.job_runs.my_run
+  delete resources.jobs.extra
+  delete resources.jobs.my_job
+  delete resources.jobs.other
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default
+
+Destroy: 4 deleted

acceptance/bundle/resources/job_runs/on_value_change_ref/script

@@ -0,0 +1,29 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+title "plan depends on both jobs in the expression"
+$CLI bundle plan -o json > tmp.plan.json
+trace jq '.plan["resources.job_runs.my_run"].depends_on' tmp.plan.json | contains.py \
+    'resources.jobs.other' '${resources.jobs.other.id}' \
+    'resources.jobs.extra' '${resources.jobs.extra.id}'
+
+title "first deploy triggers a run"
+# Not traced: readplanarg makes the command line differ per READPLAN variant.
+title "bundle deploy\n"
+$CLI bundle deploy $(readplanarg tmp.plan.json)
+trace read_id.py my_job
+trace read_id.py other
+trace read_id.py extra
+read_id.py my_run > /dev/null
+trace print_requests.py //jobs/run-now
+
+title "state records the concatenated resolved ids"
+trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle.triggers.on_value_change'
+
+title "redeploy with unchanged ids plans nothing"
+trace $CLI bundle plan
+trace $CLI bundle deploy
+trace print_requests.py //jobs/run-now

acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml

@@ -0,0 +1,5 @@
+# Deploy both by re-planning and from a plan saved on disk, so resource refs
+# in on_value_change survive plan serialization.
+EnvMatrix.READPLAN = ["", "1"]
+
+Ignore = ["tmp.plan.json"]