From c1e170d4071706cf662137dfab591b06d9c61fc5 Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Wed, 30 Sep 2026 12:16:08 -0600
Subject: [PATCH 01/13] Updating workflows and server clear for two always-on
 servers

---
 .github/scripts/run-ssh-local.sh              |  6 +++++-
 .github/scripts/setup-ssh-local.sh            |  5 +++++
 .../adhoc-auto-remote-benchmarks.yml          | 19 +++----------------
 .../adhoc-exist-remote-benchmarks.yml         | 16 +---------------
 .../workflows/compare-remote-benchmarks.yml   |  3 ++-
 .../matrix-exist-remote-benchmarks.yml        | 19 +++----------------
 .../workflows/nightly-remote-benchmarks.yml   |  1 +
 .../workflows/release-remote-benchmarks.yml   |  3 ++-
 .github/workflows/remote-benchmarks.yml       | 17 +++++++++++++++--
 docs/ForkSetup.md                             |  3 ++-
 10 files changed, 39 insertions(+), 53 deletions(-)

diff --git a/.github/scripts/run-ssh-local.sh b/.github/scripts/run-ssh-local.sh
index 54dee93d..2c7ec3f7 100755
--- a/.github/scripts/run-ssh-local.sh
+++ b/.github/scripts/run-ssh-local.sh
@@ -41,4 +41,8 @@ for ((attempt=1; attempt<=MAX_RETRIES; attempt++)); do
   fi
 done
 
-ssh -o 'ConnectTimeout 10' -o 'ServerAliveInterval 60' ${USER}@${HOST} 'bash -s' -- "${args[@]}" < ${SCRIPT_DIR}/${SCRIPT_NAME}.sh |& tee logs/${SCRIPT_NAME}.log
+# Keep the server address out of the log file, which is uploaded as a public artifact. Github masks
+# secrets in the live log stream but not in files, and ssh names the host in its connection errors.
+HOST_PATTERN=$(printf '%s' "${HOST}" | sed 's/[^a-zA-Z0-9-]/\\&/g')
+
+ssh -o 'ConnectTimeout 10' -o 'ServerAliveInterval 60' ${USER}@${HOST} 'bash -s' -- "${args[@]}" < ${SCRIPT_DIR}/${SCRIPT_NAME}.sh |& sed -u "s/${HOST_PATTERN}/<redacted-host>/g" | tee logs/${SCRIPT_NAME}.log
diff --git a/.github/scripts/setup-ssh-local.sh b/.github/scripts/setup-ssh-local.sh
index d5abb504..06634f04 100644
--- a/.github/scripts/setup-ssh-local.sh
+++ b/.github/scripts/setup-ssh-local.sh
@@ -17,6 +17,11 @@ if [[ $# != 2 ]]; then
   exit 1
 fi
 
+if [[ "${HOST}" == "<"*">" ]]; then
+  echo "$0: Host '${HOST}' was not resolved. Is the matching BENCHMARK_HOST_* secret set?"
+  exit 1
+fi
+
 mkdir -p logs
 mkdir -p results
 mkdir -p ~/.ssh/
diff --git a/.github/workflows/adhoc-auto-remote-benchmarks.yml b/.github/workflows/adhoc-auto-remote-benchmarks.yml
index 5febda41..33717c9e 100644
--- a/.github/workflows/adhoc-auto-remote-benchmarks.yml
+++ b/.github/workflows/adhoc-auto-remote-benchmarks.yml
@@ -111,26 +111,12 @@ jobs:
       scale_row_count: ${{ needs.setup-benchmark-system.outputs.test_row_count }}
       distribution: ${{ inputs.distribution }}
       test_device_addr: ${{ needs.setup-benchmark-system.outputs.metal_ip_addr }}
+      clear_server: true
       config_options: "<default>"
     secrets: inherit
 
-  clear-between-runs:
-    needs: [setup-benchmark-system, process-adhoc-benchmarks-1]
-    runs-on: ubuntu-24.04
-    env:
-      SD: .github/scripts
-      HOST: ${{ needs.setup-benchmark-system.outputs.metal_ip_addr }}
-      USER: ${{ secrets.BENCHMARK_USER }}
-    steps:
-    - uses: actions/checkout@v4
-    - name: Clear Remote Test Server
-      run: |
-        sudo chmod +x ${SD}/*
-        ${SD}/setup-ssh-local.sh ${HOST} "${{ secrets.BENCHMARK_KEY }}"
-        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} clear-test-server-remote
-
   process-adhoc-benchmarks-2:
-    needs: [setup-benchmark-system, clear-between-runs]
+    needs: [setup-benchmark-system, process-adhoc-benchmarks-1]
     uses: ./.github/workflows/remote-benchmarks.yml
     with:
       run_type: adhoc
@@ -142,6 +128,7 @@ jobs:
       scale_row_count: ${{ needs.setup-benchmark-system.outputs.test_row_count }}
       distribution: ${{ inputs.distribution }}
       test_device_addr: ${{ needs.setup-benchmark-system.outputs.metal_ip_addr }}
+      clear_server: true
       config_options: "<default>"
     secrets: inherit
 
diff --git a/.github/workflows/adhoc-exist-remote-benchmarks.yml b/.github/workflows/adhoc-exist-remote-benchmarks.yml
index 6bd6510e..fa122711 100644
--- a/.github/workflows/adhoc-exist-remote-benchmarks.yml
+++ b/.github/workflows/adhoc-exist-remote-benchmarks.yml
@@ -55,22 +55,7 @@ on:
         - runlength
 
 jobs:
-  clear-benchmark-system:
-    runs-on: ubuntu-24.04
-    env:
-      SD: .github/scripts
-      HOST: ${{ secrets.BENCHMARK_HOST }}
-      USER: ${{ secrets.BENCHMARK_USER }}
-    steps:
-    - uses: actions/checkout@v4
-    - name: Clear Remote Test Server
-      run: |
-        sudo chmod +x ${SD}/*
-        ${SD}/setup-ssh-local.sh ${HOST} "${{ secrets.BENCHMARK_KEY }}"
-        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} clear-test-server-remote
-
   setup-benchmark-system:
-    needs: clear-benchmark-system
     runs-on: ubuntu-24.04
     outputs:
       test_row_count: ${{ steps.scale-nums.outputs.TEST_ROW_COUNT }}
@@ -99,6 +84,7 @@ jobs:
       scale_row_count: ${{ needs.setup-benchmark-system.outputs.test_row_count }}
       distribution: ${{ inputs.distribution }}
       test_device_addr: "<default>"
+      clear_server: true
       config_options: "${{ inputs.config_options }}"
       
     secrets: inherit
diff --git a/.github/workflows/compare-remote-benchmarks.yml b/.github/workflows/compare-remote-benchmarks.yml
index 0bd95d6d..8e139319 100644
--- a/.github/workflows/compare-remote-benchmarks.yml
+++ b/.github/workflows/compare-remote-benchmarks.yml
@@ -28,6 +28,7 @@ jobs:
       test_iterations: 5
       scale_row_count: 70000000
       distribution: random
-      test_device_addr: "<default>"
+      test_device_addr: "<matrix>"
+      clear_server: true
       config_options: "<default>"
     secrets: inherit
diff --git a/.github/workflows/matrix-exist-remote-benchmarks.yml b/.github/workflows/matrix-exist-remote-benchmarks.yml
index 24814a00..7b4c69c7 100644
--- a/.github/workflows/matrix-exist-remote-benchmarks.yml
+++ b/.github/workflows/matrix-exist-remote-benchmarks.yml
@@ -33,22 +33,8 @@ jobs:
       run: |
         .github/scripts/build-matrix.sh ".github/matrix/${{ inputs.matrix_file }}"
 
-  clear-server:
-    runs-on: ubuntu-24.04
-    env:
-      SD: .github/scripts
-      HOST: ${{ secrets.BENCHMARK_HOST }}
-      USER: ${{ secrets.BENCHMARK_USER }}
-    steps:
-    - uses: actions/checkout@v4
-    - name: Clear Remote Server
-      run: |
-        sudo chmod +x ${SD}/*
-        ${SD}/setup-ssh-local.sh ${HOST} "${{ secrets.BENCHMARK_KEY }}"
-        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} clear-test-server-remote
-
   run-benchmarks:
-    needs: [generate-matrix, clear-server]
+    needs: generate-matrix
     strategy:
       fail-fast: false
       max-parallel: 1
@@ -64,6 +50,7 @@ jobs:
       test_iterations: ${{ matrix.test_iterations }}
       scale_row_count: ${{ matrix.scale_row_count }}
       distribution: ${{ matrix.distribution }}
-      test_device_addr: "<default>"
+      test_device_addr: "<matrix>"
+      clear_server: ${{ strategy.job-index == 0 }}
       config_options: ${{ matrix.config_options }}
     secrets: inherit
diff --git a/.github/workflows/nightly-remote-benchmarks.yml b/.github/workflows/nightly-remote-benchmarks.yml
index ba59791d..b0975309 100644
--- a/.github/workflows/nightly-remote-benchmarks.yml
+++ b/.github/workflows/nightly-remote-benchmarks.yml
@@ -24,5 +24,6 @@ jobs:
       scale_row_count: 10000000
       distribution: random
       test_device_addr: "<default>"
+      clear_server: true
       config_options: "<default>"
     secrets: inherit
diff --git a/.github/workflows/release-remote-benchmarks.yml b/.github/workflows/release-remote-benchmarks.yml
index abf16dd2..78a47006 100644
--- a/.github/workflows/release-remote-benchmarks.yml
+++ b/.github/workflows/release-remote-benchmarks.yml
@@ -38,6 +38,7 @@ jobs:
       test_iterations: 5
       scale_row_count: 10000000
       distribution: random
-      test_device_addr: "<default>"
+      test_device_addr: "<matrix>"
+      clear_server: true
       config_options: "<default>"
     secrets: inherit
diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index d6a615d2..5544496e 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -4,6 +4,11 @@
 # - This workflow is reusable with run_type set to release or nightly
 # - Scripts ending in "-local" run on the github runner
 # - Scripts ending in "-remote" run are executed on the remote test server through ssh
+# - test_device_addr picks the server: <default> is BENCHMARK_HOST_NIGHT (adhoc-existing, nightly),
+#   <matrix> is BENCHMARK_HOST_MATRIX (compare, release, matrix), anything else is used as an ip
+# - Since the concurrency group is test_device_addr, the two always-on servers do not block each other
+# - Clearing a server must happen here rather than in a calling workflow, since only this workflow holds
+#   the concurrency group. A caller that clears on its own can wipe a run already in progress.
 
 name: Benchmark Test on Docker Deephaven
 
@@ -35,14 +40,22 @@ on:
         required: true
         type: string
       test_device_addr:
+        description: 'Server ip, or <default> for the nightly host, or <matrix> for the matrix host'
         required: true
         type: string
+      clear_server:
+        description: 'Wipe artifacts, containers, images, and data before setup'
+        required: true
+        type: boolean
       config_options:
         required: true
         type: string
         
 env:
-  HOST: "${{ inputs.test_device_addr == '<default>' && secrets.BENCHMARK_HOST || inputs.test_device_addr }}"
+  HOST: >-
+    ${{ inputs.test_device_addr == '<default>' && secrets.BENCHMARK_HOST_NIGHT
+    || inputs.test_device_addr == '<matrix>' && secrets.BENCHMARK_HOST_MATRIX
+    || inputs.test_device_addr }}
 
 concurrency:
   group: ${{ inputs.test_device_addr }}
@@ -77,7 +90,7 @@ jobs:
         ${SD}/setup-ssh-local.sh ${HOST} "${{secrets.BENCHMARK_KEY}}"
 
     - name: Clear Remote Test Server
-      if: ${{ env.RUN_TYPE != 'adhoc' }}
+      if: ${{ inputs.clear_server }}
       run: |
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} clear-test-server-remote
 
diff --git a/docs/ForkSetup.md b/docs/ForkSetup.md
index b566b26d..e979dc3c 100644
--- a/docs/ForkSetup.md
+++ b/docs/ForkSetup.md
@@ -15,7 +15,8 @@ Not all secrets are required for each workflow. In the following table, each sec
 | Variable                   | Description                                                           | Shared (Ask Admin) | Optional |
 | -------------------------- | --------------------------------------------------------------------- | ------------------ | -------- |
 | BENCHMARK_GCLOUD           | The GCloud Service Account Key (credentials) in JSON format           | Yes                | No       |
-| BENCHMARK_HOST             | The host ip for an existing (non-auto-provisioned) server             | No                 | No       |
+| BENCHMARK_HOST_NIGHT       | Host ip of the always-on server used by adhoc-existing and nightly    | No                 | No       |
+| BENCHMARK_HOST_MATRIX      | Host ip of the always-on server used by compare, release, and matrix  | No                 | No       |
 | BENCHMARK_USER             | The "run as" user for running benchmarks on the server                | No                 | No       |
 | BENCHMARK_KEY              | A private key used by SSH corresponding to a public key on the server | No                 | No       |
 | BENCHMARK_METAL_AUTH_TOKEN | The key/secret required to access the bare metal provider API         | Yes                | No       |

From 638cbe7900b68c8ce6e54a5761b36f9eefe298ec Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Thu, 1 Oct 2026 12:14:14 -0600
Subject: [PATCH 02/13] Reworked Release runs to use matrix

---
 .github/matrix/example-regression-matrix.sh   |  5 +-
 .github/matrix/gc-train-1000-100.sh           | 10 ++--
 .github/matrix/gc-train-heap-fail.sh          |  5 +-
 .github/matrix/release.sh                     | 39 +++++++++++++
 .github/scripts/build-matrix.sh               | 22 +++++++-
 .github/scripts/fetch-results-local.sh        | 35 +++++++-----
 .github/scripts/run-ssh-local.sh              |  3 +-
 .../matrix-exist-remote-benchmarks.yml        |  2 +-
 .../workflows/release-remote-benchmarks.yml   | 56 +++++++++++++------
 .github/workflows/remote-benchmarks.yml       |  9 +--
 10 files changed, 136 insertions(+), 50 deletions(-)
 create mode 100755 .github/matrix/release.sh

diff --git a/.github/matrix/example-regression-matrix.sh b/.github/matrix/example-regression-matrix.sh
index 73062996..1640d97b 100755
--- a/.github/matrix/example-regression-matrix.sh
+++ b/.github/matrix/example-regression-matrix.sh
@@ -29,10 +29,11 @@
 # Class lists omit the Test suffix: ConsoleLauncherUtil appends it, so 'UserFormulaTest' matches
 # nothing and the run reports zero benchmarks rather than failing.
 #
-# TSV columns: run_label, docker_image, test_package, test_class_list,
+# TSV columns: run_type, run_label, docker_image, test_package, test_class_list,
 #              test_iterations, scale_row_count, distribution, config_options
 EXPECTED_COMBOS=5
 
+RTYP='adhoc'
 PKG=io.deephaven.benchmark.tests.standard
 CLASSES='UserFormula'
 DIST=random
@@ -49,7 +50,7 @@ jfr_opts() {
 TOP=deephaven:c50cd7f2fc02b819376a80e8767dff76c0624aa2
 
 # $3 is config_options, defaulting to a JFR recording named after the row
-row() { echo -e "$1\t$2\t$PKG\t$CLASSES\t$ITERS\t$ROWSM\t$DIST\t${3:-$(jfr_opts "$1")}"; }
+row() { echo -e "$RTYP\t$1\t$2\t$PKG\t$CLASSES\t$ITERS\t$ROWSM\t$DIST\t${3:-$(jfr_opts "$1")}"; }
 
 row anchor_img_0911 "$ANCHOR"
 row src_base deephaven:9fa9e83c55ccb2e9ddedd61d1ba59d2d8da68c85
diff --git a/.github/matrix/gc-train-1000-100.sh b/.github/matrix/gc-train-1000-100.sh
index 34e3c13f..d7d254d3 100755
--- a/.github/matrix/gc-train-1000-100.sh
+++ b/.github/matrix/gc-train-1000-100.sh
@@ -1,8 +1,9 @@
 # Compare GC configurations across cycle times and profiles
-# TSV columns: run_label, docker_image, test_package, test_class_list,
+# TSV columns: run_type, run_label, docker_image, test_package, test_class_list,
 #              test_iterations, scale_row_count, distribution, config_options
 EXPECTED_COMBOS=32
 
+RTYP='adhoc'
 IMG=ghcr.io/stanbrub/server:jvm25
 PKG=io.deephaven.benchmark.tests.train
 CLS='*Train'
@@ -23,11 +24,12 @@ R="$IMG\t$PKG\t$CLS\t$ITERS\t$ROWSM\t$DIST"
 # emit <label> <gc> <cycle_ms> <inc_load_target> — outputs regular and _huge variants
 run() {
   local opts="$2 $BASE -DPeriodicUpdateGraph.targetCycleDurationMillis=$3 -Dbench.incLoadTarget=$4"
-  echo -e "$1\t$R\t$opts"
-  echo -e "${1}_huge\t$R\t$opts $HUGE"
+  echo -e "$RTYP\t$1\t$R\t$opts"
+  echo -e "$RTYP\t${1}_huge\t$R\t$opts $HUGE"
 }
 
-# 1000ms cycle
+# 1000ms cycle/adhoc
+
 run gc_g1gc_1000_p100_j25  "$G1"   1000  1.00
 run gc_zgc_1000_p100_j25   "$ZGC"  1000  1.00
 run gc_shen_1000_p100_j25  "$SHEN" 1000  1.00
diff --git a/.github/matrix/gc-train-heap-fail.sh b/.github/matrix/gc-train-heap-fail.sh
index 144f806f..1431c6c4 100755
--- a/.github/matrix/gc-train-heap-fail.sh
+++ b/.github/matrix/gc-train-heap-fail.sh
@@ -1,7 +1,8 @@
 # Sweep heap size to find failure points per GC and test class
-# TSV columns: run_label, docker_image, test_package, test_class_list,
+# TSV columns: run_type, run_label, docker_image, test_package, test_class_list,
 #              test_iterations, scale_row_count, distribution, config_options
 
+RTYP='adhoc'
 IMG=ghcr.io/stanbrub/server:jvm25
 PKG=io.deephaven.benchmark.tests.train
 ITERS=1
@@ -51,7 +52,7 @@ for combo in "${COMBOS[@]}"; do
   read -r cls gc min_heap max_heap <<< "$combo"
   for h in $(seq ${max_heap} -${STEP} ${min_heap}); do
     opts="${GCS[$gc]} -Xms${h}g -Xmx${h}g $BASE $CYCLE"
-    echo -e "gc_${gc}_${cls}_h${h}_j25\t$IMG\t$PKG\t*${cls}\t$ITERS\t$ROWSM\t$DIST\t$opts"
+    echo -e "$RTYP\tgc_${gc}_${cls}_h${h}_j25\t$IMG\t$PKG\t*${cls}\t$ITERS\t$ROWSM\t$DIST\t$opts"
   done
 done
 
diff --git a/.github/matrix/release.sh b/.github/matrix/release.sh
new file mode 100755
index 00000000..a4b305f3
--- /dev/null
+++ b/.github/matrix/release.sh
@@ -0,0 +1,39 @@
+# Release benchmarks: one full pass per row, so every benchmark gets the same sample count
+#
+# Each row is a complete pass over the standard suite and publishes its results before the next row
+# starts. That matters for a run this long: iterations inside a single row would upload nothing until
+# the end, and a failure part way through would lose the whole run. Rows are independent, so a failed
+# row costs one pass instead of all of them.
+#
+# Row count is the sample count. With test_iterations at 1, a release row runs as two jobs, !Iterate
+# then Iterate, which together cover the suite exactly once. So 7 rows means n=7 for all 666
+# benchmarks, where the nightly scheme gives n=1 to the 562 untagged ones and n=5 to the 104 tagged.
+#
+# Keep the row count odd. Queries report the middle run by rate and carry that run's id, so an even
+# count has no real middle and the reported numbers stop matching a run that happened.
+#
+# All rows share the run_label, so they land in one set and accumulate. <version> takes the label from
+# the version the engine reports, so edge becomes a 99 in the last position and cannot collide with
+# the release of the same major.minor.
+#
+# Budget roughly 4 hours per row of test time, plus setup, engine restarts and data generation.
+#
+# TSV columns: run_type, run_label, docker_image, test_package, test_class_list,
+#              test_iterations, scale_row_count, distribution, config_options
+EXPECTED_COMBOS=7
+
+RTYP='release'
+LABEL='<version>'
+IMG=${IMG:-edge}
+PKG=io.deephaven.benchmark.tests.standard
+CLS='*'
+ITERS=1
+ROWSM=10
+DIST=random
+OPTS='<default>'
+
+PASSES=7
+
+for i in $(seq ${PASSES}); do
+  echo -e "$RTYP\t$LABEL\t$IMG\t$PKG\t$CLS\t$ITERS\t$ROWSM\t$DIST\t$OPTS"
+done
diff --git a/.github/scripts/build-matrix.sh b/.github/scripts/build-matrix.sh
index c140c009..c023de63 100755
--- a/.github/scripts/build-matrix.sh
+++ b/.github/scripts/build-matrix.sh
@@ -7,8 +7,9 @@ set -o pipefail
 
 # Runs a matrix script and converts its TSV output to a JSON matrix for GitHub Actions.
 # The matrix script outputs tab-separated rows with fields:
-#   run_label, docker_image, test_package, test_class_list,
+#   run_type, run_label, docker_image, test_package, test_class_list,
 #   test_iterations, scale_row_count, distribution, config_options
+# Every field is required, so a new field means updating every matrix script.
 # Scale_row_count is in millions and auto-scaled to actual.
 # Test_iterations is auto-forced to odd.
 #
@@ -28,8 +29,25 @@ fi
 
 TSV=$(bash "$FILE")
 
+BAD_FIELDS=$(echo "$TSV" | awk -F'\t' 'NF>0 && NF!=9 {print "  line "NR": "NF" fields"}')
+if [[ -n "${BAD_FIELDS}" ]]; then
+  echo "::error::Every row needs 9 tab-separated fields in ${FILE}"; echo "${BAD_FIELDS}"; exit 1
+fi
+
+BAD_TYPES=$(echo "$TSV" | awk -F'\t' 'NF>0 && $1 !~ /^(adhoc|release|compare)$/ {print "  line "NR": "$1}')
+if [[ -n "${BAD_TYPES}" ]]; then
+  echo "::error::run_type must be adhoc, release, or compare in ${FILE}"; echo "${BAD_TYPES}"; exit 1
+fi
+
+# Rows sharing a run_type and run_label land in one set, and queries report the middle run by rate
+BAD_PARITY=$(echo "$TSV" | awk -F'\t' 'NF>0 {c[$1"/"$2]++} END {for (k in c) if (c[k]>1 && c[k]%2==0) print "  "k": "c[k]" rows"}')
+if [[ -n "${BAD_PARITY}" ]]; then
+  echo "::error::Rows per set must be odd so the reported median is a real run. Add or remove a row in ${FILE}"
+  echo "${BAD_PARITY}"; exit 1
+fi
+
 RESULT=$(echo "$TSV" | jq -Rsc '
-  ["run_label","docker_image","test_package","test_class_list",
+  ["run_type","run_label","docker_image","test_package","test_class_list",
    "test_iterations","scale_row_count","distribution","config_options"] as $h |
   split("\n") | map(select(length > 0)) |
   [.[] | split("\t") | [range(length) as $i | {($h[$i]): .[$i]}] | add |
diff --git a/.github/scripts/fetch-results-local.sh b/.github/scripts/fetch-results-local.sh
index 5a6a65ba..c8902066 100755
--- a/.github/scripts/fetch-results-local.sh
+++ b/.github/scripts/fetch-results-local.sh
@@ -10,8 +10,8 @@ set -o nounset
 # compresses the runs before upload. Writes an output file with the 
 # SET_LABEL that was used for the set directory name
 
-if [[ $# != 7 ]]; then
-  echo "$0: Missing host, user, run type, script dir, actor, docker img, or run label arguments"
+if [[ $# != 6 ]]; then
+  echo "$0: Missing host, user, script dir, run type, actor, or run label arguments"
   exit 1
 fi
 
@@ -21,32 +21,41 @@ SCRIPT_DIR=$3
 RUN_TYPE=$4
 ACTOR=$5
 SET_LABEL=${6:-$(echo -n "set-"; ${SCRIPT_DIR}/base.sh $(date +%s%03N) 62)}
-DOCKER_IMG=$7
 RUN_DIR=/home/${USER}/run
 OUTPUT_NAME=fetch-results-local.out
 
 rm -f ${OUTPUT_NAME}; touch ${OUTPUT_NAME}
 
+# Pull results from the benchmark server (Before labelling, since <version> reads the platform csv)
+scp -r ${USER}@${HOST}:${RUN_DIR}/results .
+scp -r ${USER}@${HOST}:${RUN_DIR}/logs .
+scp -r ${USER}@${HOST}:${RUN_DIR}/*.jar .
+
 # Get the date for the Set Label, since Github Workflows don't have 'with: ${{github.date}}'
 if [ "${SET_LABEL}" = "<date>" ]; then
   SET_LABEL=$(date '+%Y-%m-%d')
 fi
 
-# Get the version for the Set Label, since Github Workflows don't have 'with: ${{github.date}}'
+# Get the Set Label from the engine-reported version (edge has none), 99 in last place for snapshots
 if [ "${SET_LABEL}" = "<version>" ]; then
-  vers=${DOCKER_IMG}
-  major=$(printf '%02d\n' $(echo ${vers} | cut -d "." -f 1))
-  minor=$(printf '%03d\n' $(echo ${vers} | cut -d "." -f 2))
-  patch=$(printf '%02d\n' $(echo ${vers} | cut -d "." -f 3))
+  PLATFORM_CSV=results/platform-summary-results.csv
+  # Match the engine only. The test-runner reports Unknown unless run from a release jar
+  vers=$(awk -F, '$2=="deephaven-engine" && $3=="deephaven.version" {print $4; exit}' ${PLATFORM_CSV})
+  # Empty would pad to a valid-looking 00.000.00, so fail instead
+  : "${vers:?no deephaven-engine deephaven.version found in ${PLATFORM_CSV}}"
+  base=${vers%-SNAPSHOT}
+  major=$(printf '%02d\n' $(echo ${base} | cut -d "." -f 1))
+  minor=$(printf '%03d\n' $(echo ${base} | cut -d "." -f 2))
+  if [ "${base}" = "${vers}" ]; then
+    patch=$(printf '%02d\n' $(echo ${base} | cut -d "." -f 3))
+  else
+    patch=99
+  fi
   SET_LABEL="${major}.${minor}.${patch}"
+  echo "Engine reported ${vers}, using set label ${SET_LABEL}"
 fi
 echo "SET_LABEL=${SET_LABEL}" | tee -a ${OUTPUT_NAME}
 
-# Pull results from the benchmark server
-scp -r ${USER}@${HOST}:${RUN_DIR}/results .
-scp -r ${USER}@${HOST}:${RUN_DIR}/logs .
-scp -r ${USER}@${HOST}:${RUN_DIR}/*.jar .
-
 # Move the results into the destination directory
 DEST_DIR=${RUN_TYPE}/${ACTOR}/${SET_LABEL}
 mkdir -p ${DEST_DIR}
diff --git a/.github/scripts/run-ssh-local.sh b/.github/scripts/run-ssh-local.sh
index 2c7ec3f7..a346c08c 100755
--- a/.github/scripts/run-ssh-local.sh
+++ b/.github/scripts/run-ssh-local.sh
@@ -41,8 +41,7 @@ for ((attempt=1; attempt<=MAX_RETRIES; attempt++)); do
   fi
 done
 
-# Keep the server address out of the log file, which is uploaded as a public artifact. Github masks
-# secrets in the live log stream but not in files, and ssh names the host in its connection errors.
+# Redact the host from the log file, since artifacts are public and Github masks only the live log
 HOST_PATTERN=$(printf '%s' "${HOST}" | sed 's/[^a-zA-Z0-9-]/\\&/g')
 
 ssh -o 'ConnectTimeout 10' -o 'ServerAliveInterval 60' ${USER}@${HOST} 'bash -s' -- "${args[@]}" < ${SCRIPT_DIR}/${SCRIPT_NAME}.sh |& sed -u "s/${HOST_PATTERN}/<redacted-host>/g" | tee logs/${SCRIPT_NAME}.log
diff --git a/.github/workflows/matrix-exist-remote-benchmarks.yml b/.github/workflows/matrix-exist-remote-benchmarks.yml
index 7b4c69c7..ed1e5099 100644
--- a/.github/workflows/matrix-exist-remote-benchmarks.yml
+++ b/.github/workflows/matrix-exist-remote-benchmarks.yml
@@ -42,7 +42,7 @@ jobs:
         include: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
     uses: ./.github/workflows/remote-benchmarks.yml
     with:
-      run_type: adhoc
+      run_type: ${{ matrix.run_type }}
       docker_image: ${{ matrix.docker_image }}
       run_label: ${{ matrix.run_label }}
       test_package: ${{ matrix.test_package }}
diff --git a/.github/workflows/release-remote-benchmarks.yml b/.github/workflows/release-remote-benchmarks.yml
index 78a47006..18db2e97 100644
--- a/.github/workflows/release-remote-benchmarks.yml
+++ b/.github/workflows/release-remote-benchmarks.yml
@@ -2,7 +2,8 @@
 
 # Run benchmarks on a remote system according to the release defined 
 # in resources/release-benchmark-docker-compose.yml
-# - Calls the reusable worflow remote-benchmarks.yml
+# - Runs the rows in matrix/release.sh, passing the requested image in as IMG
+# - Each row is a full pass that publishes before the next starts, so a failed row costs one pass
 
 name: Release Benchmark Test on Docker Deephaven
 
@@ -10,35 +11,54 @@ on:
   workflow_dispatch:
     inputs:
      docker_image:
-       description: 'Docker Image Name'
+       description: 'Docker Image Name (a release version, or edge for a pre-release run)'
        required: true
-       default: '0.36.0'
+       default: '42.0'
        type: string
 
 jobs:
-  validate-inputs:
+  generate-matrix:
     runs-on: ubuntu-24.04
+    outputs:
+      matrix: ${{ steps.build-matrix.outputs.matrix }}
     steps:
+    - uses: actions/checkout@v4
+
     - name: Validate Docker Image Version
       run: |
-        if ! [[ "${{ inputs.docker_image }}" =~ ^[0-9]+(\.[0-9]+){1,2}$ ]]; then
-          echo "::error::docker_image '${{ inputs.docker_image }}' is not a valid version (expected numeric format like 0.36.0 or 42.4)"
+        IMG="${{ inputs.docker_image }}"
+        if [[ "${IMG}" == "edge" ]]; then
+          echo "Pre-release run. The set label comes from the version the engine reports"
+        elif ! [[ "${IMG}" =~ ^[0-9]+(\.[0-9]+){1,2}$ ]]; then
+          echo "::error::docker_image '${IMG}' is not a release version (expected numeric format like 0.36.0 or 42.4) or 'edge'"
           exit 1
         fi
 
-  process-release-benchmarks:
-    needs: validate-inputs
+    - name: Build Matrix Combinations
+      id: build-matrix
+      env:
+        IMG: ${{ inputs.docker_image }}
+      run: |
+        .github/scripts/build-matrix.sh ".github/matrix/release.sh"
+
+  run-benchmarks:
+    needs: generate-matrix
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        include: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
     uses: ./.github/workflows/remote-benchmarks.yml
     with:
-      run_type: release
-      docker_image: ${{ inputs.docker_image }}
-      run_label: "<version>"
-      test_package: "io.deephaven.benchmark.tests.standard"
-      test_class_list: "*"
-      test_iterations: 5
-      scale_row_count: 10000000
-      distribution: random
+      run_type: ${{ matrix.run_type }}
+      docker_image: ${{ matrix.docker_image }}
+      run_label: ${{ matrix.run_label }}
+      test_package: ${{ matrix.test_package }}
+      test_class_list: ${{ matrix.test_class_list }}
+      test_iterations: ${{ matrix.test_iterations }}
+      scale_row_count: ${{ matrix.scale_row_count }}
+      distribution: ${{ matrix.distribution }}
       test_device_addr: "<matrix>"
-      clear_server: true
-      config_options: "<default>"
+      clear_server: ${{ strategy.job-index == 0 }}
+      config_options: ${{ matrix.config_options }}
     secrets: inherit
diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index 5544496e..5968e83b 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -4,11 +4,8 @@
 # - This workflow is reusable with run_type set to release or nightly
 # - Scripts ending in "-local" run on the github runner
 # - Scripts ending in "-remote" run are executed on the remote test server through ssh
-# - test_device_addr picks the server: <default> is BENCHMARK_HOST_NIGHT (adhoc-existing, nightly),
-#   <matrix> is BENCHMARK_HOST_MATRIX (compare, release, matrix), anything else is used as an ip
-# - Since the concurrency group is test_device_addr, the two always-on servers do not block each other
-# - Clearing a server must happen here rather than in a calling workflow, since only this workflow holds
-#   the concurrency group. A caller that clears on its own can wipe a run already in progress.
+# - test_device_addr selects both the server (<default>|<matrix>|ip) and the concurrency group
+# - Clearing runs here, not in callers, so it cannot wipe a run holding the concurrency group
 
 name: Benchmark Test on Docker Deephaven
 
@@ -181,7 +178,7 @@ jobs:
     - name: Fetch Benchmark Results and Prepare for Upload
       id: fetch-results
       run: |
-        ${SD}/fetch-results-local.sh ${HOST} ${USER} ${SD} ${RUN_TYPE} "${ACTOR}" "${RUN_LABEL}" "${DOCKER_IMG}"
+        ${SD}/fetch-results-local.sh ${HOST} ${USER} ${SD} ${RUN_TYPE} "${ACTOR}" "${RUN_LABEL}"
         cat fetch-results-local.out >> "$GITHUB_OUTPUT"
 
     - name: Authorize GCloud Credentials

From 37ed71116bdf01049775be76f51c994d02ba42bc Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Thu, 1 Oct 2026 12:52:10 -0600
Subject: [PATCH 03/13] Reduce iterations/tests for testing

---
 .github/matrix/release.sh       | 15 +++++++++------
 .github/scripts/matrix_array.sh |  3 ++-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/.github/matrix/release.sh b/.github/matrix/release.sh
index a4b305f3..eb8c7a82 100755
--- a/.github/matrix/release.sh
+++ b/.github/matrix/release.sh
@@ -5,9 +5,9 @@
 # the end, and a failure part way through would lose the whole run. Rows are independent, so a failed
 # row costs one pass instead of all of them.
 #
-# Row count is the sample count. With test_iterations at 1, a release row runs as two jobs, !Iterate
-# then Iterate, which together cover the suite exactly once. So 7 rows means n=7 for all 666
-# benchmarks, where the nightly scheme gives n=1 to the 562 untagged ones and n=5 to the 104 tagged.
+# Row count is the sample count. Release ignores the Iterate tag, so a row at test_iterations 1 is a
+# single job over every class, and 7 rows means n=7 for all 666 benchmarks. Nightly keeps the tag
+# split for its time budget, which leaves its 562 untagged benchmarks at n=1 and 104 tagged at n=5.
 #
 # Keep the row count odd. Queries report the middle run by rate and carry that run's id, so an even
 # count has no real middle and the reported numbers stop matching a run that happened.
@@ -20,19 +20,22 @@
 #
 # TSV columns: run_type, run_label, docker_image, test_package, test_class_list,
 #              test_iterations, scale_row_count, distribution, config_options
-EXPECTED_COMBOS=7
+# TEMPORARY test values for verifying the workflow. A real release run uses
+# EXPECTED_COMBOS=7, CLS='*', PASSES=7. WhereTest is tagged and AvgByTest is not,
+# so both the !Iterate and Iterate jobs in a row have benchmarks to run
+EXPECTED_COMBOS=3
 
 RTYP='release'
 LABEL='<version>'
 IMG=${IMG:-edge}
 PKG=io.deephaven.benchmark.tests.standard
-CLS='*'
+CLS='Where,AvgBy'
 ITERS=1
 ROWSM=10
 DIST=random
 OPTS='<default>'
 
-PASSES=7
+PASSES=3
 
 for i in $(seq ${PASSES}); do
   echo -e "$RTYP\t$LABEL\t$IMG\t$PKG\t$CLS\t$ITERS\t$ROWSM\t$DIST\t$OPTS"
diff --git a/.github/scripts/matrix_array.sh b/.github/scripts/matrix_array.sh
index 3fade347..c6c84e9b 100755
--- a/.github/scripts/matrix_array.sh
+++ b/.github/scripts/matrix_array.sh
@@ -9,12 +9,13 @@ set -o pipefail
 # workflow with fromJSON(array)
 # ex. matrix_array.sh matrix-arr adhoc 5
 # ex. matrix_array.sh matrix-arr release 4
+# Only nightly splits on the Iterate tag, to fit a curated subset of repeats into its time budget
 
 NAME=$1
 RUN_TYPE=$2
 ITERATIONS=$3
 
-if [ "${RUN_TYPE}" = 'release' ] || [ "${RUN_TYPE}" = 'nightly' ]; then
+if [ "${RUN_TYPE}" = 'nightly' ]; then
   FIRST='!Iterate'
   TAG='Iterate'
 else

From 8cde852ebde19f36958df6d8476de163b4947756 Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Thu, 1 Oct 2026 14:16:39 -0600
Subject: [PATCH 04/13] Fix APT deadlock

---
 .github/resources/adhoc-server-cloud-init.yml | 12 +++++
 .github/scripts/adhoc.sh                      |  7 ++-
 .github/scripts/run-benchmarks-remote.sh      |  4 +-
 .github/scripts/setup-test-server-remote.sh   | 46 +++++++++++--------
 4 files changed, 47 insertions(+), 22 deletions(-)
 create mode 100644 .github/resources/adhoc-server-cloud-init.yml

diff --git a/.github/resources/adhoc-server-cloud-init.yml b/.github/resources/adhoc-server-cloud-init.yml
new file mode 100644
index 00000000..1d1f0860
--- /dev/null
+++ b/.github/resources/adhoc-server-cloud-init.yml
@@ -0,0 +1,12 @@
+#cloud-config
+
+# Keep the apt timers from firing on a freshly provisioned server. Both ship with Persistent=true, so
+# a new box treats their schedule as missed and runs them at a random offset after boot. When that
+# offset lands in the first few minutes it collides with benchmark setup and holds the dpkg lock.
+#
+# bootcmd runs in the cloud-init local stage, well before timers.target, so the timers never fire.
+# Do not add a users: block here. The server is provisioned with installDefaultSshKeys, and a users:
+# list that omits "- default" replaces the provisioned key and locks the run out of its own server.
+bootcmd:
+  - systemctl disable --now apt-daily.timer apt-daily-upgrade.timer
+  - systemctl mask apt-daily.service apt-daily-upgrade.service unattended-upgrades.service
diff --git a/.github/scripts/adhoc.sh b/.github/scripts/adhoc.sh
index fa1bf1bb..a6b87411 100755
--- a/.github/scripts/adhoc.sh
+++ b/.github/scripts/adhoc.sh
@@ -111,8 +111,11 @@ if [[ ${ACTION} == "deploy-metal" ]]; then
   TOKEN=$(getApiToken "${PROJECT_ID}" "${API_KEY}")
 
   echo "Making Deploy POST"
-  jq --arg hostname "${ACTOR}" --arg plan "${PLAN}" \
-   '.hostname = $hostname | .type = $plan' adhoc-server-deploy.json > adhoc-server-deploy-final.json
+  # The API wants cloud-init user data base64 encoded on one line, and will not echo it back
+  USER_DATA=$(base64 -w0 adhoc-server-cloud-init.yml)
+  jq --arg hostname "${ACTOR}" --arg plan "${PLAN}" --arg userdata "${USER_DATA}" \
+   '.hostname = $hostname | .type = $plan | .osConfiguration.cloudInit.userData = $userdata' \
+   adhoc-server-deploy.json > adhoc-server-deploy-final.json
   echo "Finished Deploy POST"
 
   echo "Running Deploy API"
diff --git a/.github/scripts/run-benchmarks-remote.sh b/.github/scripts/run-benchmarks-remote.sh
index 1adc5cb4..d85b6e42 100755
--- a/.github/scripts/run-benchmarks-remote.sh
+++ b/.github/scripts/run-benchmarks-remote.sh
@@ -11,6 +11,8 @@ set -f
 # - If TAG_NAME is "Any", run all tests
 # - If TAG_NAME starts with "!", run all tests except the named tag
 # - Otherwise, run tests marked with the tag name
+# Every form is limited to the classes matching TEST_PATTERN. The name filter (-n) and the tag
+# filters (-t/-T) are independent and intersect, so a short run can narrow both at once.
 # Note: Assumes the deephaven-benchmark-*.jar artifact has been built and placed
 
 if [[ $# != 6 ]]; then
@@ -76,7 +78,7 @@ if [ "${TAG_NAME}" = "Any" ]; then
 elif [[ "${TAG_NAME}" = !* ]]; then
   java ${JAVA_OPTS} -p ${TEST_PACKAGE} -n "${TEST_PATTERN}" -T "${TAG_NAME:1}"
 else
-  java ${JAVA_OPTS} -p ${TEST_PACKAGE} -t "${TAG_NAME}"
+  java ${JAVA_OPTS} -p ${TEST_PACKAGE} -n "${TEST_PATTERN}" -t "${TAG_NAME}"
 fi
 
 title "-- Getting Docker Logs --"
diff --git a/.github/scripts/setup-test-server-remote.sh b/.github/scripts/setup-test-server-remote.sh
index 2f76a88c..0de9e848 100755
--- a/.github/scripts/setup-test-server-remote.sh
+++ b/.github/scripts/setup-test-server-remote.sh
@@ -26,40 +26,48 @@ title () { echo; echo $1; }
 
 title "- Setting Up Remote Benchmark Testing on ${HOST} -"
 
+# Keep new automatic updates from starting, but let any in-flight one finish. These services use
+# KillMode=process, so stopping them would orphan the dpkg child holding the lock rather than end it
+title "-- Disabling Automatic Updates --"
+# "disabled" on arrival means cloud-init already handled it, "enabled" means it did not run
+echo "apt timers on arrival: $(systemctl is-enabled apt-daily.timer apt-daily-upgrade.timer 2>&1 | tr '\n' ' ')"
+sudo systemctl disable --now apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true
+sudo systemctl mask unattended-upgrades.service apt-daily.service apt-daily-upgrade.service 2>/dev/null || true
+sudo systemctl stop --no-block apt-daily.service apt-daily-upgrade.service 2>/dev/null || true
+sudo tee /etc/apt/apt.conf.d/10periodic >/dev/null <<EOF
+APT::Periodic::Enable "0";
+EOF
+sudo tee /etc/apt/apt.conf.d/20auto-upgrades >/dev/null <<EOF
+APT::Periodic::Update-Package-Lists "0";
+APT::Periodic::Unattended-Upgrade "0";
+EOF
+
+# A first boot upgrade can outlast ten minutes, so wait long enough to let it finish on its own
 title "-- Waiting for APT to be Free --"
 BEGIN_SECS=$(date +%s)
 STATUS=0
-for i in {1..60}; do
-  if ! sudo fuser /var/lib/dpkg/lock >/dev/null 2>&1 && ! sudo fuser /var/lib/apt/lists/lock >/dev/null 2>&1 \
-      && ! sudo fuser /var/lib/apt/lists/lock-frontend >/dev/null 2>&1; then
+for i in {1..120}; do
+  if ! sudo fuser /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock >/dev/null 2>&1; then
     STATUS=1
     break
   fi
+  if (( i % 6 == 0 )); then
+    echo "Still waiting after $(($(date +%s) - ${BEGIN_SECS}))s. Holding the lock:"
+    sudo fuser -v /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock 2>&1 | head -5 || true
+  fi
   sleep 10
 done
 
 DURATION=$(($(date +%s) - ${BEGIN_SECS}))
 if [[ $STATUS -eq 0 ]]; then
   echo "::error::Failed to gain APT lock after ${DURATION} seconds"
+  ps -eo pid,etimes,cmd | grep -E 'apt|dpkg|unattended' | grep -v grep || true
   exit 1
 fi
 
-title "-- Disabling Automatic Updates --"
-sudo systemctl stop unattended-upgrades.service 2>/dev/null || true
-sudo systemctl stop apt-daily.service 2>/dev/null || true
-sudo systemctl stop apt-daily-upgrade.service 2>/dev/null || true
-sudo systemctl disable --now apt-daily.timer 2>/dev/null || true
-sudo systemctl disable --now apt-daily-upgrade.timer 2>/dev/null || true
-sudo systemctl mask unattended-upgrades.service 2>/dev/null || true
-sudo systemctl mask apt-daily.service 2>/dev/null || true
-sudo systemctl mask apt-daily-upgrade.service 2>/dev/null || true
-sudo tee /etc/apt/apt.conf.d/10periodic >/dev/null <<EOF
-APT::Periodic::Enable "0";
-EOF
-sudo tee /etc/apt/apt.conf.d/20auto-upgrades >/dev/null <<EOF
-APT::Periodic::Update-Package-Lists "0";
-APT::Periodic::Unattended-Upgrade "0";
-EOF
+# An interrupted upgrade leaves dpkg needing a repair before any install will work
+title "-- Repairing Any Interrupted Package Transaction --"
+sudo dpkg --configure -a || true
 
 title "-- Disabling ASLR for Current Session --"
 sudo sysctl -w kernel.randomize_va_space=0 >/dev/null

From cadae966cb12a99a6abdc0d9da7b25a2fd4da84d Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Thu, 1 Oct 2026 17:19:17 -0600
Subject: [PATCH 05/13] Put release matrix config back to production values

---
 .github/matrix/release.sh | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.github/matrix/release.sh b/.github/matrix/release.sh
index eb8c7a82..d7497fc2 100755
--- a/.github/matrix/release.sh
+++ b/.github/matrix/release.sh
@@ -20,22 +20,19 @@
 #
 # TSV columns: run_type, run_label, docker_image, test_package, test_class_list,
 #              test_iterations, scale_row_count, distribution, config_options
-# TEMPORARY test values for verifying the workflow. A real release run uses
-# EXPECTED_COMBOS=7, CLS='*', PASSES=7. WhereTest is tagged and AvgByTest is not,
-# so both the !Iterate and Iterate jobs in a row have benchmarks to run
-EXPECTED_COMBOS=3
+EXPECTED_COMBOS=7
 
 RTYP='release'
 LABEL='<version>'
 IMG=${IMG:-edge}
 PKG=io.deephaven.benchmark.tests.standard
-CLS='Where,AvgBy'
+CLS='*'
 ITERS=1
 ROWSM=10
 DIST=random
 OPTS='<default>'
 
-PASSES=3
+PASSES=7
 
 for i in $(seq ${PASSES}); do
   echo -e "$RTYP\t$LABEL\t$IMG\t$PKG\t$CLS\t$ITERS\t$ROWSM\t$DIST\t$OPTS"

From a474200c1883811b61f85ced2c891a78ecdf632b Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Thu, 1 Oct 2026 17:30:40 -0600
Subject: [PATCH 06/13] fixed another apt stop issue

---
 .github/scripts/setup-test-server-remote.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/setup-test-server-remote.sh b/.github/scripts/setup-test-server-remote.sh
index 0de9e848..90ef5a78 100755
--- a/.github/scripts/setup-test-server-remote.sh
+++ b/.github/scripts/setup-test-server-remote.sh
@@ -26,14 +26,13 @@ title () { echo; echo $1; }
 
 title "- Setting Up Remote Benchmark Testing on ${HOST} -"
 
-# Keep new automatic updates from starting, but let any in-flight one finish. These services use
-# KillMode=process, so stopping them would orphan the dpkg child holding the lock rather than end it
+# Mask to keep new updates from starting, but never stop a running one. apt-daily.service leaves
+# KillMode at the control-group default, so stopping it would signal the apt child holding the lock
 title "-- Disabling Automatic Updates --"
 # "disabled" on arrival means cloud-init already handled it, "enabled" means it did not run
 echo "apt timers on arrival: $(systemctl is-enabled apt-daily.timer apt-daily-upgrade.timer 2>&1 | tr '\n' ' ')"
 sudo systemctl disable --now apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true
 sudo systemctl mask unattended-upgrades.service apt-daily.service apt-daily-upgrade.service 2>/dev/null || true
-sudo systemctl stop --no-block apt-daily.service apt-daily-upgrade.service 2>/dev/null || true
 sudo tee /etc/apt/apt.conf.d/10periodic >/dev/null <<EOF
 APT::Periodic::Enable "0";
 EOF

From 84fa407a103cfef8f2101fc89f59d36fc0108fd8 Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Thu, 1 Oct 2026 17:44:27 -0600
Subject: [PATCH 07/13] Fixed GH upload artifact name to be unique to the run

---
 .github/scripts/fetch-results-local.sh  | 3 +++
 .github/workflows/remote-benchmarks.yml | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/scripts/fetch-results-local.sh b/.github/scripts/fetch-results-local.sh
index c8902066..62524ead 100755
--- a/.github/scripts/fetch-results-local.sh
+++ b/.github/scripts/fetch-results-local.sh
@@ -62,6 +62,9 @@ mkdir -p ${DEST_DIR}
 rm -rf ${DEST_DIR}
 mv results/ ${DEST_DIR}/
 
+# Rows of a matrix share a set, so name artifacts after a run that only this fetch produced
+echo "RUN_ID=$(basename $(ls -d ${DEST_DIR}/run-* | head -1))" | tee -a ${OUTPUT_NAME}
+
 # For now remove any unwanted summaries before uploading to GCloud
 rm -f ${DEST_DIR}/*.csv
 
diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index 5968e83b..9c9c8247 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -214,7 +214,7 @@ jobs:
     - name: Archive Results
       uses: actions/upload-artifact@v4
       with:
-        name: Benchmarks and Logs for Set ${{steps.fetch-results.outputs.SET_LABEL}}
+        name: Benchmarks and Logs for Set ${{steps.fetch-results.outputs.SET_LABEL}} ${{steps.fetch-results.outputs.RUN_ID}}
         path: |
           ${{env.RUN_TYPE}}/**/*
           logs/**/*

From e8164b1ce1065002430ae4c0aa4753a1ab24351e Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Thu, 1 Oct 2026 19:10:05 -0600
Subject: [PATCH 08/13] Pull any images only if absent

---
 .github/scripts/manage-deephaven-remote.sh | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/manage-deephaven-remote.sh b/.github/scripts/manage-deephaven-remote.sh
index 8c2ee613..6dc5809f 100755
--- a/.github/scripts/manage-deephaven-remote.sh
+++ b/.github/scripts/manage-deephaven-remote.sh
@@ -38,16 +38,26 @@ fi
 echo "CONFIG_OPTS=${CONFIG_OPTS}" > .env
 echo "ENV_DEEPHAVEN_HOST_OS_DIR=${DEEPHAVEN_DIR}" >> .env
 
+# Reuse a pulled image the same way a built one is reused. A clear wipes images, so the first run
+# after it pulls, and later rows of a long matrix cannot drift onto a newly published tag
+pull_if_absent () {
+  if docker image inspect "$1" &>/dev/null 2>&1; then
+    echo "Image $1 already pulled. Skipping pull."
+  else
+    docker compose pull
+  fi
+}
+
 if [[ ${DOCKER_IMG} == ghcr.io/* ]]; then
   echo "DOCKER_IMG=${DOCKER_IMG}" >> .env
-  docker compose pull
+  pull_if_absent "${DOCKER_IMG}"
 elif [[ ${DOCKER_IMG} == *":"* ]]; then
   # Locally built from <owner>:<ref>, under the per-ref tag recorded by the distribution build.
   LOCAL_TAG=$(cat ${GIT_DIR}/benchmark-tag)
   echo "DOCKER_IMG=deephaven/server:${LOCAL_TAG}" >> .env
 else
   echo "DOCKER_IMG=ghcr.io/deephaven/server:${DOCKER_IMG}" >> .env
-  docker compose pull
+  pull_if_absent "ghcr.io/deephaven/server:${DOCKER_IMG}"
 fi
 
 if [[ ${DIRECTIVE} == 'start' ]]; then

From 5c0fd2f62ce80c29f8c0d1450ad736fe6750ddc1 Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Thu, 1 Oct 2026 19:12:53 -0600
Subject: [PATCH 09/13] Test short matrix run for edge caching

---
 .github/matrix/release.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/matrix/release.sh b/.github/matrix/release.sh
index d7497fc2..6fbe108f 100755
--- a/.github/matrix/release.sh
+++ b/.github/matrix/release.sh
@@ -20,19 +20,21 @@
 #
 # TSV columns: run_type, run_label, docker_image, test_package, test_class_list,
 #              test_iterations, scale_row_count, distribution, config_options
-EXPECTED_COMBOS=7
+# TEMPORARY test values for verifying the workflow. A real release run uses
+# EXPECTED_COMBOS=7, CLS='*', PASSES=7
+EXPECTED_COMBOS=3
 
 RTYP='release'
 LABEL='<version>'
 IMG=${IMG:-edge}
 PKG=io.deephaven.benchmark.tests.standard
-CLS='*'
+CLS='Where,AvgBy'
 ITERS=1
 ROWSM=10
 DIST=random
 OPTS='<default>'
 
-PASSES=7
+PASSES=3
 
 for i in $(seq ${PASSES}); do
   echo -e "$RTYP\t$LABEL\t$IMG\t$PKG\t$CLS\t$ITERS\t$ROWSM\t$DIST\t$OPTS"

From 263480aeffe8756903c46b19bb17e721212f439c Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Thu, 1 Oct 2026 20:34:00 -0600
Subject: [PATCH 10/13] back to the officialy release iterations

---
 .github/matrix/release.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/matrix/release.sh b/.github/matrix/release.sh
index 6fbe108f..d7497fc2 100755
--- a/.github/matrix/release.sh
+++ b/.github/matrix/release.sh
@@ -20,21 +20,19 @@
 #
 # TSV columns: run_type, run_label, docker_image, test_package, test_class_list,
 #              test_iterations, scale_row_count, distribution, config_options
-# TEMPORARY test values for verifying the workflow. A real release run uses
-# EXPECTED_COMBOS=7, CLS='*', PASSES=7
-EXPECTED_COMBOS=3
+EXPECTED_COMBOS=7
 
 RTYP='release'
 LABEL='<version>'
 IMG=${IMG:-edge}
 PKG=io.deephaven.benchmark.tests.standard
-CLS='Where,AvgBy'
+CLS='*'
 ITERS=1
 ROWSM=10
 DIST=random
 OPTS='<default>'
 
-PASSES=3
+PASSES=7
 
 for i in $(seq ${PASSES}); do
   echo -e "$RTYP\t$LABEL\t$IMG\t$PKG\t$CLS\t$ITERS\t$ROWSM\t$DIST\t$OPTS"

From 904e6c24eec1ca23a5aadc8b3ba36805a5a57c6c Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Thu, 1 Oct 2026 21:07:24 -0600
Subject: [PATCH 11/13] Fixed some copilot gripes

---
 .github/scripts/build-matrix.sh         | 1 +
 .github/workflows/remote-benchmarks.yml | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/scripts/build-matrix.sh b/.github/scripts/build-matrix.sh
index c023de63..90e35846 100755
--- a/.github/scripts/build-matrix.sh
+++ b/.github/scripts/build-matrix.sh
@@ -34,6 +34,7 @@ if [[ -n "${BAD_FIELDS}" ]]; then
   echo "::error::Every row needs 9 tab-separated fields in ${FILE}"; echo "${BAD_FIELDS}"; exit 1
 fi
 
+# Nightly is excluded on purpose, since a matrix run would collide with that day's real nightly set
 BAD_TYPES=$(echo "$TSV" | awk -F'\t' 'NF>0 && $1 !~ /^(adhoc|release|compare)$/ {print "  line "NR": "$1}')
 if [[ -n "${BAD_TYPES}" ]]; then
   echo "::error::run_type must be adhoc, release, or compare in ${FILE}"; echo "${BAD_TYPES}"; exit 1
diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index 9c9c8247..08a89acd 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -1,7 +1,7 @@
 # Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending
 
 # Run benchmarks on a remote system for the release the project is using
-# - This workflow is reusable with run_type set to release or nightly
+# - This workflow is reusable with run_type set to nightly, release, compare, or adhoc
 # - Scripts ending in "-local" run on the github runner
 # - Scripts ending in "-remote" run are executed on the remote test server through ssh
 # - test_device_addr selects both the server (<default>|<matrix>|ip) and the concurrency group

From 4f74b39aa03003f2e32206e5e62bc0c8a0d14f89 Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Thu, 1 Oct 2026 21:15:22 -0600
Subject: [PATCH 12/13] Added validation like in release

---
 .../workflows/compare-remote-benchmarks.yml    | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/compare-remote-benchmarks.yml b/.github/workflows/compare-remote-benchmarks.yml
index 8e139319..46326e7c 100644
--- a/.github/workflows/compare-remote-benchmarks.yml
+++ b/.github/workflows/compare-remote-benchmarks.yml
@@ -11,13 +11,27 @@ on:
   workflow_dispatch:
     inputs:
      docker_image:
-       description: 'Docker Image Name'
+       description: 'Docker Image Name (a release version, or edge for a pre-release run)'
        required: true
-       default: '0.36.0'
+       default: '42.0'
        type: string
 
 jobs:
+  validate-inputs:
+    runs-on: ubuntu-24.04
+    steps:
+    - name: Validate Docker Image Version
+      run: |
+        IMG="${{ inputs.docker_image }}"
+        if [[ "${IMG}" == "edge" ]]; then
+          echo "Pre-release run. The set label comes from the version the engine reports"
+        elif ! [[ "${IMG}" =~ ^[0-9]+(\.[0-9]+){1,2}$ ]]; then
+          echo "::error::docker_image '${IMG}' is not a release version (expected numeric format like 0.36.0 or 42.4) or 'edge'"
+          exit 1
+        fi
+
   process-compare-benchmarks:
+    needs: validate-inputs
     uses: ./.github/workflows/remote-benchmarks.yml
     with:
       run_type: compare

From 5ea6413fb7f99f6a332850111bff6ad29908b372 Mon Sep 17 00:00:00 2001
From: Stanley Brubaker <stanbrubaker@deephaven.io>
Date: Fri, 2 Oct 2026 11:45:24 -0600
Subject: [PATCH 13/13] Fixed some comments

---
 .github/workflows/adhoc-auto-remote-benchmarks.yml  | 2 +-
 .github/workflows/adhoc-exist-remote-benchmarks.yml | 2 +-
 .github/workflows/compare-remote-benchmarks.yml     | 7 +++----
 .github/workflows/nightly-remote-benchmarks.yml     | 2 +-
 4 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/adhoc-auto-remote-benchmarks.yml b/.github/workflows/adhoc-auto-remote-benchmarks.yml
index 33717c9e..1c3a8d8b 100644
--- a/.github/workflows/adhoc-auto-remote-benchmarks.yml
+++ b/.github/workflows/adhoc-auto-remote-benchmarks.yml
@@ -2,7 +2,7 @@
 
 # Provision a server and run benchmarks on that system, cleaning up after
 # - Deploys a new server on the fly
-# - Calls the reusable worflow remote-benchmarks.yml to run and upload benchmarks
+# - Calls the reusable workflow remote-benchmarks.yml to run and upload benchmarks
 # - Runs a comparison between two images/branches
 # - Deletes the server on success or failure
 
diff --git a/.github/workflows/adhoc-exist-remote-benchmarks.yml b/.github/workflows/adhoc-exist-remote-benchmarks.yml
index fa122711..a5271761 100644
--- a/.github/workflows/adhoc-exist-remote-benchmarks.yml
+++ b/.github/workflows/adhoc-exist-remote-benchmarks.yml
@@ -1,7 +1,7 @@
 # Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending
 
 # Run benchmarks on an already provisioned system
-# - Calls the reusable worflow remote-benchmarks.yml to run and upload benchmarks
+# - Calls the reusable workflow remote-benchmarks.yml to run and upload benchmarks
 # - Runs the given options with the given image/branch
 
 name: Adhoc Benchmarks (Existing Server)
diff --git a/.github/workflows/compare-remote-benchmarks.yml b/.github/workflows/compare-remote-benchmarks.yml
index 46326e7c..f4f28ff9 100644
--- a/.github/workflows/compare-remote-benchmarks.yml
+++ b/.github/workflows/compare-remote-benchmarks.yml
@@ -1,9 +1,8 @@
 # Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending
 
-# Run comparison (competitive) benchmarks on a remote system 
-# according to the release defined in resources/release-benchmark-docker-compose.yml
-# in resources/release-benchmark-docker-compose.yml
-# - Calls the reusable worflow remote-benchmarks.yml
+# Run comparison (competitive) benchmarks on a remote system, measuring Deephaven against DuckDB,
+# Flink, Pandas, and PyArrow, which the tests run as python inside the Deephaven container
+# - Calls the reusable workflow remote-benchmarks.yml
 
 name: Compare Benchmark Test on Docker Deephaven
 
diff --git a/.github/workflows/nightly-remote-benchmarks.yml b/.github/workflows/nightly-remote-benchmarks.yml
index b0975309..cd52a16e 100644
--- a/.github/workflows/nightly-remote-benchmarks.yml
+++ b/.github/workflows/nightly-remote-benchmarks.yml
@@ -1,7 +1,7 @@
 # Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending
 
 # Run benchmarks on a remote system for the nightly (edge) build
-# - Calls the reusable worflow remote-benchmarks.yml
+# - Calls the reusable workflow remote-benchmarks.yml
 
 name: Nightly Benchmark Test on Docker Deephaven