feat: DH-23748: Multipass Release Matrix Workflow

.github/matrix/example-regression-matrix.sh

@@ -29,10 +29,11 @@
 # Class lists omit the Test suffix: ConsoleLauncherUtil appends it, so 'UserFormulaTest' matches
 # nothing and the run reports zero benchmarks rather than failing.
 #
-# TSV columns: run_label, docker_image, test_package, test_class_list,
+# TSV columns: run_type, run_label, docker_image, test_package, test_class_list,
 #              test_iterations, scale_row_count, distribution, config_options
 EXPECTED_COMBOS=5
 
+RTYP='adhoc'
 PKG=io.deephaven.benchmark.tests.standard
 CLASSES='UserFormula'
 DIST=random
@@ -49,7 +50,7 @@ jfr_opts() {
 TOP=deephaven:c50cd7f2fc02b819376a80e8767dff76c0624aa2
 
 # $3 is config_options, defaulting to a JFR recording named after the row
-row() { echo -e "$1\t$2\t$PKG\t$CLASSES\t$ITERS\t$ROWSM\t$DIST\t${3:-$(jfr_opts "$1")}"; }
+row() { echo -e "$RTYP\t$1\t$2\t$PKG\t$CLASSES\t$ITERS\t$ROWSM\t$DIST\t${3:-$(jfr_opts "$1")}"; }
 
 row anchor_img_0911 "$ANCHOR"
 row src_base deephaven:9fa9e83c55ccb2e9ddedd61d1ba59d2d8da68c85

.github/matrix/gc-train-1000-100.sh

@@ -1,8 +1,9 @@
 # Compare GC configurations across cycle times and profiles
-# TSV columns: run_label, docker_image, test_package, test_class_list,
+# TSV columns: run_type, run_label, docker_image, test_package, test_class_list,
 #              test_iterations, scale_row_count, distribution, config_options
 EXPECTED_COMBOS=32
 
+RTYP='adhoc'
 IMG=ghcr.io/stanbrub/server:jvm25
 PKG=io.deephaven.benchmark.tests.train
 CLS='*Train'
@@ -23,11 +24,12 @@ R="$IMG\t$PKG\t$CLS\t$ITERS\t$ROWSM\t$DIST"
 # emit <label> <gc> <cycle_ms> <inc_load_target> — outputs regular and _huge variants
 run() {
   local opts="$2 $BASE -DPeriodicUpdateGraph.targetCycleDurationMillis=$3 -Dbench.incLoadTarget=$4"
-  echo -e "$1\t$R\t$opts"
-  echo -e "${1}_huge\t$R\t$opts $HUGE"
+  echo -e "$RTYP\t$1\t$R\t$opts"
+  echo -e "$RTYP\t${1}_huge\t$R\t$opts $HUGE"
 }
 
-# 1000ms cycle
+# 1000ms cycle/adhoc
+
 run gc_g1gc_1000_p100_j25  "$G1"   1000  1.00
 run gc_zgc_1000_p100_j25   "$ZGC"  1000  1.00
 run gc_shen_1000_p100_j25  "$SHEN" 1000  1.00

.github/matrix/gc-train-heap-fail.sh

@@ -1,7 +1,8 @@
 # Sweep heap size to find failure points per GC and test class
-# TSV columns: run_label, docker_image, test_package, test_class_list,
+# TSV columns: run_type, run_label, docker_image, test_package, test_class_list,
 #              test_iterations, scale_row_count, distribution, config_options
 
+RTYP='adhoc'
 IMG=ghcr.io/stanbrub/server:jvm25
 PKG=io.deephaven.benchmark.tests.train
 ITERS=1
@@ -51,7 +52,7 @@ for combo in "${COMBOS[@]}"; do
   read -r cls gc min_heap max_heap <<< "$combo"
   for h in $(seq ${max_heap} -${STEP} ${min_heap}); do
     opts="${GCS[$gc]} -Xms${h}g -Xmx${h}g $BASE $CYCLE"
-    echo -e "gc_${gc}_${cls}_h${h}_j25\t$IMG\t$PKG\t*${cls}\t$ITERS\t$ROWSM\t$DIST\t$opts"
+    echo -e "$RTYP\tgc_${gc}_${cls}_h${h}_j25\t$IMG\t$PKG\t*${cls}\t$ITERS\t$ROWSM\t$DIST\t$opts"
   done
 done
 

.github/matrix/release.sh

@@ -0,0 +1,39 @@
+# Release benchmarks: one full pass per row, so every benchmark gets the same sample count
+#
+# Each row is a complete pass over the standard suite and publishes its results before the next row
+# starts. That matters for a run this long: iterations inside a single row would upload nothing until
+# the end, and a failure part way through would lose the whole run. Rows are independent, so a failed
+# row costs one pass instead of all of them.
+#
+# Row count is the sample count. Release ignores the Iterate tag, so a row at test_iterations 1 is a
+# single job over every class, and 7 rows means n=7 for all 666 benchmarks. Nightly keeps the tag
+# split for its time budget, which leaves its 562 untagged benchmarks at n=1 and 104 tagged at n=5.
+#
+# Keep the row count odd. Queries report the middle run by rate and carry that run's id, so an even
+# count has no real middle and the reported numbers stop matching a run that happened.
+#
+# All rows share the run_label, so they land in one set and accumulate. <version> takes the label from
+# the version the engine reports, so edge becomes a 99 in the last position and cannot collide with
+# the release of the same major.minor.
+#
+# Budget roughly 4 hours per row of test time, plus setup, engine restarts and data generation.
+#
+# TSV columns: run_type, run_label, docker_image, test_package, test_class_list,
+#              test_iterations, scale_row_count, distribution, config_options
+EXPECTED_COMBOS=7
+
+RTYP='release'
+LABEL='<version>'
+IMG=${IMG:-edge}
+PKG=io.deephaven.benchmark.tests.standard
+CLS='*'
+ITERS=1
+ROWSM=10
+DIST=random
+OPTS='<default>'
+
+PASSES=7
+
+for i in $(seq ${PASSES}); do
+  echo -e "$RTYP\t$LABEL\t$IMG\t$PKG\t$CLS\t$ITERS\t$ROWSM\t$DIST\t$OPTS"
+done

.github/resources/adhoc-server-cloud-init.yml

@@ -0,0 +1,12 @@
+#cloud-config
+
+# Keep the apt timers from firing on a freshly provisioned server. Both ship with Persistent=true, so
+# a new box treats their schedule as missed and runs them at a random offset after boot. When that
+# offset lands in the first few minutes it collides with benchmark setup and holds the dpkg lock.
+#
+# bootcmd runs in the cloud-init local stage, well before timers.target, so the timers never fire.
+# Do not add a users: block here. The server is provisioned with installDefaultSshKeys, and a users:
+# list that omits "- default" replaces the provisioned key and locks the run out of its own server.
+bootcmd:
+  - systemctl disable --now apt-daily.timer apt-daily-upgrade.timer
+  - systemctl mask apt-daily.service apt-daily-upgrade.service unattended-upgrades.service

.github/scripts/adhoc.sh

@@ -111,8 +111,11 @@ if [[ ${ACTION} == "deploy-metal" ]]; then
   TOKEN=$(getApiToken "${PROJECT_ID}" "${API_KEY}")
 
   echo "Making Deploy POST"
-  jq --arg hostname "${ACTOR}" --arg plan "${PLAN}" \
-   '.hostname = $hostname | .type = $plan' adhoc-server-deploy.json > adhoc-server-deploy-final.json
+  # The API wants cloud-init user data base64 encoded on one line, and will not echo it back
+  USER_DATA=$(base64 -w0 adhoc-server-cloud-init.yml)
+  jq --arg hostname "${ACTOR}" --arg plan "${PLAN}" --arg userdata "${USER_DATA}" \
+   '.hostname = $hostname | .type = $plan | .osConfiguration.cloudInit.userData = $userdata' \
+   adhoc-server-deploy.json > adhoc-server-deploy-final.json
   echo "Finished Deploy POST"
 
   echo "Running Deploy API"

.github/scripts/build-matrix.sh

@@ -7,8 +7,9 @@ set -o pipefail
 
 # Runs a matrix script and converts its TSV output to a JSON matrix for GitHub Actions.
 # The matrix script outputs tab-separated rows with fields:
-#   run_label, docker_image, test_package, test_class_list,
+#   run_type, run_label, docker_image, test_package, test_class_list,
 #   test_iterations, scale_row_count, distribution, config_options
+# Every field is required, so a new field means updating every matrix script.
 # Scale_row_count is in millions and auto-scaled to actual.
 # Test_iterations is auto-forced to odd.
 #
@@ -28,8 +29,26 @@ fi
 
 TSV=$(bash "$FILE")
 
+BAD_FIELDS=$(echo "$TSV" | awk -F'\t' 'NF>0 && NF!=9 {print "  line "NR": "NF" fields"}')
+if [[ -n "${BAD_FIELDS}" ]]; then
+  echo "::error::Every row needs 9 tab-separated fields in ${FILE}"; echo "${BAD_FIELDS}"; exit 1
+fi
+
+# Nightly is excluded on purpose, since a matrix run would collide with that day's real nightly set
+BAD_TYPES=$(echo "$TSV" | awk -F'\t' 'NF>0 && $1 !~ /^(adhoc|release|compare)$/ {print "  line "NR": "$1}')
+if [[ -n "${BAD_TYPES}" ]]; then
+  echo "::error::run_type must be adhoc, release, or compare in ${FILE}"; echo "${BAD_TYPES}"; exit 1
+fi
+
+# Rows sharing a run_type and run_label land in one set, and queries report the middle run by rate
+BAD_PARITY=$(echo "$TSV" | awk -F'\t' 'NF>0 {c[$1"/"$2]++} END {for (k in c) if (c[k]>1 && c[k]%2==0) print "  "k": "c[k]" rows"}')
+if [[ -n "${BAD_PARITY}" ]]; then
+  echo "::error::Rows per set must be odd so the reported median is a real run. Add or remove a row in ${FILE}"
+  echo "${BAD_PARITY}"; exit 1
+fi
+
 RESULT=$(echo "$TSV" | jq -Rsc '
-  ["run_label","docker_image","test_package","test_class_list",
+  ["run_type","run_label","docker_image","test_package","test_class_list",
    "test_iterations","scale_row_count","distribution","config_options"] as $h |
   split("\n") | map(select(length > 0)) |
   [.[] | split("\t") | [range(length) as $i | {($h[$i]): .[$i]}] | add |

.github/scripts/fetch-results-local.sh

@@ -10,8 +10,8 @@ set -o nounset
 # compresses the runs before upload. Writes an output file with the 
 # SET_LABEL that was used for the set directory name
 
-if [[ $# != 7 ]]; then
-  echo "$0: Missing host, user, run type, script dir, actor, docker img, or run label arguments"
+if [[ $# != 6 ]]; then
+  echo "$0: Missing host, user, script dir, run type, actor, or run label arguments"
   exit 1
 fi
 
@@ -21,38 +21,50 @@ SCRIPT_DIR=$3
 RUN_TYPE=$4
 ACTOR=$5
 SET_LABEL=${6:-$(echo -n "set-"; ${SCRIPT_DIR}/base.sh $(date +%s%03N) 62)}
-DOCKER_IMG=$7
 RUN_DIR=/home/${USER}/run
 OUTPUT_NAME=fetch-results-local.out
 
 rm -f ${OUTPUT_NAME}; touch ${OUTPUT_NAME}
 
+# Pull results from the benchmark server (Before labelling, since <version> reads the platform csv)
+scp -r ${USER}@${HOST}:${RUN_DIR}/results .
+scp -r ${USER}@${HOST}:${RUN_DIR}/logs .
+scp -r ${USER}@${HOST}:${RUN_DIR}/*.jar .
+
 # Get the date for the Set Label, since Github Workflows don't have 'with: ${{github.date}}'
 if [ "${SET_LABEL}" = "<date>" ]; then
   SET_LABEL=$(date '+%Y-%m-%d')
 fi
 
-# Get the version for the Set Label, since Github Workflows don't have 'with: ${{github.date}}'
+# Get the Set Label from the engine-reported version (edge has none), 99 in last place for snapshots
 if [ "${SET_LABEL}" = "<version>" ]; then
-  vers=${DOCKER_IMG}
-  major=$(printf '%02d\n' $(echo ${vers} | cut -d "." -f 1))
-  minor=$(printf '%03d\n' $(echo ${vers} | cut -d "." -f 2))
-  patch=$(printf '%02d\n' $(echo ${vers} | cut -d "." -f 3))
+  PLATFORM_CSV=results/platform-summary-results.csv
+  # Match the engine only. The test-runner reports Unknown unless run from a release jar
+  vers=$(awk -F, '$2=="deephaven-engine" && $3=="deephaven.version" {print $4; exit}' ${PLATFORM_CSV})
+  # Empty would pad to a valid-looking 00.000.00, so fail instead
+  : "${vers:?no deephaven-engine deephaven.version found in ${PLATFORM_CSV}}"
+  base=${vers%-SNAPSHOT}
+  major=$(printf '%02d\n' $(echo ${base} | cut -d "." -f 1))
+  minor=$(printf '%03d\n' $(echo ${base} | cut -d "." -f 2))
+  if [ "${base}" = "${vers}" ]; then
+    patch=$(printf '%02d\n' $(echo ${base} | cut -d "." -f 3))
+  else
+    patch=99
+  fi
   SET_LABEL="${major}.${minor}.${patch}"
+  echo "Engine reported ${vers}, using set label ${SET_LABEL}"
 fi
 echo "SET_LABEL=${SET_LABEL}" | tee -a ${OUTPUT_NAME}
 
-# Pull results from the benchmark server
-scp -r ${USER}@${HOST}:${RUN_DIR}/results .
-scp -r ${USER}@${HOST}:${RUN_DIR}/logs .
-scp -r ${USER}@${HOST}:${RUN_DIR}/*.jar .
-
 # Move the results into the destination directory
 DEST_DIR=${RUN_TYPE}/${ACTOR}/${SET_LABEL}
 mkdir -p ${DEST_DIR}
 rm -rf ${DEST_DIR}
 mv results/ ${DEST_DIR}/
 
+# Rows of a matrix share a set, so name artifacts after a run that only this fetch produced
+echo "RUN_ID=$(basename $(ls -d ${DEST_DIR}/run-* | head -1))" | tee -a ${OUTPUT_NAME}
+
 # For now remove any unwanted summaries before uploading to GCloud
 rm -f ${DEST_DIR}/*.csv
 

.github/scripts/manage-deephaven-remote.sh

@@ -38,16 +38,26 @@ fi
 echo "CONFIG_OPTS=${CONFIG_OPTS}" > .env
 echo "ENV_DEEPHAVEN_HOST_OS_DIR=${DEEPHAVEN_DIR}" >> .env
 
+# Reuse a pulled image the same way a built one is reused. A clear wipes images, so the first run
+# after it pulls, and later rows of a long matrix cannot drift onto a newly published tag
+pull_if_absent () {
+  if docker image inspect "$1" &>/dev/null 2>&1; then
+    echo "Image $1 already pulled. Skipping pull."
+  else
+    docker compose pull
+  fi
+}
+
 if [[ ${DOCKER_IMG} == ghcr.io/* ]]; then
   echo "DOCKER_IMG=${DOCKER_IMG}" >> .env
-  docker compose pull
+  pull_if_absent "${DOCKER_IMG}"
 elif [[ ${DOCKER_IMG} == *":"* ]]; then
   # Locally built from <owner>:<ref>, under the per-ref tag recorded by the distribution build.
   LOCAL_TAG=$(cat ${GIT_DIR}/benchmark-tag)
   echo "DOCKER_IMG=deephaven/server:${LOCAL_TAG}" >> .env
 else
   echo "DOCKER_IMG=ghcr.io/deephaven/server:${DOCKER_IMG}" >> .env
-  docker compose pull
+  pull_if_absent "ghcr.io/deephaven/server:${DOCKER_IMG}"
 fi
 
 if [[ ${DIRECTIVE} == 'start' ]]; then

.github/scripts/matrix_array.sh

@@ -9,12 +9,13 @@ set -o pipefail
 # workflow with fromJSON(array)
 # ex. matrix_array.sh matrix-arr adhoc 5
 # ex. matrix_array.sh matrix-arr release 4
+# Only nightly splits on the Iterate tag, to fit a curated subset of repeats into its time budget
 
 NAME=$1
 RUN_TYPE=$2
 ITERATIONS=$3
 
-if [ "${RUN_TYPE}" = 'release' ] || [ "${RUN_TYPE}" = 'nightly' ]; then
+if [ "${RUN_TYPE}" = 'nightly' ]; then
   FIRST='!Iterate'
   TAG='Iterate'
 else

.github/scripts/run-benchmarks-remote.sh

@@ -11,6 +11,8 @@ set -f
 # - If TAG_NAME is "Any", run all tests
 # - If TAG_NAME starts with "!", run all tests except the named tag
 # - Otherwise, run tests marked with the tag name
+# Every form is limited to the classes matching TEST_PATTERN. The name filter (-n) and the tag
+# filters (-t/-T) are independent and intersect, so a short run can narrow both at once.
 # Note: Assumes the deephaven-benchmark-*.jar artifact has been built and placed
 
 if [[ $# != 6 ]]; then
@@ -76,7 +78,7 @@ if [ "${TAG_NAME}" = "Any" ]; then
 elif [[ "${TAG_NAME}" = !* ]]; then
   java ${JAVA_OPTS} -p ${TEST_PACKAGE} -n "${TEST_PATTERN}" -T "${TAG_NAME:1}"
 else
-  java ${JAVA_OPTS} -p ${TEST_PACKAGE} -t "${TAG_NAME}"
+  java ${JAVA_OPTS} -p ${TEST_PACKAGE} -n "${TEST_PATTERN}" -t "${TAG_NAME}"
 fi
 
 title "-- Getting Docker Logs --"

.github/scripts/run-ssh-local.sh

@@ -41,4 +41,7 @@ for ((attempt=1; attempt<=MAX_RETRIES; attempt++)); do
   fi
 done
 
-ssh -o 'ConnectTimeout 10' -o 'ServerAliveInterval 60' ${USER}@${HOST} 'bash -s' -- "${args[@]}" < ${SCRIPT_DIR}/${SCRIPT_NAME}.sh |& tee logs/${SCRIPT_NAME}.log
+# Redact the host from the log file, since artifacts are public and Github masks only the live log
+HOST_PATTERN=$(printf '%s' "${HOST}" | sed 's/[^a-zA-Z0-9-]/\\&/g')
+
+ssh -o 'ConnectTimeout 10' -o 'ServerAliveInterval 60' ${USER}@${HOST} 'bash -s' -- "${args[@]}" < ${SCRIPT_DIR}/${SCRIPT_NAME}.sh |& sed -u "s/${HOST_PATTERN}/<redacted-host>/g" | tee logs/${SCRIPT_NAME}.log

.github/scripts/setup-ssh-local.sh

@@ -17,6 +17,11 @@ if [[ $# != 2 ]]; then
   exit 1
 fi
 
+if [[ "${HOST}" == "<"*">" ]]; then
+  echo "$0: Host '${HOST}' was not resolved. Is the matching BENCHMARK_HOST_* secret set?"
+  exit 1
+fi
+
 mkdir -p logs
 mkdir -p results
 mkdir -p ~/.ssh/

.github/scripts/setup-test-server-remote.sh

@@ -26,40 +26,47 @@ title () { echo; echo $1; }
 
 title "- Setting Up Remote Benchmark Testing on ${HOST} -"
 
+# Mask to keep new updates from starting, but never stop a running one. apt-daily.service leaves
+# KillMode at the control-group default, so stopping it would signal the apt child holding the lock
+title "-- Disabling Automatic Updates --"
+# "disabled" on arrival means cloud-init already handled it, "enabled" means it did not run
+echo "apt timers on arrival: $(systemctl is-enabled apt-daily.timer apt-daily-upgrade.timer 2>&1 | tr '\n' ' ')"
+sudo systemctl disable --now apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true
+sudo systemctl mask unattended-upgrades.service apt-daily.service apt-daily-upgrade.service 2>/dev/null || true
+sudo tee /etc/apt/apt.conf.d/10periodic >/dev/null <<EOF
+APT::Periodic::Enable "0";
+EOF
+sudo tee /etc/apt/apt.conf.d/20auto-upgrades >/dev/null <<EOF
+APT::Periodic::Update-Package-Lists "0";
+APT::Periodic::Unattended-Upgrade "0";
+EOF
+
+# A first boot upgrade can outlast ten minutes, so wait long enough to let it finish on its own
 title "-- Waiting for APT to be Free --"
 BEGIN_SECS=$(date +%s)
 STATUS=0
-for i in {1..60}; do
-  if ! sudo fuser /var/lib/dpkg/lock >/dev/null 2>&1 && ! sudo fuser /var/lib/apt/lists/lock >/dev/null 2>&1 \
-      && ! sudo fuser /var/lib/apt/lists/lock-frontend >/dev/null 2>&1; then
+for i in {1..120}; do
+  if ! sudo fuser /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock >/dev/null 2>&1; then
     STATUS=1
     break
   fi
+  if (( i % 6 == 0 )); then
+    echo "Still waiting after $(($(date +%s) - ${BEGIN_SECS}))s. Holding the lock:"
+    sudo fuser -v /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock 2>&1 | head -5 || true
+  fi
   sleep 10
 done
 
 DURATION=$(($(date +%s) - ${BEGIN_SECS}))
 if [[ $STATUS -eq 0 ]]; then
   echo "::error::Failed to gain APT lock after ${DURATION} seconds"
+  ps -eo pid,etimes,cmd | grep -E 'apt|dpkg|unattended' | grep -v grep || true
   exit 1
 fi
 
-title "-- Disabling Automatic Updates --"
-sudo systemctl stop unattended-upgrades.service 2>/dev/null || true
-sudo systemctl stop apt-daily.service 2>/dev/null || true
-sudo systemctl stop apt-daily-upgrade.service 2>/dev/null || true
-sudo systemctl disable --now apt-daily.timer 2>/dev/null || true
-sudo systemctl disable --now apt-daily-upgrade.timer 2>/dev/null || true
-sudo systemctl mask unattended-upgrades.service 2>/dev/null || true
-sudo systemctl mask apt-daily.service 2>/dev/null || true
-sudo systemctl mask apt-daily-upgrade.service 2>/dev/null || true
-sudo tee /etc/apt/apt.conf.d/10periodic >/dev/null <<EOF
-APT::Periodic::Enable "0";
-EOF
-sudo tee /etc/apt/apt.conf.d/20auto-upgrades >/dev/null <<EOF
-APT::Periodic::Update-Package-Lists "0";
-APT::Periodic::Unattended-Upgrade "0";
-EOF
+# An interrupted upgrade leaves dpkg needing a repair before any install will work
+title "-- Repairing Any Interrupted Package Transaction --"
+sudo dpkg --configure -a || true
 
 title "-- Disabling ASLR for Current Session --"
 sudo sysctl -w kernel.randomize_va_space=0 >/dev/null