From 39f870a4e74ef7fed00d48aceb2f3ea6a3df9388 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 27 Jul 2026 19:36:29 +0300 Subject: [PATCH 1/9] Add ducktests-remote CLI for running ducktests on a real VM cluster Until now ducktests could be run either locally in Docker via tests/docker/run_tests.sh, or on a real VM cluster from Jenkins by hand-assembling a long `ducktape ...` command line with a large inline --globals JSON blob. There was no supported way to launch a run against the VM cluster from an engineer's machine. This fills that gap. New package modules/ducktests/tests/ducktests_remote/, exposed as the `ducktests-remote` console script: run compose the artifacts, launch ducktape detached, follow the log status state of a run, or a table of every run (--all) logs print or follow a run's ducktape.log stop terminate a run and clean the workers behind it fetch download the reports to the coordinator doctor check the coordinator, the runner and every worker provision bring the workers to the state the Docker image guarantees deploy copy distributions to / on every worker clean kill stale Ignite JVMs and remove work directories keys install the runner identity and authorise it on the workers Design points worth calling out: * Every command goes through a Transport (local, ssh, or proxied through a third host), so --runner local and --runner some-host take identical code paths above the transport boundary. Nothing shells out to ssh outside a transport implementation. * All run state lives on the runner, never on the coordinator, so any coordinator can inspect, follow or stop a run that a different one started. * The package never imports ducktape. It drives ducktape on the runner and must stay installable on a coordinator that has none; a unit check fails if that ever changes. * Runs are detached from second zero via setsid + a pid/exit_code wrapper. ducktape's runner is an ordinary foreground process, so a dropped SSH session would otherwise SIGHUP it mid-run and leave Ignite JVMs alive on every worker. Ctrl-C during --follow detaches; stopping takes a deliberate second interrupt. * The Jenkins --globals blob is replaced by layered YAML profiles with ${env:}/${file:} interpolation. A missing variable is a hard error naming the variable and the file. Resolved secrets are registered with a value-based redactor and masked in everything the CLI prints; globals.json is written mode 0600 and always excluded from fetch. --globals-json/--globals-file keep the existing blob working verbatim as the migration path. * No cluster lease: this deployment has a single runner, and queueing belongs to whatever schedules the runs. Exit code 3 is reserved. Verified against the sources rather than assumed: * ducktape 0.13 accepts a file path for --globals (command_line/main.py checks os.path.isfile first), so the composed blob is referenced by path and never crosses a shell command line. * The cluster file schema is the one RemoteAccountSSHConfig accepts. * ignitetest resolves a distribution home as / where product is str(IgniteVersion(v)), which normalises: ise--6 maps to /opt/ise-6. Because a fork can override product, doctor reports a missing directory as a WARN listing what it did find. * sudo is needed only for `sudo iptables` from IgniteAwareService.drop_network, reached by exactly two suites: discovery_test.py and cellular_affinity_test.py. Everything else runs unprivileged, and doctor says so. * ducktape's loader puts the test tree on sys.path itself (loader.py::_add_top_level_dirs_to_sys_path), so synced sources do not need installing; only the pinned requirements have to be in the venv. * provision's package list is derived from docker/Dockerfile, with a comment naming it as the source of truth. Adds PyYAML to docker/requirements.txt: config and profile parsing is the package's only dependency beyond the standard library. Unit checks live in ducktests_remote/checks/ as check_*.py with Check classes, matching what [pytest] in tox.ini collects. 164 checks, no network, no Docker, no ducktape. flake8 clean. --- modules/ducktests/tests/MANIFEST.in | 2 + .../ducktests/tests/docker/requirements.txt | 3 + .../tests/ducktests_remote/README.md | 374 +++++++++++ .../tests/ducktests_remote/__init__.py | 25 + .../tests/ducktests_remote/__main__.py | 23 + .../checks/check_remote_cluster.py | 140 ++++ .../checks/check_remote_config.py | 141 ++++ .../checks/check_remote_deploy.py | 232 +++++++ .../checks/check_remote_globals.py | 143 +++++ .../checks/check_remote_runs.py | 195 ++++++ .../checks/check_remote_sshdiag.py | 125 ++++ .../checks/check_remote_transport.py | 190 ++++++ .../ducktests_remote/checks/fake_transport.py | 88 +++ .../ducktests/tests/ducktests_remote/cli.py | 284 +++++++++ .../tests/ducktests_remote/cluster.py | 185 ++++++ .../ducktests_remote/commands/__init__.py | 16 + .../tests/ducktests_remote/commands/clean.py | 138 ++++ .../tests/ducktests_remote/commands/deploy.py | 307 +++++++++ .../tests/ducktests_remote/commands/doctor.py | 602 ++++++++++++++++++ .../tests/ducktests_remote/commands/fetch.py | 128 ++++ .../tests/ducktests_remote/commands/keys.py | 135 ++++ .../tests/ducktests_remote/commands/logs.py | 86 +++ .../ducktests_remote/commands/provision.py | 409 ++++++++++++ .../tests/ducktests_remote/commands/run.py | 503 +++++++++++++++ .../tests/ducktests_remote/commands/status.py | 117 ++++ .../tests/ducktests_remote/commands/stop.py | 104 +++ .../tests/ducktests_remote/config.py | 371 +++++++++++ .../ducktests_remote/examples/cluster.yaml | 67 ++ .../examples/profile-ise-perf.yaml | 51 ++ .../examples/profile-smoke.yaml | 21 + .../tests/ducktests_remote/fanout.py | 153 +++++ .../tests/ducktests_remote/globals_builder.py | 199 ++++++ .../ducktests/tests/ducktests_remote/runs.py | 357 +++++++++++ .../tests/ducktests_remote/sshdiag.py | 271 ++++++++ .../ducktests_remote/templates/run.sh.tmpl | 28 + .../tests/ducktests_remote/transport.py | 510 +++++++++++++++ modules/ducktests/tests/setup.py | 7 + 37 files changed, 6730 insertions(+) create mode 100644 modules/ducktests/tests/ducktests_remote/README.md create mode 100644 modules/ducktests/tests/ducktests_remote/__init__.py create mode 100644 modules/ducktests/tests/ducktests_remote/__main__.py create mode 100644 modules/ducktests/tests/ducktests_remote/checks/check_remote_cluster.py create mode 100644 modules/ducktests/tests/ducktests_remote/checks/check_remote_config.py create mode 100644 modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py create mode 100644 modules/ducktests/tests/ducktests_remote/checks/check_remote_globals.py create mode 100644 modules/ducktests/tests/ducktests_remote/checks/check_remote_runs.py create mode 100644 modules/ducktests/tests/ducktests_remote/checks/check_remote_sshdiag.py create mode 100644 modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py create mode 100644 modules/ducktests/tests/ducktests_remote/checks/fake_transport.py create mode 100644 modules/ducktests/tests/ducktests_remote/cli.py create mode 100644 modules/ducktests/tests/ducktests_remote/cluster.py create mode 100644 modules/ducktests/tests/ducktests_remote/commands/__init__.py create mode 100644 modules/ducktests/tests/ducktests_remote/commands/clean.py create mode 100644 modules/ducktests/tests/ducktests_remote/commands/deploy.py create mode 100644 modules/ducktests/tests/ducktests_remote/commands/doctor.py create mode 100644 modules/ducktests/tests/ducktests_remote/commands/fetch.py create mode 100644 modules/ducktests/tests/ducktests_remote/commands/keys.py create mode 100644 modules/ducktests/tests/ducktests_remote/commands/logs.py create mode 100644 modules/ducktests/tests/ducktests_remote/commands/provision.py create mode 100644 modules/ducktests/tests/ducktests_remote/commands/run.py create mode 100644 modules/ducktests/tests/ducktests_remote/commands/status.py create mode 100644 modules/ducktests/tests/ducktests_remote/commands/stop.py create mode 100644 modules/ducktests/tests/ducktests_remote/config.py create mode 100644 modules/ducktests/tests/ducktests_remote/examples/cluster.yaml create mode 100644 modules/ducktests/tests/ducktests_remote/examples/profile-ise-perf.yaml create mode 100644 modules/ducktests/tests/ducktests_remote/examples/profile-smoke.yaml create mode 100644 modules/ducktests/tests/ducktests_remote/fanout.py create mode 100644 modules/ducktests/tests/ducktests_remote/globals_builder.py create mode 100644 modules/ducktests/tests/ducktests_remote/runs.py create mode 100644 modules/ducktests/tests/ducktests_remote/sshdiag.py create mode 100644 modules/ducktests/tests/ducktests_remote/templates/run.sh.tmpl create mode 100644 modules/ducktests/tests/ducktests_remote/transport.py diff --git a/modules/ducktests/tests/MANIFEST.in b/modules/ducktests/tests/MANIFEST.in index 6fcceb37144dd..523152cba3824 100644 --- a/modules/ducktests/tests/MANIFEST.in +++ b/modules/ducktests/tests/MANIFEST.in @@ -14,3 +14,5 @@ # limitations under the License. recursive-include ignitetest **.j2 +recursive-include ducktests_remote/templates *.tmpl +recursive-include ducktests_remote/examples *.yaml diff --git a/modules/ducktests/tests/docker/requirements.txt b/modules/ducktests/tests/docker/requirements.txt index aba4f25a86d1d..57492847d3c99 100644 --- a/modules/ducktests/tests/docker/requirements.txt +++ b/modules/ducktests/tests/docker/requirements.txt @@ -16,3 +16,6 @@ filelock==3.8.2 ducktape==0.13.0 looseversion==1.3.0 +# ducktests_remote (the `ducktests-remote` CLI) parses YAML config and profiles. +# It is the only runtime dependency that package has beyond the standard library. +PyYAML==6.0.2 diff --git a/modules/ducktests/tests/ducktests_remote/README.md b/modules/ducktests/tests/ducktests_remote/README.md new file mode 100644 index 0000000000000..ff214c1604c4c --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/README.md @@ -0,0 +1,374 @@ + + +# ducktests-remote + +Run Apache Ignite ducktests against a real VM cluster, from wherever you happen to be. + +`tests/docker/run_tests.sh` covers the local Docker flow. This covers the other one: a +cluster of real machines, driven by a long `ducktape ...` command line that until now +only existed inside a Jenkins job. + +## The model, in three sentences + +The **coordinator** is the machine where this CLI runs — your laptop, a VM inside the +cluster, or a Jenkins agent. The **runner** is the host where the `ducktape` process +itself lives; it may be the coordinator (`--runner local`) or an SSH host. The +**workers** are the cluster hosts that ducktape drives over SSH and that actually run +Ignite nodes. + +All run state lives on the runner, so any coordinator can inspect, follow or stop a run +that a different coordinator started. Everything the CLI executes goes through a single +`Transport` abstraction, so `--runner local` and `--runner build-vm-01` take identical +code paths. + +## Install + +```bash +cd modules/ducktests/tests +pip install -e . # provides the `ducktests-remote` console script +``` + +Runtime dependencies: the standard library and `PyYAML`. The CLI deliberately **never +imports ducktape** — it drives ducktape on the runner, so it stays installable on a +coordinator that has none. There is a unit check that fails if that ever changes. + +## Quickstart + +Every command accepts `--dry-run`, and `--dry-run` is genuinely side-effect free: it +prints the commands it would run and the files it would generate, including the rendered +`run.sh`, the `cluster.json`, and a redacted `globals.json`. Start there. + +### 1. Describe your cluster + +Copy `examples/cluster.yaml` to `~/.ducktests-remote/config.yaml` and edit it. The +minimum: + +```yaml +cluster: + name: lab + user: max # your account; there is no `ducker` on a real VM + identity_file: ~/.ssh/id_rsa # path AS THE RUNNER SEES IT + runner: build-vm-01 # or "local" + nodes: + - host: node[01-12].dc.local +``` + +### 2. Find out what is missing + +```bash +ducktests-remote doctor +``` + +`doctor` probes the coordinator, the runner and every worker in parallel, and never +stops at the first failure. When hosts are unusable it ends with a copy-pasteable +**"what to ask your administrator"** block naming the hosts, the account, and the exact +line to append to `authorized_keys`. + +### 3. Run + +```bash +ducktests-remote run ./modules/ducktests/tests/ignitetest/tests/smoke_test.py +``` + +## The three coordinators + +**Laptop.** The runner is remote, so the key ducktape uses has to live on the runner. +Agent forwarding will not do: a detached run outlives your SSH session, and the agent +dies with it. + +```bash +ducktests-remote keys push # installs the identity on the runner, authorises it on the workers +ducktests-remote doctor +ducktests-remote run -t ./modules/ducktests/tests/ignitetest/tests/smoke_test.py --detach +ducktests-remote logs -f # reattach later, from anywhere +``` + +**A VM inside the cluster.** Set `cluster.runner: local`. The venv is created under +`state_root` on first run and populated from `docker/requirements.txt`. + +```bash +ducktests-remote --runner local doctor +ducktests-remote --runner local run ./modules/ducktests/tests/ignitetest/ +``` + +**Jenkins agent.** Use `--detach` plus `status --json`, and read the exit code. + +```bash +ducktests-remote --profile ise-perf run -t "$TC_PATHS" --detach +ducktests-remote status --json > status.json +``` + +## Migrating from the Jenkins one-liner + +Do it in two steps, and keep a working run at each one. + +**Step 1 — paste the blob verbatim.** Whatever JSON the Jenkins job passes to +`--globals`, hand it over unchanged: + +```bash +ducktests-remote run \ + --globals-json '{"project":"ise","ignite_versions":["ise-0-32"],"ssl":{"enabled":true}}' \ + --cluster-file ./49_cluster.json \ + -t ./isetest/perftests/ +``` + +`--cluster-file` is uploaded byte for byte, so an existing hand-written cluster file +keeps working. `--globals-file` reads the same JSON from a file. + +**Step 2 — split it into a profile.** Move the keys into +`~/.ducktests-remote/profiles/ise-perf.yaml`, replacing every secret with a placeholder: + +```yaml +globals: + project: ise + ignite_versions: ["ise-0-32"] + ssl: {enabled: true} + authentication: + enabled: true + username: ${env:ISE_USER} + password: ${env:ISE_PASSWORD} +``` + +Then `ducktests-remote --profile ise-perf run -t ./isetest/perftests/`. Compare the two +with `--dry-run` until the rendered `globals.json` matches, and delete the blob. + +`${env:NAME}` and `${file:PATH}` are resolved on the coordinator at launch. A missing +variable is a hard error naming the variable and the file it came from — never an empty +string, never a run that fails on authentication three hours later. + +Layering, later winning: built-in defaults → `~/.ducktests-remote/config.yaml` → +`--config` files → `--profile` files → `DTR_*` environment → command-line flags → +`-g KEY=VALUE`. Dicts deep-merge; **lists replace**, so a later layer can shrink one. +`-g` values are parsed as JSON when they parse, so `-g ssl.enabled=true` is a boolean and +`-g project=ise` is a string. + +## Secrets + +- The composed `globals.json` is written to the run directory with mode `0600`. +- Any value resolved from `${env:}` or `${file:}` is registered with a redactor and + replaced with `***` in everything the CLI prints — including `--dry-run` output, log + streaming and error messages. Redaction is keyed on the *value*, so a password that + leaks into an unrelated field is still caught. Key-name matching is only a fallback. +- `fetch` always excludes `globals.json`. +- The example profiles in `examples/` contain no real hostnames, addresses, accounts or + passwords. Keep it that way: this directory is in a public Apache repository. + +## Deploying distributions + +`deploy` is deliberately dumb. Each subdirectory of `--dist-dir` is copied verbatim to +`/`; the name is never interpreted or checked against version +parsing, so you name the directories to match what the tests expect. + +``` +dist/ +├── ignite-dev/ -> /opt/ignite-dev +├── ignite-2.17.0/ -> /opt/ignite-2.17.0 +└── ise-0-32/ -> /opt/ise-0-32 +``` + +```bash +ducktests-remote deploy --dry-run # plan and total bytes, transfers nothing +ducktests-remote deploy --only ignite-dev +ducktests-remote deploy --via build-vm-01 # upload once, fan out from there +ducktests-remote deploy --sudo --owner max # when /opt is root-owned +``` + +Each host gets a `.ducktests-deploy.json` manifest (sorted paths + sizes + mtimes; +`--checksum` hashes contents instead). Hosts whose manifest already matches are skipped +unless `--force`. Extraction goes to a temporary directory and is then swapped into +place, because a half-copied distribution that looks present is worse than an absent one. + +On a twelve-host cluster a 300 MB distribution is 3.7 GB over the wire from a laptop. +`deploy` prints that total before it starts, and suggests `--via`. + +### Where the directory names come from + +`ignitetest` resolves a distribution home as `/`, where `product` +is `str(IgniteVersion(version))` (`services/utils/path.py`, `services/utils/ignite_aware.py`). +`IgniteVersion.__str__` **normalises**, so: + +| `ignite_versions` entry | directory under `/opt` | +| --- | --- | +| `dev` | `ignite-dev` | +| `2.17.0` | `ignite-2.17.0` | +| `ise-0-32` | `ise-0-32` | +| `ise--6` | `ise-6` — note the collapsed dash | + +A fork can override `product`, so `doctor` reports a missing directory as a WARN listing +what it *did* find under the install root rather than failing on a guessed mapping. + +## Provisioning + +`modules/ducktests/tests/docker/Dockerfile` is the source of truth for what a prepared +node looks like; the package list in `config.py` is derived from it and carries a comment +saying so. `provision` is not Ansible and must not become it. + +```bash +ducktests-remote provision --dry-run # recommended first invocation +ducktests-remote provision --sudo --only packages --only dirs +ducktests-remote provision --only ssh-env +ducktests-remote provision --sudo --write-hosts +``` + +| Step | What it does | +| --- | --- | +| `packages` | Installs the Dockerfile's system utilities. Detects apt/dnf/yum; an unknown package manager is a clear failure, not a guess. Needs `--sudo`. | +| `jdk` | Verifies `java -version` matches the expected major. Installing is opt-in (`--install-jdk`) because where a JDK comes from is site-specific. | +| `python` | Verifies only. Workers do not need Python — ducktape drives them over plain SSH. The runner's venv is created by `run`. | +| `user` | `--create-user NAME` plus `--authorize-key`. Not run by default; most operators use their own account. Needs `--sudo`. | +| `ssh-env` | Writes `PATH`/`JAVA_HOME` into `~/.ssh/environment`. **The one that is easiest to forget.** | +| `dirs` | Creates and chowns `/mnt/service` and the install root. Needs `--sudo`. | +| `hosts` | `--write-hosts` rewrites only the block between `# BEGIN ducktests-remote` and `# END ducktests-remote` in `/etc/hosts`. Needs `--sudo`. | + +Anything needing root goes through `sudo -n`. If that fails, the step is reported as +`no-sudo`, skipped, and the remaining steps still run — a partial provision with an +honest report beats an all-or-nothing failure. `provision` always finishes by running +the `doctor` checks, so it ends with evidence rather than an assumption. + +### Why `ssh-env` matters + +ducktape runs every command over **non-interactive** SSH, where `~/.profile` is not +sourced. A `java` that works fine when you log in by hand is simply absent during a test +run, and the failure surfaces as an unrelated timeout. The Dockerfile solves this with +`PermitUserEnvironment yes` plus `~/.ssh/environment`; this step does the same and then +proves it by running `java -version` non-interactively. + +## Privileges the tests actually need + +Grepped from the `ignitetest` sources, not assumed: + +- **Ordinary unprivileged account** for everything except the two suites below. It needs + write access to `persistent_root` (default `/mnt/service`) and read access to + `install_root` (default `/opt`). +- **Passwordless `sudo` for `iptables`** only, and only for the network-segmentation + suites: `ignitetest/tests/discovery_test.py` and + `ignitetest/tests/cellular_affinity_test.py`. They reach `sudo iptables`, + `iptables-save` and `iptables-restore` through `IgniteAwareService.drop_network` + (`services/utils/ignite_aware.py`). +- **Write access to `install_root`** only if you use `deploy`; otherwise `deploy --sudo`. + +Nothing else in `ignitetest` needs root. If your administrator is offering you a +privileged account you do not need, this is the list to show them. + +## Cleaning up + +```bash +ducktests-remote clean --dry-run # prints exactly what it would kill and remove +ducktests-remote clean +``` + +Kills processes matching `clean.process_pattern` (default `org.apache.ignite`, which +covers `CommandLineStartup`, `CdcCommandLineStartup`, `IgniteAwareApplicationService` and +`KafkaToIgniteCommandLineStartup`), then removes `clean.paths` (default `/mnt/service`). + +Every path is checked against `clean.allowed_roots` before it is sent anywhere, and the +roots themselves are not removable. A bug here would delete distributions across every +machine at once, so the rule is deliberately blunt. + +`stop` runs `clean` afterwards unless you pass `--no-clean`. + +## Ctrl-C during `--follow` + +`run` launches detached from second zero and then attaches to the log. Therefore: + +- **Ctrl-C detaches. It does not stop the run.** The CLI prints how to reattach and how + to stop. +- A **second Ctrl-C within 3 seconds** offers to stop the run (and, in a non-interactive + shell, simply detaches). +- `--detach` skips the following entirely. + +Reattach with `ducktests-remote logs -f`; stop with +`ducktests-remote stop `. + +## Run directory + +On the runner, under `/runs//`: + +``` +meta.json run id, coordinator, start time, test paths, redacted config summary +cluster.json what ducktape was given +globals.json composed globals, mode 0600 +run.sh the exact command; ssh in and rerun it to reproduce by hand +launch.sh wrapper that waits on run.sh and records exit_code +pid, pgid for stop +exit_code written when the process ends +ducktape.log combined stdout and stderr +results/ ducktape --results-root, with ducktape's own `latest` symlink inside +``` + +`/runs/latest` points at the newest run. Run ids look like +`max-20260727-141233-9f2a`. + +## Exit codes + +| Code | Meaning | +| --- | --- | +| 0 | success | +| 1 | usage or configuration error | +| 2 | preflight failed | +| 3 | reserved (this deployment has a single runner and takes no cluster lease) | +| 4 | ducktape ran and reported test failures | +| 5 | transport or infrastructure error | +| 130 | interrupted by the operator | + +4 and 5 are deliberately distinct: Jenkins needs "tests failed" separate from "the +cluster is broken". Note that ducktape itself exits `1` both for test failures and for +its own startup errors, so a `4` means "ducktape ran and exited non-zero" — the log +distinguishes the two. + +There is no cluster lease. This deployment has one runner and one operator; queueing is +the job of whatever schedules these runs. + +## Troubleshooting + +**Stale Ignite JVMs.** The single most common source of baffling failures on a shared +cluster: a previous run was killed and its JVMs are still holding ports and +`/mnt/service`. `doctor` reports this as a FAIL with the host list. Fix with +`ducktests-remote clean --dry-run` and then `clean`. + +**`identity_file` is a runner-side path.** It is the path *ducktape* will open, on the +runner. If the runner is not the coordinator, a file that exists on your laptop proves +nothing. `doctor` checks it on the runner and reports mode; `keys push` installs it. + +**Agent forwarding does not survive a detached run.** `ssh -A` gives you an agent for the +lifetime of your session. A run that lasts hours outlives it, and every subsequent +worker connection then fails. Use a real key file on the runner. + +**`java: command not found` deep inside a test.** Non-interactive SSH does not source +`~/.profile`. Run `provision --only ssh-env`. + +**Discovery failures with no useful message.** Workers that cannot resolve each other's +hostnames fail inside discovery. `doctor` runs an N-way resolution probe; `provision +--write-hosts` is the escape hatch when cluster DNS cannot be fixed. + +**"source payload is N MB, above the limit".** A build directory leaked into the sync. +Distributions go through `deploy`, never through the source sync. Adjust `--exclude` or +add a `.ducktestsignore` file at the source root. + +## Development + +```bash +cd modules/ducktests/tests +pytest ducktests_remote/checks # unit only: no network, no Docker, no ducktape +flake8 ducktests_remote +``` + +The checks live in `ducktests_remote/checks/` and are named `check_*.py` with `Check` +classes and `check_*` methods, because that is what `[pytest]` in `tox.ini` collects. +`checks/fake_transport.py` provides a recording transport; nothing in the unit checks +touches a network or a real process except the deliberate subprocess in the import guard. diff --git a/modules/ducktests/tests/ducktests_remote/__init__.py b/modules/ducktests/tests/ducktests_remote/__init__.py new file mode 100644 index 0000000000000..ddd5782f052f3 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/__init__.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +ducktests-remote: run Apache Ignite ducktests against a real VM cluster. + +This package deliberately never imports ``ducktape``. The CLI is a coordinator-side +orchestrator: it renders artifacts and drives the ``ducktape`` process that lives on the +runner. Keeping the dependency out means the CLI stays installable on a laptop that has +no ducktape at all, and lets the two be upgraded independently. +""" + +__version__ = "0.1.0" diff --git a/modules/ducktests/tests/ducktests_remote/__main__.py b/modules/ducktests/tests/ducktests_remote/__main__.py new file mode 100644 index 0000000000000..52a9da8c830ed --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/__main__.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Entry point for ``python -m ducktests_remote``.""" + +import sys + +from ducktests_remote.cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_cluster.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_cluster.py new file mode 100644 index 0000000000000..729abf68f38d5 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_cluster.py @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for the inventory and the generated ducktape cluster file.""" + +import json + +import pytest + +from ducktests_remote.cluster import (cluster_json, dumps, expand_hosts, load_nodes, + select_nodes) +from ducktests_remote.config import ConfigError + + +def _cfg(nodes, **kw): + base = {"user": "tester", "port": 22, "identity_file": "/home/tester/.ssh/id_rsa", + "nodes": nodes} + base.update(kw) + return base + + +class CheckRangeExpansion: + """``node[01-12].dc.local`` shorthand.""" + + def check_zero_padding_follows_the_lower_bound(self): + assert expand_hosts("node[01-03].dc.local") == [ + "node01.dc.local", "node02.dc.local", "node03.dc.local"] + + def check_unpadded_range(self): + assert expand_hosts("node[8-11]") == ["node8", "node9", "node10", "node11"] + + def check_a_plain_hostname_is_left_alone(self): + assert expand_hosts("10.0.0.13") == ["10.0.0.13"] + + def check_forty_nine_nodes(self): + assert len(expand_hosts("node[01-49].dc.local")) == 49 + + def check_two_nodes(self): + assert len(expand_hosts("node[01-02].dc.local")) == 2 + + def check_inverted_range_is_rejected(self): + with pytest.raises(ConfigError): + expand_hosts("node[9-2]") + + +class CheckInventory: + """Loading nodes from the config section.""" + + def check_bare_string_shorthand(self): + nodes = load_nodes(_cfg(["10.0.0.13"])) + assert nodes[0].host == "10.0.0.13" and nodes[0].user == "tester" + + def check_per_host_user_override(self): + nodes = load_nodes(_cfg([{"host": "a"}, {"host": "b", "user": "other"}])) + assert [n.user for n in nodes] == ["tester", "other"] + + def check_externally_routable_ip_falls_back_to_the_hostname(self): + nodes = load_nodes(_cfg([{"host": "a"}, {"host": "b", "ip": "10.0.0.2"}])) + assert nodes[0].externally_routable_ip == "a" + assert nodes[1].externally_routable_ip == "10.0.0.2" + + def check_duplicate_hosts_are_rejected(self): + with pytest.raises(ConfigError): + load_nodes(_cfg(["a", "a"])) + + def check_unknown_node_key_is_rejected(self): + with pytest.raises(ConfigError): + load_nodes(_cfg([{"host": "a", "usr": "typo"}])) + + def check_ip_cannot_be_combined_with_a_range(self): + with pytest.raises(ConfigError): + load_nodes(_cfg([{"host": "n[1-3]", "ip": "10.0.0.1"}])) + + +class CheckSelection: + """``--num-nodes``, the analogue of IGNITE_NUM_CONTAINERS.""" + + def check_truncation_takes_the_first_n(self): + nodes = load_nodes(_cfg(["a", "b", "c", "d"])) + assert [n.host for n in select_nodes(nodes, 2)] == ["a", "b"] + + def check_none_means_everything(self): + nodes = load_nodes(_cfg(["a", "b"])) + assert len(select_nodes(nodes, None)) == 2 + + def check_too_many_names_the_inventory_size(self): + nodes = load_nodes(_cfg(["a", "b"])) + with pytest.raises(ConfigError) as ex: + select_nodes(nodes, 5) + assert "2 hosts" in str(ex.value) + + def check_zero_is_rejected(self): + with pytest.raises(ConfigError): + select_nodes(load_nodes(_cfg(["a"])), 0) + + +class CheckClusterFile: + """The schema ducktape's JsonCluster reads.""" + + def check_shape_matches_ducktape(self): + nodes = load_nodes(_cfg([{"host": "node01.dc.local", "ip": "10.0.0.11"}])) + payload = cluster_json(nodes) + entry = payload["nodes"][0] + assert entry["externally_routable_ip"] == "10.0.0.11" + # RemoteAccountSSHConfig(host, hostname, user, port, password, identityfile) + assert set(entry["ssh_config"]) == { + "host", "hostname", "user", "port", "identityfile", "password"} + assert entry["ssh_config"]["port"] == 22 + assert entry["ssh_config"]["identityfile"] == "/home/tester/.ssh/id_rsa" + + def check_runner_side_identity_fallback(self): + nodes = load_nodes(_cfg(["a"], identity_file=None)) + payload = cluster_json(nodes, identity_file="/runner/side/key") + assert payload["nodes"][0]["ssh_config"]["identityfile"] == "/runner/side/key" + + def check_two_and_forty_nine_have_the_same_shape(self): + small = cluster_json(load_nodes(_cfg(["n[01-02]"]))) + large = cluster_json(load_nodes(_cfg(["n[01-49]"]))) + assert len(small["nodes"]) == 2 and len(large["nodes"]) == 49 + assert small["nodes"][0].keys() == large["nodes"][0].keys() + + def check_empty_inventory_is_rejected(self): + with pytest.raises(ConfigError): + cluster_json([]) + + def check_output_is_valid_json(self): + rendered = dumps(cluster_json(load_nodes(_cfg(["a", "b"])))) + assert len(json.loads(rendered)["nodes"]) == 2 diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_config.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_config.py new file mode 100644 index 0000000000000..c310f0ddd40e7 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_config.py @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for configuration discovery, layering and validation.""" + +import pytest + +from ducktests_remote.config import (ConfigError, coerce_scalar, deep_merge, env_overrides, + get_dotted, load_config, parse_document, set_dotted, + validate) + + +class CheckMerge: + """Layering semantics.""" + + def check_dicts_merge_recursively(self): + base = {"a": {"x": 1, "y": 2}, "b": 1} + overlay = {"a": {"y": 3, "z": 4}} + assert deep_merge(base, overlay) == {"a": {"x": 1, "y": 3, "z": 4}, "b": 1} + + def check_lists_replace_and_do_not_concatenate(self): + merged = deep_merge({"v": ["a", "b", "c"]}, {"v": ["d"]}) + assert merged["v"] == ["d"], "a later layer must be able to shrink a list" + + def check_base_is_not_mutated(self): + base = {"a": {"x": 1}} + deep_merge(base, {"a": {"x": 2}}) + assert base == {"a": {"x": 1}} + + +class CheckDocuments: + """Parser selection by content, not by extension.""" + + def check_json_body(self): + assert parse_document('{"cluster": {"name": "lab"}}') == {"cluster": {"name": "lab"}} + + def check_yaml_body(self): + assert parse_document("cluster:\n name: lab\n") == {"cluster": {"name": "lab"}} + + def check_broken_body_names_the_source(self): + with pytest.raises(ConfigError) as ex: + parse_document("cluster: [unclosed", source="profile.yaml") + assert "profile.yaml" in str(ex.value) + + +class CheckValidation: + """Unknown keys are a hard error with a suggestion.""" + + def check_unknown_top_level_key_is_rejected(self): + with pytest.raises(ConfigError) as ex: + validate({"clustr": {}}) + assert "clustr" in str(ex.value) and "cluster" in str(ex.value) + + def check_unknown_nested_key_is_rejected(self): + with pytest.raises(ConfigError) as ex: + validate({"cluster": {"instal_root": "/opt"}}) + assert "cluster.instal_root" in str(ex.value) + + def check_globals_are_free_form(self): + validate({"globals": {"anything_at_all": {"nested": 1}}}) + + def check_parameters_are_free_form(self): + validate({"parameters": {"whatever": 1}}) + + +class CheckDotted: + """Dotted path helpers.""" + + def check_set_creates_intermediate_dicts(self): + target = {} + set_dotted(target, "a.b.c", 1) + assert target == {"a": {"b": {"c": 1}}} + + def check_get_returns_default_for_missing(self): + assert get_dotted({"a": {"b": 1}}, "a.z", "fallback") == "fallback" + + def check_scalar_coercion(self): + assert coerce_scalar("true") is True + assert coerce_scalar("12") == 12 + assert coerce_scalar("ise") == "ise" + + +class CheckEnvironment: + """DTR_* overrides.""" + + def check_double_underscore_is_a_path_separator(self): + overlay = env_overrides({"DTR_CLUSTER__RUNNER": "build-vm-01"}) + assert overlay == {"cluster": {"runner": "build-vm-01"}} + + def check_single_underscores_survive_inside_a_key(self): + overlay = env_overrides({"DTR_RUN__MAX_PAYLOAD_MB": "50"}) + assert overlay == {"run": {"max_payload_mb": 50}} + + def check_aliases(self): + assert env_overrides({"DTR_RUNNER": "vm"}) == {"cluster": {"runner": "vm"}} + + def check_secret_variables_are_not_treated_as_config(self): + # Profiles interpolate ${env:DTR_...}; those must not become config paths. + assert env_overrides({"DTR_ISE_PASSWORD": "hunter2"}) == {} + + +class CheckLoad: + """Whole-stack layering.""" + + def check_later_config_file_wins(self, tmp_path): + first = tmp_path / "a.yaml" + first.write_text("cluster:\n name: one\n port: 2222\n", encoding="utf-8") + second = tmp_path / "b.yaml" + second.write_text("cluster:\n name: two\n", encoding="utf-8") + config = load_config(config_files=[first, second], environ={}, user_config=None) + assert config["cluster"]["name"] == "two" + assert config["cluster"]["port"] == 2222, "unrelated keys survive the overlay" + + def check_flags_beat_environment(self, tmp_path): + config = load_config(environ={"DTR_RUNNER": "from-env"}, + overrides={"cluster": {"runner": "from-flag"}}, + user_config=None) + assert config["cluster"]["runner"] == "from-flag" + + def check_user_defaults_to_the_coordinator_account_not_ducker(self): + config = load_config(environ={}, user_config=None) + assert config["cluster"]["user"], "an ssh user must always be resolved" + assert config["cluster"]["user"] != "ducker", \ + "nothing may default to the Docker image's account" + + def check_missing_config_file_is_reported(self): + with pytest.raises(ConfigError) as ex: + load_config(config_files=["/nonexistent/nope.yaml"], environ={}, user_config=None) + assert "not found" in str(ex.value) diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py new file mode 100644 index 0000000000000..2913d815fcbf6 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py @@ -0,0 +1,232 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for the deploy manifest, the clean allow-list, and provision idempotency.""" + +import json +import os +import time + +import pytest +from fake_transport import FakeTransport + +from ducktests_remote.commands import clean as clean_cmd +from ducktests_remote.commands import deploy, provision +from ducktests_remote.config import DEFAULTS, ConfigError +from ducktests_remote.fanout import CHANGED, FAILED, OK, HostResult, fanout, summarise + + +def _dist(tmp_path, name="ignite-dev", body="binary"): + root = tmp_path / name + (root / "bin").mkdir(parents=True) + (root / "bin" / "ignite.sh").write_text(body, encoding="utf-8") + (root / "libs").mkdir() + (root / "libs" / "core.jar").write_text("jar", encoding="utf-8") + return root + + +class CheckManifest: + """Skip-if-unchanged.""" + + def check_identical_trees_hash_the_same(self, tmp_path): + first = _dist(tmp_path / "a") + second = _dist(tmp_path / "b") + os.utime(second / "bin" / "ignite.sh", + (os.stat(first / "bin" / "ignite.sh").st_atime, + os.stat(first / "bin" / "ignite.sh").st_mtime)) + os.utime(second / "libs" / "core.jar", + (os.stat(first / "libs" / "core.jar").st_atime, + os.stat(first / "libs" / "core.jar").st_mtime)) + assert deploy.build_manifest(first)["hash"] == deploy.build_manifest(second)["hash"] + + def check_changed_content_changes_the_hash(self, tmp_path): + dist = _dist(tmp_path / "a") + before = deploy.build_manifest(dist, checksum=True)["hash"] + (dist / "bin" / "ignite.sh").write_text("different", encoding="utf-8") + assert deploy.build_manifest(dist, checksum=True)["hash"] != before + + def check_changed_mtime_changes_the_default_hash(self, tmp_path): + dist = _dist(tmp_path / "a") + before = deploy.build_manifest(dist)["hash"] + os.utime(dist / "bin" / "ignite.sh", (time.time() + 60, time.time() + 60)) + assert deploy.build_manifest(dist)["hash"] != before + + def check_manifest_records_size_and_count(self, tmp_path): + manifest = deploy.build_manifest(_dist(tmp_path / "a")) + assert manifest["files"] == 2 and manifest["bytes"] > 0 + assert manifest["mode"] == "size+mtime" + + def check_checksum_mode_is_recorded(self, tmp_path): + assert deploy.build_manifest(_dist(tmp_path / "a"), + checksum=True)["mode"] == "checksum" + + +class CheckSkipLogic: + """The decision the fan-out makes per host, given a remote manifest.""" + + @staticmethod + def _remote_says(hash_value): + transport = FakeTransport() + transport.when("cat", stdout=json.dumps({"hash": hash_value})) + return transport + + def check_matching_manifest_means_skip(self): + transport = self._remote_says("abc") + existing = json.loads(transport.read_file("/opt/x/.ducktests-deploy.json")) + assert existing["hash"] == "abc" + + def check_manifest_filename_is_stable(self): + assert deploy.MANIFEST_NAME == ".ducktests-deploy.json" + + def check_swap_removes_the_old_tree_only_after_the_move(self, tmp_path): + script = deploy._swap_script("/opt/.x.tmp.1", "/opt/x", False, None) # noqa: SLF001 + move_index = script.index('mv -- "$staging" "$target"') + remove_index = script.index('rm -rf -- "$old"') + assert move_index < remove_index, \ + "a half-copied distribution that looks present is worse than an absent one" + + def check_sudo_prefixes_every_privileged_command(self): + script = deploy._swap_script("/opt/.x.tmp.1", "/opt/x", True, "max") # noqa: SLF001 + assert script.count("sudo -n ") >= 3 + assert "chown -R max" in script + + +class CheckCleanAllowList: + """A bug here deletes distributions across every machine at once.""" + + def check_default_paths_are_accepted(self): + assert clean_cmd.validated_paths(DEFAULTS["clean"]) == ["/mnt/service"] + + def check_path_outside_the_allow_list_is_rejected(self): + with pytest.raises(ConfigError): + clean_cmd.validated_paths({"paths": ["/opt/ignite-dev"], + "allowed_roots": ["/mnt"]}) + + def check_root_itself_is_rejected(self): + with pytest.raises(ConfigError): + clean_cmd.validated_paths({"paths": ["/"], "allowed_roots": ["/mnt"]}) + + def check_the_allowed_root_itself_is_not_removable(self): + with pytest.raises(ConfigError): + clean_cmd.validated_paths({"paths": ["/mnt"], "allowed_roots": ["/mnt"]}) + + def check_relative_paths_are_rejected(self): + with pytest.raises(ConfigError): + clean_cmd.validated_paths({"paths": ["service"], "allowed_roots": ["/mnt"]}) + + def check_traversal_is_normalised_away(self): + with pytest.raises(ConfigError): + clean_cmd.validated_paths({"paths": ["/mnt/../opt/ignite-dev"], + "allowed_roots": ["/mnt"]}) + + def check_dry_run_script_kills_and_removes_nothing(self): + script = clean_cmd._script("org.apache.ignite", ["/mnt/service"], # noqa: SLF001 + dry_run=True) + assert "dry=1" in script + assert 'if [ "$dry" -eq 0 ]; then rm -rf -- "$d"; fi' in script + + +class CheckProvisionIdempotency: + """Running a step twice reports ``changed`` and then ``ok``.""" + + def check_changed_then_ok(self): + first = FakeTransport() + first.when("bash", stdout="CHANGED installed: rsync jq\n") + second = FakeTransport() + second.when("bash", stdout="all 10 packages present\n") + + def classify(transport): + result = transport.run_script("script", check=False) + return CHANGED if "CHANGED" in result.stdout else OK + + assert classify(first) == CHANGED + assert classify(second) == OK + + def check_hosts_step_rewrites_only_between_its_markers(self): + class _Args: # pylint: disable=too-few-public-methods + write_hosts = True + + class _Ctx: # pylint: disable=too-few-public-methods + args = _Args() + config = {"provision": DEFAULTS["provision"]} + + from ducktests_remote.cluster import Node # pylint: disable=import-outside-toplevel + script = provision._hosts_script( # noqa: SLF001 + _Ctx(), [Node(host="node01"), Node(host="node02", ip="10.0.0.2")]) + assert provision.HOSTS_BEGIN in script and provision.HOSTS_END in script + assert "awk" in script and "/etc/hosts" in script + assert "10.0.0.2 node02" in script + assert "node01 node01" in script + assert "> /etc/hosts" not in script.replace('> "$tmp"', ""), \ + "the whole file must never be truncated" + + def check_package_list_is_derived_from_the_dockerfile(self): + packages = DEFAULTS["provision"]["packages"] + for expected in ("rsync", "unzip", "curl", "jq", "iptables", "net-tools", "coreutils"): + assert expected in packages + + def check_user_step_is_not_run_by_default(self): + class _Args: # pylint: disable=too-few-public-methods + only = [] + skip = [] + create_user = None + write_hosts = False + + steps = provision._selected_steps(_Args()) # noqa: SLF001 + assert "user" not in steps, "most operators use their own existing account" + assert "hosts" not in steps + assert "ssh-env" in steps + + def check_only_selects_exactly_one_step(self): + class _Args: # pylint: disable=too-few-public-methods + only = ["packages"] + skip = [] + create_user = None + write_hosts = False + + assert provision._selected_steps(_Args()) == ["packages"] # noqa: SLF001 + + +class CheckFanout: + """Per-host isolation and reporting.""" + + def check_one_failure_does_not_abort_the_batch(self): + def operation(host): + if host == "b": + raise RuntimeError("boom") + return HostResult(host, OK) + + results = fanout(["a", "b", "c"], operation, jobs=4) + assert [r.status for r in results] == [OK, FAILED, OK] + assert summarise(results) == "1 failed, 2 ok" + + def check_fail_fast_stops_scheduling(self): + def operation(host): + if host == "a": + return HostResult(host, FAILED, "no") + return HostResult(host, OK) + + results = fanout(["a", "b", "c"], operation, jobs=1, fail_fast=True) + assert results[0].status == FAILED + assert all(r.status != OK for r in results[1:]) + + def check_two_and_forty_nine_hosts_produce_the_same_shape(self): + small = fanout(["a", "b"], lambda h: HostResult(h, OK), jobs=4) + large = fanout(["h%02d" % i for i in range(49)], lambda h: HostResult(h, OK), jobs=8) + assert {type(r) for r in small} == {type(r) for r in large} + assert len(small) == 2 and len(large) == 49 + + def check_empty_inventory_is_not_an_error(self): + assert fanout([], lambda h: None) == [] diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_globals.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_globals.py new file mode 100644 index 0000000000000..509250428c647 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_globals.py @@ -0,0 +1,143 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for globals composition, interpolation and secret redaction.""" + +import json + +import pytest + +from ducktests_remote.config import ConfigError +from ducktests_remote.globals_builder import (Redactor, build, dumps, load_raw_layer, + parse_kv_override) + +SECRET = "s3cr3t-passphrase" + + +class CheckLayering: + """Deep merge, list replacement, override precedence.""" + + def check_later_layer_wins_per_key(self): + composed, _ = build([("base", {"ssl": {"enabled": False}, "project": "ignite"}), + ("profile", {"ssl": {"enabled": True}})]) + assert composed == {"ssl": {"enabled": True}, "project": "ignite"} + + def check_lists_replace(self): + composed, _ = build([("base", {"ignite_versions": ["2.16.0", "2.17.0"]}), + ("profile", {"ignite_versions": ["dev"]})]) + assert composed["ignite_versions"] == ["dev"] + + def check_raw_jenkins_blob_is_a_usable_base_layer(self): + blob = '{"project": "ise", "ignite_versions": ["ise-0-32"]}' + raw = load_raw_layer(blob, None) + composed, _ = build([("blob", raw), ("profile", {"project": "ignite"})]) + assert composed == {"project": "ignite", "ignite_versions": ["ise-0-32"]} + + def check_raw_blob_must_be_an_object(self): + with pytest.raises(ConfigError): + load_raw_layer("[1, 2, 3]", None) + + +class CheckOverrides: + """``-g a.b.c=value`` mechanics.""" + + def check_dotted_path_nests(self): + composed, _ = build([], ["ssl.key_store.path=/tmp/ks"]) + assert composed == {"ssl": {"key_store": {"path": "/tmp/ks"}}} + + def check_json_values_are_coerced(self): + composed, _ = build([], ["ssl.enabled=true", "count=3", "project=ise"]) + assert composed["ssl"]["enabled"] is True + assert composed["count"] == 3 + assert composed["project"] == "ise", "a bare word stays a string" + + def check_override_beats_the_layers(self): + composed, _ = build([("profile", {"project": "ignite"})], ["project=ise"]) + assert composed["project"] == "ise" + + def check_missing_equals_is_an_error(self): + with pytest.raises(ConfigError): + parse_kv_override("just-a-key") + + +class CheckInterpolation: + """``${env:}`` and ``${file:}``.""" + + def check_env_is_resolved(self): + composed, _ = build([("p", {"authentication": {"password": "${env:ISE_PASSWORD}"}})], + environ={"ISE_PASSWORD": SECRET}) + assert composed["authentication"]["password"] == SECRET + + def check_missing_env_names_the_variable_and_the_file(self): + with pytest.raises(ConfigError) as ex: + build([("profile-ise.yaml", {"p": "${env:NOT_SET_ANYWHERE}"})], environ={}) + message = str(ex.value) + assert "NOT_SET_ANYWHERE" in message and "profile-ise.yaml" in message + + def check_missing_env_never_becomes_an_empty_string(self): + with pytest.raises(ConfigError): + build([("p", {"password": "${env:ABSENT}"})], environ={}) + + def check_file_is_read_and_trimmed(self, tmp_path): + path = tmp_path / "pass.txt" + path.write_text(SECRET + "\n", encoding="utf-8") + composed, _ = build([("p", {"password": "${file:%s}" % path})], environ={}) + assert composed["password"] == SECRET + + def check_missing_file_is_an_error(self, tmp_path): + with pytest.raises(ConfigError): + build([("p", {"password": "${file:%s}" % (tmp_path / "nope")})], environ={}) + + def check_placeholders_inside_lists_are_resolved(self): + composed, _ = build([("p", {"versions": ["${env:V}"]})], environ={"V": "ise-0-32"}) + assert composed["versions"] == ["ise-0-32"] + + +class CheckRedaction: + """A resolved secret must not survive anywhere the CLI writes.""" + + def _composed(self): + redactor = Redactor() + composed, _ = build([("p", {"authentication": {"password": "${env:ISE_PASSWORD}"}, + "note": "connect with %s please" % SECRET})], + redactor=redactor, environ={"ISE_PASSWORD": SECRET}) + return composed, redactor + + def check_value_based_redaction_catches_an_unrelated_field(self): + composed, redactor = self._composed() + masked = redactor.redact_structure(composed) + assert SECRET not in json.dumps(masked) + assert masked["note"] == "connect with *** please" + + def check_rendered_output_is_clean(self): + composed, redactor = self._composed() + rendered = dumps(redactor.redact_structure(composed)) + assert SECRET not in rendered + + def check_arbitrary_text_is_redacted(self): + _, redactor = self._composed() + line = "ssh failed: tried password %s" % SECRET + assert SECRET not in redactor.redact(line) + + def check_key_name_fallback_masks_unresolved_secrets(self): + redactor = Redactor() + masked = redactor.redact_structure({"password": "typed-inline-not-from-env"}) + assert masked["password"] == "***" + + def check_the_written_globals_file_still_contains_the_real_value(self): + # Redaction is for output only; ducktape needs the real credentials on the runner, + # which is why globals.json is written with mode 0600 and excluded from fetch. + composed, _ = self._composed() + assert composed["authentication"]["password"] == SECRET diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_runs.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_runs.py new file mode 100644 index 0000000000000..5499a12a9089d --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_runs.py @@ -0,0 +1,195 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for run ids, run directory layout, state derivation, and run.sh rendering.""" + +from datetime import datetime + +from fake_transport import FakeTransport + +from ducktests_remote import runs + +GOLDEN_RUN_SH = """set -euo pipefail + +cd '/opt/my sources/ignite' +# activate the runner venv; `set +u` because older activate +# scripts read unset variables +set +u +. '/opt/venvs/dt env/bin/activate' +set -u + +exec ducktape \\ + --results-root /state/runs/r/results \\ + --cluster-file /state/runs/r/cluster.json \\ + --globals /state/runs/r/globals.json \\ + --parameters /state/runs/r/parameters.json \\ + --repeat 3 \\ + --max-parallel 4 \\ + --test-runner-timeout 900000 \\ + --debug \\ + './tests/a b.py::Cls.test' './tests/{braces}.py' +""" + + +class CheckRunId: + """Format and uniqueness.""" + + def check_format(self): + run_id = runs.new_run_id("max", now=datetime(2026, 7, 27, 14, 12, 33), entropy="9f2a") + assert run_id == "max-20260727-141233-9f2a" + assert runs.is_run_id(run_id) + + def check_unsafe_characters_in_the_user_are_replaced(self): + run_id = runs.new_run_id("DOMAIN\\user", now=datetime(2026, 1, 1), entropy="0000") + assert runs.is_run_id(run_id) + + def check_uniqueness_within_the_same_second(self): + now = datetime(2026, 7, 27, 14, 12, 33) + ids = {runs.new_run_id("max", now=now) for _ in range(200)} + assert len(ids) > 190, "the hex suffix must keep same-second runs apart" + + def check_rejects_arbitrary_directory_names(self): + assert not runs.is_run_id("latest") + assert not runs.is_run_id("results") + + +class CheckPaths: + """Run directory layout.""" + + def check_layout(self): + paths = runs.RunPaths("/state", "max-20260727-141233-9f2a") + assert paths.run_dir == "/state/runs/max-20260727-141233-9f2a" + assert paths.globals_file.endswith("/globals.json") + assert paths.results_dir.endswith("/results") + assert paths.latest_link == "/state/runs/latest" + assert paths.src_dir == "/state/src/max-20260727-141233-9f2a" + + +class CheckStateDerivation: + """The three observable facts map onto one state.""" + + def check_running(self): + assert runs.derive_state(pid_alive=True, exit_code=None, stopped=False) == runs.RUNNING + + def check_finished(self): + assert runs.derive_state(pid_alive=False, exit_code=0, stopped=False) == runs.FINISHED + + def check_failed(self): + assert runs.derive_state(pid_alive=False, exit_code=1, stopped=False) == runs.FAILED + + def check_stopped(self): + assert runs.derive_state(pid_alive=False, exit_code=143, stopped=True) == runs.STOPPED + + def check_exit_code_beats_a_reused_pid(self): + assert runs.derive_state(pid_alive=True, exit_code=0, stopped=False) == runs.FINISHED + + def check_unknown(self): + assert runs.derive_state(pid_alive=False, exit_code=None, stopped=False) == runs.UNKNOWN + + def check_read_state_parses_the_probe_output(self): + transport = FakeTransport() + transport.when("bash", stdout=("pid=4242\npgid=4242\nalive=1\n" + "meta={\"test_paths\": [\"a.py\"]}\n")) + state = runs.read_state(transport, runs.RunPaths("/state", "r")) + assert state.pid == 4242 and state.state == runs.RUNNING + assert state.meta["test_paths"] == ["a.py"] + + +class CheckListing: + """Only real run directories are listed.""" + + def check_latest_symlink_is_not_a_run(self): + transport = FakeTransport() + transport.when("ls", stdout="latest\nmax-20260727-141233-9f2a\n" + "max-20260726-090000-aaaa\n") + ids = runs.list_run_ids(transport, "/state") + assert ids == ["max-20260727-141233-9f2a", "max-20260726-090000-aaaa"] + + +class CheckRunScript: + """Golden-file rendering, including paths with spaces and shell metacharacters.""" + + def _render(self): + return runs.render_run_script( + version="0.1.0", timestamp="2026-07-27T00:00:00+00:00", author="max@laptop", + work_dir="/opt/my sources/ignite", + results_root="/state/runs/r/results", + cluster_file="/state/runs/r/cluster.json", + globals_file="/state/runs/r/globals.json", + parameters_file="/state/runs/r/parameters.json", + test_paths=["./tests/a b.py::Cls.test", "./tests/{braces}.py"], + venv="/opt/venvs/dt env", + repeat=3, max_parallel=4, test_runner_timeout=900000, + extra_args=["--debug"]) + + def check_golden_body(self): + rendered = self._render() + body = rendered[rendered.index("set -euo pipefail"):] + assert body == GOLDEN_RUN_SH + + def check_header_records_provenance(self): + rendered = self._render() + assert "Generated by ducktests-remote 0.1.0" in rendered + assert "max@laptop" in rendered + + def check_globals_is_passed_as_a_file_path(self): + # ducktape 0.13 checks os.path.isfile before parsing --globals as JSON + # (command_line/main.py::get_user_defined_globals), so the blob never has to + # cross a shell command line. + rendered = self._render() + assert "--globals /state/runs/r/globals.json" in rendered + assert "{" not in rendered.split("exec ducktape")[1].replace("{braces}", "") + + def check_optional_flags_are_omitted_when_unset(self): + rendered = runs.render_run_script( + version="0.1.0", timestamp="t", author="a", work_dir="/w", + results_root="/r", cluster_file="/c", globals_file="/g", + test_paths=["./t.py"], venv=None) + assert "--parameters" not in rendered + assert "--repeat" not in rendered + assert "--max-parallel" not in rendered + assert "no venv configured" in rendered + + def check_a_json_like_test_path_is_quoted(self): + rendered = runs.render_run_script( + version="0.1.0", timestamp="t", author="a", work_dir="/w", + results_root="/r", cluster_file="/c", globals_file="/g", + test_paths=['./t.py::C.m@{"x": 1}'], venv=None) + assert """'./t.py::C.m@{"x": 1}'""" in rendered + + +class CheckLaunchScripts: + """Detachment mechanics.""" + + def check_launch_records_the_exit_code(self): + script = runs.render_launch_script(runs.RunPaths("/state", "r")) + assert 'echo $? > "$rd/exit_code"' in script + assert 'pgid' in script + + def check_detach_prefers_setsid_and_falls_back(self): + script = runs.render_detach_script(runs.RunPaths("/state", "r")) + assert "setsid nohup bash" in script + assert "disown" in script, "a runner without setsid must still detach" + assert 'echo $! > "$rd/pid"' in script + + +class CheckFormatting: + """Duration rendering.""" + + def check_durations(self): + assert runs.format_duration(None) == "-" + assert runs.format_duration(42) == "42s" + assert runs.format_duration(125) == "2m 05s" + assert runs.format_duration(3852) == "1h 04m 12s" diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_sshdiag.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_sshdiag.py new file mode 100644 index 0000000000000..3d989b4497529 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_sshdiag.py @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Table-driven checks over recorded OpenSSH stderr samples.""" + +import pytest + +from ducktests_remote import sshdiag + +SAMPLES = [ + (sshdiag.UNRESOLVED, + "ssh: Could not resolve hostname node07.dc.local: Name or service not known"), + (sshdiag.UNRESOLVED, + "ssh: Could not resolve hostname host: nodename nor servname provided, or not known"), + (sshdiag.UNRESOLVED, + "ssh: Could not resolve hostname x: Temporary failure in name resolution"), + (sshdiag.NO_SSHD, + "ssh: connect to host node03.dc.local port 22: Connection refused"), + (sshdiag.UNREACHABLE, + "ssh: connect to host node04.dc.local port 22: Connection timed out"), + (sshdiag.UNREACHABLE, + "ssh: connect to host node05 port 22: No route to host"), + (sshdiag.UNREACHABLE, + "ssh: connect to host node06 port 22: Network is unreachable"), + (sshdiag.NO_ACCESS, + "max@node02.dc.local: Permission denied (publickey,gssapi-keyex,password)."), + (sshdiag.NO_ACCESS, + "Received disconnect from 10.0.0.5 port 22:2: Too many authentication failures"), + (sshdiag.NO_USER, + "Invalid user max from 10.0.0.9 port 51234"), + (sshdiag.NO_USER, + "Please login as the user \"ec2-user\" rather than the user \"root\"."), + (sshdiag.HOSTKEY, + "@@@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @@@"), + (sshdiag.HOSTKEY, + "Host key verification failed."), + (sshdiag.NO_SUDO, + "sudo: a password is required"), + (sshdiag.NO_SUDO, + "sudo: no tty present and no askpass program specified"), + (sshdiag.NO_SUDO, + "max is not in the sudoers file. This incident will be reported."), + (sshdiag.UNKNOWN, + "some entirely novel failure nobody has seen before"), +] + + +class CheckClassification: + """Every recorded sample maps to exactly one class.""" + + @pytest.mark.parametrize("expected,stderr", SAMPLES) + def check_sample(self, expected, stderr): + assert sshdiag.classify(255, stderr) == expected + + def check_success_is_ok_regardless_of_stderr(self): + assert sshdiag.classify(0, "Warning: Permanently added 'x' to known hosts.") \ + == sshdiag.OK + + def check_advice_is_host_and_user_specific(self): + diagnosis = sshdiag.SshDiagnosis("node02", sshdiag.NO_ACCESS, user="max", port=2222) + assert "max" in diagnosis.advice + + def check_unreachable_advice_names_the_port(self): + diagnosis = sshdiag.SshDiagnosis("node02", sshdiag.UNREACHABLE, user="max", port=2222) + assert "2222" in diagnosis.advice + + +class CheckMixedCluster: + """A partially working cluster is the normal case, not an edge case.""" + + def _diagnoses(self): + return ([sshdiag.SshDiagnosis("node0%d" % i, sshdiag.OK, user="max") + for i in range(1, 10)] + + [sshdiag.SshDiagnosis("node10", sshdiag.NO_ACCESS, user="max"), + sshdiag.SshDiagnosis("node11", sshdiag.NO_ACCESS, user="max"), + sshdiag.SshDiagnosis("node12", sshdiag.NO_USER, user="max")]) + + def check_summary_counts_every_class(self): + assert sshdiag.summarise(self._diagnoses()) == "9 ok, 2 no-access, 1 no-user" + + def check_admin_block_names_the_right_hosts_per_class(self): + block = sshdiag.admin_request_block(self._diagnoses(), user="max") + no_access = block.split("does not exist")[0] + assert "node10" in no_access and "node11" in no_access + assert "node12" not in no_access, "a no-user host must not be listed as no-access" + assert "node12" in block + no_user = block.split("does not exist")[1] + assert "node10" not in no_user.split("it does exist on")[0], \ + "a no-access host must not be listed as no-user" + + def check_admin_block_mentions_the_account_and_the_authorized_keys_line(self): + block = sshdiag.admin_request_block(self._diagnoses(), user="max") + assert "'max'" in block + assert "authorized_keys" in block + + def check_admin_block_lists_hosts_that_do_have_the_account(self): + block = sshdiag.admin_request_block(self._diagnoses(), user="max") + assert "it does exist on" in block + + def check_no_block_when_everything_works(self): + healthy = [sshdiag.SshDiagnosis("node01", sshdiag.OK, user="max")] + assert sshdiag.admin_request_block(healthy, user="max") == "" + + def check_sudo_block_names_the_tests_that_need_it(self): + block = sshdiag.admin_request_block( + [sshdiag.SshDiagnosis("node01", sshdiag.NO_SUDO, user="max")], user="max") + assert "discovery_test.py" in block and "cellular_affinity_test.py" in block + + def check_hostkey_block_never_removes_anything_itself(self): + block = sshdiag.admin_request_block( + [sshdiag.SshDiagnosis("node01", sshdiag.HOSTKEY, user="max")], user="max") + assert "ssh-keygen -R node01" in block + assert "verify before removing" in block diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py new file mode 100644 index 0000000000000..d367e38e59fe5 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py @@ -0,0 +1,190 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for the transport boundary, the import guard, and exit-code mapping.""" + +import subprocess +import sys +from pathlib import Path + +import pytest +from fake_transport import FakeTransport + +from ducktests_remote import cli, runs +from ducktests_remote.transport import (LocalTransport, ProxiedTransport, Result, + SshTransport, TransportError, is_excluded) + +PACKAGE_ROOT = Path(__file__).resolve().parent.parent + + +class CheckImportGuard: + """``ducktests_remote`` must be usable on a coordinator with no ducktape at all.""" + + def check_ducktape_is_not_in_the_import_graph(self): + script = ( + "import sys\n" + "import ducktests_remote.cli as c\n" + "c.build_parser()\n" + "leaked = [m for m in sys.modules if m == 'ducktape' " + "or m.startswith('ducktape.')]\n" + "assert not leaked, leaked\n" + "print('clean')\n") + result = subprocess.run([sys.executable, "-c", script], capture_output=True, + text=True, check=False, cwd=str(PACKAGE_ROOT.parent)) + assert result.returncode == 0, result.stderr + assert "clean" in result.stdout + + def check_no_source_file_imports_ducktape(self): + offenders = [] + for path in PACKAGE_ROOT.rglob("*.py"): + if "checks" in path.parts: + continue + text = path.read_text(encoding="utf-8") + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith(("import ducktape", "from ducktape")): + offenders.append("%s: %s" % (path.name, stripped)) + assert not offenders, offenders + + +class CheckTransportEquivalence: + """Above the transport boundary, local and ssh runs are indistinguishable.""" + + def check_identical_command_sequences(self): + sequences = [] + for name in ("local", "build-vm-01"): + transport = FakeTransport(name=name) + transport.mkdirs("/state/runs/r") + transport.run(["mkdir", "-p", "/state/runs/r/results"]) + transport.write_file("{}", "/state/runs/r/globals.json", mode=0o600) + runs.read_state(transport, runs.RunPaths("/state", "r")) + sequences.append((transport.commands, sorted(transport.files))) + assert sequences[0] == sequences[1] + + +class CheckSshOptions: + """The ssh client is the system one, driven with deliberate options.""" + + def check_batch_mode_is_always_on(self): + # An interactive password prompt inside a fan-out across N hosts hangs the whole + # run, so a missing key is a diagnosable failure rather than a prompt. + opts = SshTransport(name="h").ssh_options() + assert "BatchMode=yes" in opts + + def check_identity_is_used_exclusively_when_given(self): + opts = SshTransport(name="h", identity_file="/k").ssh_options() + assert "IdentitiesOnly=yes" in opts and "/k" in opts + + def check_non_default_port_uses_the_right_flag(self): + assert "-p" in SshTransport(name="h", port=2222).ssh_options() + assert "-P" in SshTransport(name="h", port=2222).ssh_options(for_scp=True) + + def check_default_port_is_not_passed(self): + assert "-p" not in SshTransport(name="h", port=22).ssh_options() + + def check_target_includes_the_user(self): + assert SshTransport(name="h", user="max").target == "max@h" + assert SshTransport(name="h").target == "h" + + +class CheckProxiedTransport: + """``deploy --via`` and runner-side probing hop through another transport.""" + + def check_command_is_wrapped_in_ssh_on_the_intermediate_host(self): + via = FakeTransport(name="jump") + proxied = ProxiedTransport(name="node01", via=via, user="max") + proxied.run(["true"], check=False) + argv = via.commands[-1] + assert argv[0] == "ssh" and "max@node01" in argv + + +class CheckLocalTransport: + """The local transport really does run things.""" + + def check_run_captures_output(self): + transport = LocalTransport() + result = transport.run([sys.executable, "-c", "print('hi')"]) + assert result.out == "hi" + + def check_failure_raises_with_the_command_in_the_message(self): + transport = LocalTransport() + with pytest.raises(TransportError) as ex: + transport.run([sys.executable, "-c", "import sys;sys.exit(3)"]) + assert "exit 3" in str(ex.value) + + def check_missing_binary_is_a_transport_error(self): + with pytest.raises(TransportError): + LocalTransport().run(["definitely-not-a-real-binary-xyz"]) + + def check_dry_run_executes_nothing(self): + printed = [] + transport = LocalTransport(dry_run=True, printer=printed.append) + result = transport.run([sys.executable, "-c", "raise SystemExit(9)"]) + assert result.ok and printed and printed[0].startswith("[dry-run]") + + def check_tilde_expansion_uses_the_remote_home(self): + transport = FakeTransport(home="/home/tester") + assert transport.expand("~/.ducktests-remote") == "/home/tester/.ducktests-remote" + assert transport.expand("/absolute") == "/absolute" + + +class CheckExcludes: + """Sync exclusions behave like rsync patterns.""" + + @pytest.mark.parametrize("path", [".git/config", "target/classes/A.class", + "mod/target/x.jar", "a/b/__pycache__/c.pyc", + "x/y.pyc", "ignitetest.egg-info/PKG-INFO"]) + def check_excluded(self, path): + assert is_excluded(path, [".git", "target", "__pycache__", "*.pyc", "*.egg-info"]) + + @pytest.mark.parametrize("path", ["ignitetest/tests/smoke_test.py", "README.md", + "targeting/notes.txt"]) + def check_kept(self, path): + assert not is_excluded(path, [".git", "target", "__pycache__", "*.pyc"]) + + +class CheckExitCodes: + """Jenkins needs 'tests failed' to be distinguishable from 'the cluster is broken'.""" + + def check_values(self): + assert (cli.EXIT_OK, cli.EXIT_USAGE, cli.EXIT_PREFLIGHT, cli.EXIT_BUSY, + cli.EXIT_TESTS_FAILED, cli.EXIT_TRANSPORT, cli.EXIT_INTERRUPTED) \ + == (0, 1, 2, 3, 4, 5, 130) + + def check_tests_failed_is_not_a_transport_error(self): + assert cli.EXIT_TESTS_FAILED != cli.EXIT_TRANSPORT + + def check_config_error_maps_to_usage(self): + assert cli.main(["--config", "/nope/missing.yaml", "doctor"]) == cli.EXIT_USAGE + + def check_no_command_prints_help_and_reports_usage(self, capsys): + assert cli.main([]) == cli.EXIT_USAGE + assert "ducktests-remote" in capsys.readouterr().out + + def check_passthrough_split(self): + head, tail = cli.split_passthrough(["run", "-t", "a.py", "--", "--debug", "--sample", "3"]) + assert head == ["run", "-t", "a.py"] + assert tail == ["--debug", "--sample", "3"] + + def check_passthrough_absent(self): + assert cli.split_passthrough(["status"]) == (["status"], []) + + +class CheckResult: + """Result helpers.""" + + def check_ok_and_out(self): + result = Result(["true"], 0, " value \n") + assert result.ok and result.out == "value" diff --git a/modules/ducktests/tests/ducktests_remote/checks/fake_transport.py b/modules/ducktests/tests/ducktests_remote/checks/fake_transport.py new file mode 100644 index 0000000000000..a3ffa5ae4223d --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/fake_transport.py @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A recording Transport used by the unit checks. No processes, no network.""" + +import posixpath + +from ducktests_remote.transport import Result, Transport + + +class FakeTransport(Transport): + """ + Records every command and simulates a small filesystem. + + Responses are registered as ``(predicate, Result)`` pairs; anything unmatched + succeeds with empty output, which keeps the checks focused on what they assert. + """ + + def __init__(self, name="fake", home="/home/tester", **kw): + super().__init__(name=name, **kw) + self.commands = [] + self.scripts = [] + self.uploads = [] + self.downloads = [] + self.dirs = [] + self.files = {} + self.responses = [] + self._home = home + + # -- registration ------------------------------------------------------------ + + def when(self, needle, stdout="", returncode=0, stderr=""): + """Reply to any command whose joined argv contains ``needle``.""" + self.responses.append((needle, Result([], returncode, stdout, stderr, self.name))) + return self + + # -- Transport --------------------------------------------------------------- + + def run(self, argv, *, check=True, timeout=None, input=None): # noqa: A002 + argv = [str(a) for a in argv] + self.commands.append(argv) + if input is not None: + self.scripts.append(input if isinstance(input, str) else input.decode("utf-8")) + joined = " ".join(argv) + for needle, canned in self.responses: + if needle in joined or (input and needle in str(input)): + result = Result(argv, canned.returncode, canned.stdout, canned.stderr, self.name) + return result.check() if check and canned.returncode else result + return Result(argv, 0, "", "", self.name) + + def upload(self, local_path, remote_path, *, mode=None): + self.uploads.append((str(local_path), remote_path, mode)) + + def download(self, remote_path, local_path): + self.downloads.append((remote_path, str(local_path))) + + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False): + self.uploads.append((str(local_dir), remote_dir, "dir")) + + def write_file(self, content, remote_path, *, mode=None): + self.files[remote_path] = (content, mode) + + def exists(self, remote_path): + return remote_path in self.files + + def mkdirs(self, remote_path, *, mode=None): + self.dirs.append(remote_path) + + def home(self): + return self._home + + def expand(self, path): + text = str(path) + if text.startswith("~/"): + return posixpath.join(self._home, text[2:]) + return text diff --git a/modules/ducktests/tests/ducktests_remote/cli.py b/modules/ducktests/tests/ducktests_remote/cli.py new file mode 100644 index 0000000000000..ab2b2a973428d --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/cli.py @@ -0,0 +1,284 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Argument parsing, the shared command context, and exit-code mapping.""" + +import argparse +import os +import sys + +from ducktests_remote import __version__ +from ducktests_remote.cluster import load_extra_hosts, load_nodes, select_nodes +from ducktests_remote.config import ConfigError, expand_path, load_config, set_dotted +from ducktests_remote.globals_builder import Redactor +from ducktests_remote.transport import TransportError, build_transport + +EXIT_OK = 0 +EXIT_USAGE = 1 +EXIT_PREFLIGHT = 2 +EXIT_BUSY = 3 # reserved: this deployment has a single runner and no leases +EXIT_TESTS_FAILED = 4 +EXIT_TRANSPORT = 5 +EXIT_INTERRUPTED = 130 + +_COLORS = {"red": "\033[31m", "green": "\033[32m", "yellow": "\033[33m", + "blue": "\033[34m", "bold": "\033[1m", "dim": "\033[2m"} +_RESET = "\033[0m" + + +class Console: + """Everything the CLI prints goes through here, so redaction cannot be bypassed.""" + + def __init__(self, *, verbose=False, quiet=False, color=True, redactor=None): + self.verbose = verbose + self.quiet = quiet + self.color = color and sys.stdout.isatty() and os.environ.get("TERM") != "dumb" + self.redactor = redactor or Redactor() + + def paint(self, text, style): + """:return: ``text`` wrapped in an ANSI style when colour is enabled.""" + if not self.color or style not in _COLORS: + return text + return "%s%s%s" % (_COLORS[style], text, _RESET) + + def out(self, message=""): + """Print a line of normal output.""" + print(self.redactor.redact(message)) + + def info(self, message): + """Print a line suppressed by ``--quiet``.""" + if not self.quiet: + self.out(message) + + def detail(self, message): + """Print a line shown only with ``--verbose``.""" + if self.verbose and not self.quiet: + self.out(self.paint(message, "dim")) + + def warn(self, message): + """Print a warning to stderr.""" + print(self.redactor.redact(self.paint("WARN " + message, "yellow")), file=sys.stderr) + + def error(self, message): + """Print an error to stderr.""" + print(self.redactor.redact(self.paint("ERROR " + message, "red")), file=sys.stderr) + + def heading(self, message): + """Print a section heading.""" + if not self.quiet: + self.out("") + self.out(self.paint(message, "bold")) + + +class Context: # pylint: disable=too-many-instance-attributes + """Config, transports and console, shared by every command.""" + + def __init__(self, config, args, console): + self.config = config + self.args = args + self.console = console + self.dry_run = bool(getattr(args, "dry_run", False)) + self.jobs = int(config.get("jobs", 16)) + self._runner = None + self._workers = {} + + # -- inventory --------------------------------------------------------------- + + @property + def cluster_cfg(self): + """:return: the ``cluster`` config section.""" + return self.config["cluster"] + + @property + def nodes(self): + """:return: worker nodes, truncated by ``--num-nodes`` when given.""" + nodes = load_nodes(self.cluster_cfg) + return select_nodes(nodes, getattr(self.args, "num_nodes", None)) + + @property + def all_nodes(self): + """:return: worker nodes plus ``extra_hosts`` (provision/deploy/clean/doctor).""" + return self.nodes + load_extra_hosts(self.cluster_cfg) + + @property + def runner_host(self): + """:return: the host ducktape will run on.""" + return self.cluster_cfg.get("runner") or "local" + + @property + def state_root(self): + """:return: the runner-side state root, ``~`` unexpanded.""" + return self.cluster_cfg.get("state_root") or "~/.ducktests-remote" + + @property + def identity_file(self): + """:return: the ssh identity path *as the runner sees it*.""" + return self.cluster_cfg.get("identity_file") + + # -- transports -------------------------------------------------------------- + + @property + def runner(self): + """:return: a transport to the runner, created once.""" + if self._runner is None: + self._runner = build_transport( + self.runner_host, + user=self.cluster_cfg.get("user") if self.runner_host != "local" else None, + port=self.cluster_cfg.get("port", 22), + identity_file=self._coordinator_identity(), + connect_timeout=self.config["ssh"]["connect_timeout"], + dry_run=self.dry_run, verbose=self.console.verbose, + printer=self.console.out) + return self._runner + + def worker(self, node): + """:return: a transport to one worker, created once per host.""" + if node.host not in self._workers: + self._workers[node.host] = build_transport( + node.host, user=node.user, port=node.port, + identity_file=self._coordinator_identity(node.identity_file), + connect_timeout=self.config["ssh"]["connect_timeout"], + dry_run=self.dry_run, verbose=self.console.verbose, + printer=self.console.out) + return self._workers[node.host] + + def _coordinator_identity(self, identity=None): + """ + The configured ``identity_file`` is a runner-side path; it may or may not name a + file that exists here. Use it when it does, otherwise let the system ssh client + fall back to ``~/.ssh/config`` and the agent. + """ + candidate = expand_path(identity or self.identity_file) + if candidate and os.path.isfile(candidate): + return candidate + return None + + def state_root_resolved(self): + """:return: the state root with ``~`` expanded against the runner's home.""" + return self.runner.expand(self.state_root) + + +def _common_parser(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--config", action="append", default=argparse.SUPPRESS, metavar="FILE", + help="configuration file (YAML or JSON); repeatable, applied in order") + parser.add_argument("--profile", action="append", default=argparse.SUPPRESS, metavar="NAME", + help="named profile from profiles_dir; repeatable, applied in order") + parser.add_argument("--runner", default=argparse.SUPPRESS, metavar="HOST", + help="host running ducktape, or 'local'; overrides the config") + parser.add_argument("--jobs", type=int, default=argparse.SUPPRESS, metavar="N", + help="fan-out parallelism (default 16)") + parser.add_argument("-v", "--verbose", action="store_true", default=argparse.SUPPRESS, + help="show every command and per-host output") + parser.add_argument("-q", "--quiet", action="store_true", default=argparse.SUPPRESS, + help="only print results and errors") + parser.add_argument("--dry-run", action="store_true", default=argparse.SUPPRESS, + help="print every command and generated file, execute nothing") + parser.add_argument("--no-color", action="store_true", default=argparse.SUPPRESS, + help="disable ANSI colour") + parser.add_argument("--fail-fast", action="store_true", default=argparse.SUPPRESS, + help="abort a fan-out after the first host failure") + return parser + + +def build_parser(): + """:return: the fully wired argument parser.""" + common = _common_parser() + parser = argparse.ArgumentParser( + prog="ducktests-remote", parents=[common], + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Run Apache Ignite ducktests against a real VM cluster.", + epilog="Start with `ducktests-remote doctor` and `--dry-run`.") + parser.add_argument("--version", action="version", version="ducktests-remote " + __version__) + subparsers = parser.add_subparsers(dest="command", metavar="") + + # Imported here so `ducktests_remote.cli` stays importable without the command + # modules having been loaded, which keeps the import-guard test cheap. + from ducktests_remote.commands import ( # pylint: disable=import-outside-toplevel + clean, deploy, doctor, fetch, keys, logs, provision, run, status, stop) + + for module in (run, status, logs, fetch, stop, provision, deploy, clean, doctor, keys): + module.register(subparsers, common) + + return parser + + +def _flag_overrides(args): + """:return: a config overlay built from the explicit command-line flags.""" + overlay = {} + if getattr(args, "runner", None): + set_dotted(overlay, "cluster.runner", args.runner) + if getattr(args, "jobs", None): + set_dotted(overlay, "jobs", int(args.jobs)) + for flag, dotted in (("install_root", "cluster.install_root"), + ("state_root", "cluster.state_root"), + ("dist_dir", "deploy.dist_dir"), + ("source_root", "run.source_root"), + ("work_dir", "run.work_dir")): + value = getattr(args, flag, None) + if value: + set_dotted(overlay, dotted, value) + return overlay + + +def split_passthrough(argv): + """ + Split ``-- `` off the end of the command line. + + argparse cannot express "everything after the first bare ``--`` belongs to another + program" without swallowing legitimate arguments, so it is done up front. + """ + if "--" not in argv: + return argv, [] + index = argv.index("--") + return argv[:index], argv[index + 1:] + + +def main(argv=None): + """CLI entry point. :return: the process exit code.""" + argv = list(sys.argv[1:] if argv is None else argv) + argv, passthrough = split_passthrough(argv) + + parser = build_parser() + args = parser.parse_args(argv) + args.passthrough = passthrough + + if not getattr(args, "command", None): + parser.print_help() + return EXIT_USAGE + + console = Console(verbose=getattr(args, "verbose", False), + quiet=getattr(args, "quiet", False), + color=not getattr(args, "no_color", False)) + try: + config = load_config(config_files=getattr(args, "config", None) or [], + profiles=getattr(args, "profile", None) or [], + overrides=_flag_overrides(args)) + ctx = Context(config, args, console) + return args.handler(ctx) + except ConfigError as ex: + console.error(str(ex)) + return EXIT_USAGE + except TransportError as ex: + console.error(str(ex)) + if console.verbose: + raise + return EXIT_TRANSPORT + except KeyboardInterrupt: + console.out("") + console.error("interrupted") + return EXIT_INTERRUPTED + except BrokenPipeError: + return EXIT_OK diff --git a/modules/ducktests/tests/ducktests_remote/cluster.py b/modules/ducktests/tests/ducktests_remote/cluster.py new file mode 100644 index 0000000000000..b5d56ab22b83e --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/cluster.py @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Cluster inventory: from the YAML ``cluster.nodes`` list to ducktape's ``cluster.json``. + +Nothing here assumes a cluster size. Two nodes and forty-nine nodes go down the same +code path and produce the same output shape. +""" + +import json +import re +from dataclasses import dataclass +from typing import List, Optional + +from ducktests_remote.config import ConfigError + +_RANGE = re.compile(r"\[(\d+)-(\d+)\]") + + +@dataclass(frozen=True) +class Node: + """One worker host as ducktape will see it.""" + + host: str + ip: Optional[str] = None + user: Optional[str] = None + port: int = 22 + identity_file: Optional[str] = None + + @property + def target(self): + """:return: ``user@host`` for ssh, or the bare host when no user is set.""" + return "%s@%s" % (self.user, self.host) if self.user else self.host + + @property + def externally_routable_ip(self): + """:return: the ip when the inventory gave one, else the hostname.""" + return self.ip or self.host + + def to_ssh_config(self): + """:return: the ``ssh_config`` block ducktape's RemoteAccountSSHConfig accepts.""" + return { + "host": self.host, + "hostname": self.host, + "user": self.user, + "port": int(self.port), + "identityfile": self.identity_file, + "password": "", + } + + +def expand_hosts(pattern): + """ + Expand ``node[01-12].dc.local`` into twelve hostnames. + + Zero padding follows the width of the lower bound, so ``[01-12]`` yields ``node01`` + through ``node12`` and ``[1-12]`` yields ``node1`` through ``node12``. Several + ranges in one pattern expand as a cartesian product, left to right. + """ + match = _RANGE.search(pattern) + if not match: + return [pattern] + start_raw, end_raw = match.group(1), match.group(2) + start, end = int(start_raw), int(end_raw) + if end < start: + raise ConfigError("invalid host range %r: %d > %d" % (pattern, start, end)) + width = len(start_raw) + expanded = [] + for value in range(start, end + 1): + replaced = pattern[:match.start()] + str(value).zfill(width) + pattern[match.end():] + expanded.extend(expand_hosts(replaced)) + return expanded + + +def load_nodes(cluster_cfg) -> List[Node]: + """ + Build the node list from the ``cluster`` config section. + + Entries may be a bare string (``10.0.0.13`` or ``node[01-12].dc.local``) or a mapping + with ``host`` and optional ``ip`` / ``user`` / ``port`` / ``identity_file``. Per-host + overrides exist because mixed clusters, where an account exists on some machines and + not others, are the normal case rather than the exception. + """ + default_user = cluster_cfg.get("user") + default_port = cluster_cfg.get("port", 22) + default_identity = cluster_cfg.get("identity_file") + + nodes = [] + for index, entry in enumerate(cluster_cfg.get("nodes") or []): + if isinstance(entry, str): + entry = {"host": entry} + if not isinstance(entry, dict): + raise ConfigError("cluster.nodes[%d]: expected a string or a mapping, found %s" + % (index, type(entry).__name__)) + host = entry.get("host") + if not host: + raise ConfigError("cluster.nodes[%d]: missing 'host'" % index) + unknown = set(entry) - {"host", "ip", "user", "port", "identity_file"} + if unknown: + raise ConfigError("cluster.nodes[%d]: unknown keys %s" + % (index, ", ".join(sorted(unknown)))) + expanded = expand_hosts(host) + if len(expanded) > 1 and entry.get("ip"): + raise ConfigError("cluster.nodes[%d]: 'ip' cannot be combined with a host range" + % index) + for name in expanded: + nodes.append(Node(host=name, + ip=entry.get("ip"), + user=entry.get("user", default_user), + port=int(entry.get("port", default_port)), + identity_file=entry.get("identity_file", default_identity))) + + seen = set() + for node in nodes: + if node.host in seen: + raise ConfigError("cluster.nodes: duplicate host %r" % node.host) + seen.add(node.host) + return nodes + + +def load_extra_hosts(cluster_cfg) -> List[Node]: + """:return: hosts targeted by provision/deploy/clean/doctor but kept out of cluster.json.""" + sub = dict(cluster_cfg) + sub["nodes"] = cluster_cfg.get("extra_hosts") or [] + return load_nodes(sub) + + +def select_nodes(nodes, num_nodes=None): + """ + Take the first ``num_nodes`` inventory entries. + + This is the analogue of ``IGNITE_NUM_CONTAINERS`` in ``docker/run_tests.sh``. + """ + if num_nodes is None: + return list(nodes) + if num_nodes < 1: + raise ConfigError("--num-nodes must be at least 1") + if num_nodes > len(nodes): + raise ConfigError( + "--num-nodes %d exceeds the inventory, which lists %d host%s. " + "Add hosts to cluster.nodes or lower --num-nodes." + % (num_nodes, len(nodes), "" if len(nodes) == 1 else "s")) + return list(nodes[:num_nodes]) + + +def cluster_json(nodes, identity_file=None): + """ + :param nodes: the selected :class:`Node` list. + :param identity_file: runner-side fallback identity for nodes without their own. + :return: the ducktape cluster file as a dict. + + The schema is the one ``ducktape.cluster.json.JsonCluster`` reads: a ``nodes`` list of + ``{externally_routable_ip, ssh_config}``, where ``ssh_config`` is passed straight into + ``RemoteAccountSSHConfig(host, hostname, user, port, password, identityfile)``. + """ + if not nodes: + raise ConfigError("cluster.nodes is empty; there is nothing for ducktape to run on") + payload = {"nodes": []} + for node in nodes: + ssh_config = node.to_ssh_config() + if not ssh_config.get("identityfile"): + ssh_config["identityfile"] = identity_file + payload["nodes"].append({ + "externally_routable_ip": node.externally_routable_ip, + "ssh_config": ssh_config, + }) + return payload + + +def dumps(payload): + """:return: the cluster file rendered as JSON text.""" + return json.dumps(payload, indent=2, sort_keys=True) + "\n" diff --git a/modules/ducktests/tests/ducktests_remote/commands/__init__.py b/modules/ducktests/tests/ducktests_remote/commands/__init__.py new file mode 100644 index 0000000000000..ddddae46b451c --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI subcommands. Each module exposes ``register(subparsers, common)``.""" diff --git a/modules/ducktests/tests/ducktests_remote/commands/clean.py b/modules/ducktests/tests/ducktests_remote/commands/clean.py new file mode 100644 index 0000000000000..a215c37d4d959 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/clean.py @@ -0,0 +1,138 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``clean`` - kill leftover Ignite JVMs and remove work directories. + +Stale JVMs from an aborted run are the single most common source of baffling failures on +a shared VM cluster, and the second most common is a work directory that still holds a +previous run's persistence. Both are removed here, and only here. +""" + +import posixpath +import shlex + +from ducktests_remote.cli import EXIT_OK, EXIT_TRANSPORT +from ducktests_remote.config import IGNITE_MAIN_CLASSES, ConfigError +from ducktests_remote.fanout import (CHANGED, FAILED, HostResult, OK, any_failed, fanout, + render_table, summarise) + + +def register(subparsers, common): + """Wire up the ``clean`` subcommand.""" + parser = subparsers.add_parser( + "clean", parents=[common], help="kill stale Ignite processes and remove work dirs", + description="Fan out across the workers, terminate anything matching " + "clean.process_pattern, and remove clean.paths. Run it with --dry-run " + "first: it prints exactly what it would kill and delete.") + parser.add_argument("-n", "--num-nodes", type=int, default=None, + help="only clean the first N inventory hosts") + parser.add_argument("--keep-paths", action="store_true", + help="kill processes but leave the work directories in place") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Clean the workers. :return: the process exit code.""" + nodes = ctx.all_nodes + if not nodes: + raise ConfigError("cluster.nodes is empty; there is nothing to clean") + results = clean_hosts(ctx, nodes, dry_run=ctx.dry_run, + remove_paths=not ctx.args.keep_paths) + return EXIT_TRANSPORT if any_failed(results) else EXIT_OK + + +def clean_hosts(ctx, nodes, *, dry_run=False, remove_paths=True): + """Run the clean fan-out. :return: the per-host results.""" + pattern = ctx.config["clean"]["process_pattern"] + paths = validated_paths(ctx.config["clean"]) if remove_paths else [] + script = _script(pattern, paths, dry_run=dry_run) + + ctx.console.info("pattern: %s" % pattern) + ctx.console.info("paths : %s" % (", ".join(paths) if paths else "(none)")) + if dry_run: + ctx.console.info("--dry-run: nothing will be killed or removed") + + def operation(node): + result = ctx.worker(node).run_script(script, check=False) + if not result.ok: + return HostResult(node.host, FAILED, "clean failed", detail=result.stderr.strip()) + lines = [ln for ln in result.stdout.splitlines() if ln.strip()] + killed = [ln for ln in lines if ln.startswith("proc ")] + removed = [ln for ln in lines if ln.startswith("path ")] + status = CHANGED if (killed or removed) else OK + message = "%d process(es), %d path(s)" % (len(killed), len(removed)) + return HostResult(node.host, status, message, detail="\n".join(lines)) + + results = fanout(nodes, operation, jobs=ctx.jobs, + fail_fast=getattr(ctx.args, "fail_fast", False)) + ctx.console.out(render_table(results, verbose=ctx.console.verbose)) + ctx.console.out("") + ctx.console.out(summarise(results)) + return results + + +def validated_paths(clean_cfg): + """ + Check every configured path against the allow-list before it is ever sent to a host. + + A bug here deletes distributions across every machine in the cluster at once, so the + rule is deliberately blunt: a path must sit under one of ``clean.allowed_roots``, and + the roots themselves are not removable. + """ + allowed = [posixpath.normpath(p) for p in clean_cfg.get("allowed_roots") or []] + checked = [] + for raw in clean_cfg.get("paths") or []: + path = posixpath.normpath(str(raw)) + if not path.startswith("/"): + raise ConfigError("clean.paths: %r must be absolute" % raw) + if path in ("/", "") or path in allowed: + raise ConfigError("clean.paths: refusing to remove %r" % raw) + if not any(path == root or path.startswith(root.rstrip("/") + "/") for root in allowed): + raise ConfigError( + "clean.paths: %r is outside clean.allowed_roots (%s). Add the root " + "explicitly if you really mean it." % (raw, ", ".join(allowed))) + checked.append(path) + return checked + + +def _script(pattern, paths, *, dry_run): + quoted_paths = " ".join(shlex.quote(p) for p in paths) + return """set -u +pattern=%(pattern)s +dry=%(dry)d +pids=$(pgrep -f "$pattern" 2>/dev/null || true) +for p in $pids; do + cmd=$(ps -o args= -p "$p" 2>/dev/null | cut -c1-100) + echo "proc $p $cmd" +done +if [ "$dry" -eq 0 ] && [ -n "$pids" ]; then + kill -TERM $pids 2>/dev/null || true + sleep 5 + left=$(pgrep -f "$pattern" 2>/dev/null || true) + [ -n "$left" ] && kill -KILL $left 2>/dev/null || true +fi +for d in %(paths)s; do + [ -e "$d" ] || continue + echo "path $d ($(du -sh "$d" 2>/dev/null | cut -f1))" + if [ "$dry" -eq 0 ]; then rm -rf -- "$d"; fi +done +exit 0 +""" % {"pattern": shlex.quote(pattern), "dry": 1 if dry_run else 0, "paths": quoted_paths} + + +def known_main_classes(): + """:return: the Ignite main classes ignitetest launches, for documentation and help.""" + return IGNITE_MAIN_CLASSES diff --git a/modules/ducktests/tests/ducktests_remote/commands/deploy.py b/modules/ducktests/tests/ducktests_remote/commands/deploy.py new file mode 100644 index 0000000000000..a52072892be85 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/deploy.py @@ -0,0 +1,307 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``deploy`` - push distributions from a coordinator-local directory to the install root. + +Deliberately dumb. Each subdirectory of ``--dist-dir`` is copied verbatim to +``/``; the name is never interpreted, rewritten, or checked against +version-parsing logic. The operator names the directories to match what the tests +expect, which is also what makes fork layouts work without special cases. +""" + +import hashlib +import json +import os +import posixpath +import shlex +import tempfile +import uuid +from pathlib import Path + +from ducktests_remote.cli import EXIT_OK, EXIT_TRANSPORT, EXIT_USAGE +from ducktests_remote.config import ConfigError, expand_path +from ducktests_remote.fanout import (CHANGED, FAILED, HostResult, SKIPPED, any_failed, + fanout, render_table, summarise) +from ducktests_remote.transport import ProxiedTransport, make_tarball + +MANIFEST_NAME = ".ducktests-deploy.json" + + +def register(subparsers, common): + """Wire up the ``deploy`` subcommand.""" + parser = subparsers.add_parser( + "deploy", parents=[common], help="copy distributions to the workers", + description="Copy each subdirectory of --dist-dir to / on " + "every worker, skipping hosts that already have identical content.") + parser.add_argument("--dist-dir", metavar="PATH", + help="directory holding one subdirectory per distribution") + parser.add_argument("--only", action="append", default=[], metavar="NAME", + help="restrict to this distribution; repeatable") + parser.add_argument("--install-root", metavar="PATH", help="target root on the workers") + parser.add_argument("--via", metavar="HOST", + help="upload once to HOST, then fan out from there") + parser.add_argument("--sudo", action="store_true", + help="prefix remote commands with `sudo -n`") + parser.add_argument("--owner", metavar="USER", help="chown -R the extracted tree") + parser.add_argument("--force", action="store_true", + help="redeploy even when the manifest already matches") + parser.add_argument("--checksum", action="store_true", + help="hash file contents for the manifest instead of size+mtime") + parser.add_argument("-n", "--num-nodes", type=int, default=None, + help="only deploy to the first N inventory hosts") + parser.add_argument("--json", action="store_true", help="machine-readable output") + parser.set_defaults(handler=execute) + + +def execute(ctx): # pylint: disable=too-many-locals + """Deploy distributions. :return: the process exit code.""" + args = ctx.args + console = ctx.console + + dist_dir = Path(expand_path(args.dist_dir or ctx.config["deploy"]["dist_dir"])) + if not dist_dir.is_dir(): + raise ConfigError("--dist-dir %s does not exist" % dist_dir) + + install_root = (args.install_root or ctx.config["deploy"].get("install_root") + or ctx.cluster_cfg.get("install_root", "/opt")) + nodes = ctx.all_nodes + if not nodes: + raise ConfigError("cluster.nodes is empty; there is nowhere to deploy") + + dists = _distributions(dist_dir, args.only) + if not dists: + console.error("no distributions found under %s" % dist_dir) + return EXIT_USAGE + + use_checksum = args.checksum or ctx.config["deploy"].get("checksum", False) + plans = [] + for name in dists: + manifest = build_manifest(dist_dir / name, checksum=use_checksum) + plans.append((name, manifest)) + + _print_cost(ctx, plans, nodes) + + overall = [] + for name, manifest in plans: + console.heading("%s -> %s/%s" % (name, install_root, name)) + results = _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes) + overall.extend(results) + console.out(render_table(results, verbose=console.verbose)) + console.out(summarise(results)) + + if args.json: + console.out(json.dumps([{"host": r.host, "status": r.status, "message": r.message} + for r in overall], indent=2)) + + return EXIT_TRANSPORT if any_failed(overall) else EXIT_OK + + +def _distributions(dist_dir, only): + names = sorted(p.name for p in dist_dir.iterdir() + if p.is_dir() and not p.name.startswith(".")) + if only: + missing = [n for n in only if n not in names] + if missing: + raise ConfigError("--only %s: not found under %s (available: %s)" + % (", ".join(missing), dist_dir, ", ".join(names) or "none")) + return [n for n in names if n in only] + return names + + +def build_manifest(path, *, checksum=False): + """ + :return: a manifest describing ``path``, used to skip hosts that already match. + + Sorted relative paths plus sizes and mtimes by default; ``--checksum`` adds a content + sha256 per file, which is exact but reads gigabytes off disk every time. + """ + entries = [] + total = 0 + root = Path(path) + for entry in sorted(root.rglob("*")): + if entry.is_dir() or entry.is_symlink(): + continue + stat = entry.stat() + rel = entry.relative_to(root).as_posix() + total += stat.st_size + if checksum: + entries.append("%s\0%d\0%s" % (rel, stat.st_size, _sha256(entry))) + else: + entries.append("%s\0%d\0%d" % (rel, stat.st_size, int(stat.st_mtime))) + digest = hashlib.sha256("\n".join(entries).encode("utf-8")).hexdigest() + return {"hash": digest, "files": len(entries), "bytes": total, + "mode": "checksum" if checksum else "size+mtime"} + + +def _sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _print_cost(ctx, plans, nodes): + total = sum(m["bytes"] for _, m in plans) + per_host = _human(total) + console = ctx.console + console.info("%d distribution(s), %s each, %d host(s) = %s total" + % (len(plans), per_host, len(nodes), _human(total * len(nodes)))) + if not ctx.args.via and len(nodes) > 3 and total > 200 * 1024 * 1024: + console.warn("that is %s over the wire from this machine. `--via ` uploads it once and fans out from there." + % _human(total * len(nodes))) + + +def _human(size): + value = float(size) + for unit in ("B", "KB", "MB", "GB", "TB"): + if value < 1024 or unit == "TB": + return "%.0f %s" % (value, unit) if unit in ("B", "KB") else "%.1f %s" % (value, unit) + value /= 1024 + return "%.1f TB" % value + + +def _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes): + args = ctx.args + target = posixpath.join(install_root, name) + manifest_body = json.dumps({ + **manifest, + "name": name, + "source": str((Path(dist_dir) / name).resolve()), + "deployed_by": os.environ.get("USER") or os.environ.get("USERNAME") or "unknown", + "deployed_at": _now(), + }, indent=2, sort_keys=True) + + if ctx.dry_run: + return [HostResult(node.host, SKIPPED, + "would send %s to %s" % (_human(manifest["bytes"]), target)) + for node in nodes] + + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / ("%s.tar.gz" % name) + make_tarball(Path(dist_dir) / name, archive) + + via_transport = None + staged_on_via = None + if args.via: + via_transport = ctx.worker(_via_node(ctx, args.via)) + staged_dir = ctx.config["deploy"]["staging_dir"] + via_transport.mkdirs(staged_dir) + staged_on_via = posixpath.join(staged_dir, "%s-%s.tar.gz" + % (name, uuid.uuid4().hex[:8])) + ctx.console.info("staging %s on %s" % (name, args.via)) + via_transport.upload(archive, staged_on_via) + + def operation(node, _archive=archive, _staged=staged_on_via, _via=via_transport): + return _deploy_to_host(ctx, node, name, target, manifest, manifest_body, + _archive, _staged, _via) + + try: + return fanout(nodes, operation, jobs=ctx.jobs, + fail_fast=getattr(ctx.args, "fail_fast", False)) + finally: + if via_transport is not None and staged_on_via: + via_transport.run(["rm", "-f", "--", staged_on_via], check=False) + + +def _via_node(ctx, host): + for node in ctx.all_nodes: + if node.host == host: + return node + from ducktests_remote.cluster import Node # pylint: disable=import-outside-toplevel + return Node(host=host, user=ctx.cluster_cfg.get("user"), + port=ctx.cluster_cfg.get("port", 22), + identity_file=ctx.cluster_cfg.get("identity_file")) + + +def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, archive, + staged_on_via, via_transport): + transport = ctx.worker(node) + remote_manifest = posixpath.join(target, MANIFEST_NAME) + + if not ctx.args.force: + existing = transport.read_file(remote_manifest) + if existing: + try: + if json.loads(existing).get("hash") == manifest["hash"]: + return HostResult(node.host, SKIPPED, "already at %s" + % manifest["hash"][:12]) + except ValueError: + pass + + install_root = posixpath.dirname(target) + writable = transport.run(["test", "-w", install_root], check=False).ok + if not writable and not ctx.args.sudo: + return HostResult(node.host, FAILED, + "%s is not writable by %s and --sudo was not passed" + % (install_root, node.user or "this account")) + + staging = "%s/.%s.tmp.%s" % (install_root, name, uuid.uuid4().hex[:8]) + + if via_transport is not None: + proxied = ProxiedTransport(name=node.host, via=via_transport, user=node.user, + port=node.port, + identity_file=node.identity_file, + staging_dir=ctx.config["deploy"]["staging_dir"], + dry_run=ctx.dry_run, verbose=ctx.console.verbose) + proxied.run_script(_prepare_script(staging, ctx.args.sudo)).check() + proxied.push_archive(staged_on_via, staging) + else: + transport.run_script(_prepare_script(staging, ctx.args.sudo)).check() + remote_archive = "%s/.payload.tar.gz" % staging + transport.upload(archive, remote_archive) + transport.run_script( + "set -eu\ntar -xzf %s -C %s\nrm -f -- %s\n" + % (shlex.quote(remote_archive), shlex.quote(staging), + shlex.quote(remote_archive))).check() + + transport.write_file(manifest_body, posixpath.join(staging, MANIFEST_NAME)) + transport.run_script(_swap_script(staging, target, ctx.args.sudo, ctx.args.owner)).check() + return HostResult(node.host, CHANGED, "%s files, %s" + % (manifest["files"], _human(manifest["bytes"]))) + + +def _prepare_script(staging, use_sudo): + sudo = "sudo -n " if use_sudo else "" + return "set -eu\n%(sudo)srm -rf -- %(staging)s\n%(sudo)smkdir -p %(staging)s\n" % { + "sudo": sudo, "staging": shlex.quote(staging)} + + +def _swap_script(staging, target, use_sudo, owner): + """ + Swap the freshly extracted tree into place, then delete the old one. + + A half-copied distribution that looks present is worse than an absent one: the tests + start, fail somewhere inside the JVM, and nobody suspects the copy. + """ + sudo = "sudo -n " if use_sudo else "" + script = """set -eu +staging=%(staging)s +target=%(target)s +old="$target.old.$$" +if [ -e "$target" ]; then %(sudo)smv -- "$target" "$old"; fi +%(sudo)smv -- "$staging" "$target" +if [ -e "$old" ]; then %(sudo)srm -rf -- "$old"; fi +""" % {"staging": shlex.quote(staging), "target": shlex.quote(target), "sudo": sudo} + if owner: + script += "%schown -R %s -- %s\n" % (sudo, shlex.quote(owner), shlex.quote(target)) + return script + + +def _now(): + from datetime import datetime, timezone # pylint: disable=import-outside-toplevel + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() diff --git a/modules/ducktests/tests/ducktests_remote/commands/doctor.py b/modules/ducktests/tests/ducktests_remote/commands/doctor.py new file mode 100644 index 0000000000000..b0ad82205169f --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/doctor.py @@ -0,0 +1,602 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``doctor`` - preflight, run standalone or implicitly before ``run``. + +Every check is driven by the inventory, so a two-node cluster produces the same output +shape as a forty-nine node one. The command never stops at the first failure: an +operator who has to ask an administrator for access needs the complete list in one go. +""" + +import json +import posixpath +import re +import shlex +import shutil +import time + +from ducktests_remote import sshdiag +from ducktests_remote.cli import EXIT_OK, EXIT_PREFLIGHT +from ducktests_remote.config import (REQUIREMENTS_RELPATH, SUDO_DEPENDENT_TESTS, + expand_path) +from ducktests_remote.fanout import HostResult, fanout +from ducktests_remote.transport import TransportError, run_local + +OK = "OK" +WARN = "WARN" +FAIL = "FAIL" + +DISK_WARN_GB = 10 +CLOCK_WARN_SEC = 5 +CLOCK_FAIL_SEC = 60 + + +class Check: + """One preflight observation.""" + + def __init__(self, scope, host, name, status, message=""): + self.scope = scope + self.host = host + self.name = name + self.status = status + self.message = message + + def as_dict(self): + """:return: a JSON-serialisable view.""" + return {"scope": self.scope, "host": self.host, "name": self.name, + "status": self.status, "message": self.message} + + +def register(subparsers, common): + """Wire up the ``doctor`` subcommand.""" + parser = subparsers.add_parser( + "doctor", parents=[common], + help="check the coordinator, the runner and every worker", + description="Check that this cluster can actually run ducktests, and when it " + "cannot, say precisely what is missing and who has to grant it.") + parser.add_argument("--json", action="store_true", help="machine-readable output") + parser.add_argument("-n", "--num-nodes", type=int, default=None, + help="only check the first N inventory hosts") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Run every check and print the report. :return: the process exit code.""" + checks, diagnoses = run_checks(ctx) + + if getattr(ctx.args, "json", False): + ctx.console.out(json.dumps({ + "checks": [c.as_dict() for c in checks], + "ssh": [{"host": d.host, "classification": d.classification, + "advice": d.advice} for d in diagnoses], + "ok": not has_failures(checks), + }, indent=2)) + else: + print_report(ctx, checks, diagnoses) + + return EXIT_PREFLIGHT if has_failures(checks) else EXIT_OK + + +def print_report(ctx, checks, diagnoses): + """Print the human-readable doctor report.""" + console = ctx.console + width_host = max([len(c.host) for c in checks] + [4]) + width_name = max([len(c.name) for c in checks] + [5]) + + last_scope = None + for check in checks: + if check.scope != last_scope: + console.heading(check.scope.upper()) + last_scope = check.scope + style = {"OK": "green", "WARN": "yellow", "FAIL": "red"}[check.status] + console.out(" %s %s %s %s" % ( + console.paint(check.status.ljust(4), style), + check.host.ljust(width_host), + check.name.ljust(width_name), + check.message)) + + counts = {} + for check in checks: + counts[check.status] = counts.get(check.status, 0) + 1 + console.out("") + console.out("%d checks: %s" % ( + len(checks), + ", ".join("%d %s" % (counts[s], s) for s in (OK, WARN, FAIL) if s in counts))) + + block = sshdiag.admin_request_block( + diagnoses, user=ctx.cluster_cfg.get("user"), + identity_file=expand_path(ctx.identity_file)) + if block: + console.out(block) + + +def has_failures(checks): + """:return: True when any check failed.""" + return any(c.status == FAIL for c in checks) + + +def run_checks(ctx, versions=None): + """ + Run every preflight check. + + :param versions: ``globals.ignite_versions`` when known, used for the install-root check. + :return: ``(checks, ssh_diagnoses)``. + """ + checks = [] + checks += coordinator_checks(ctx) + checks += runner_checks(ctx) + + nodes = ctx.all_nodes + if not nodes: + checks.append(Check("workers", "-", "inventory", FAIL, + "cluster.nodes is empty; nothing to check")) + return checks, [] + + diagnoses = probe_workers(ctx, nodes) + by_host = {d.host: d for d in diagnoses} + for node in nodes: + diagnosis = by_host.get(node.host) + if diagnosis and not diagnosis.ok: + checks.append(Check("workers", node.host, "ssh", FAIL, + "%s: %s" % (diagnosis.classification, diagnosis.advice))) + else: + checks.append(Check("workers", node.host, "ssh", OK, "reachable")) + + reachable = [n for n in nodes if by_host.get(n.host) and by_host[n.host].ok] + if reachable: + checks += worker_checks(ctx, reachable, versions or _configured_versions(ctx)) + checks += resolution_checks(ctx, reachable) + checks += runner_to_worker_checks(ctx, reachable) + return checks, diagnoses + + +# -- coordinator ------------------------------------------------------------------ + + +def coordinator_checks(ctx): + """Checks about the machine the CLI is running on.""" + if ctx.runner_host != "local": + checks = [] + for tool in ("ssh", "scp"): + found = _which(tool) + checks.append(Check("coordinator", "local", tool, + OK if found else FAIL, + "on PATH" if found else "not on PATH; install openssh-client")) + return checks + + if ctx.dry_run: + return [Check("coordinator", "local", "ducktape", OK, "not probed (--dry-run)")] + + importable, version = _local_ducktape(_venv_python(ctx)) + if not importable: + return [Check("coordinator", "local", "ducktape", FAIL, + "--runner local but ducktape is not importable here")] + pinned = pinned_ducktape_version(ctx) + status = OK if (pinned is None or version == pinned) else WARN + return [Check("coordinator", "local", "ducktape", status, + "%s (pinned %s)" % (version, pinned or "unknown"))] + + +def _which(tool): + return shutil.which(tool) is not None + + +def _local_ducktape(python): + """ + :return: ``(importable, version)`` for a locally installed ducktape. + + Done in a subprocess on purpose: ``ducktests_remote`` must never import ducktape, so + the boundary holds even for this check. + """ + result = run_local([python, "-c", + "import ducktape,sys;sys.stdout.write(ducktape.__version__)"]) + return result.ok, result.out + + +# -- runner ----------------------------------------------------------------------- + + +def runner_checks(ctx): + """Checks about the host that will run the ducktape process.""" + host = ctx.runner_host + checks = [] + try: + # `exit 0` at the end: a missing setsid is a WARN, not a reason for the whole + # probe to report the runner as unreachable. + probe = ctx.runner.run_script("for t in bash python3 setsid; do\n" + " command -v \"$t\" >/dev/null 2>&1 && echo \"$t\"\n" + "done\nexit 0\n", + check=False) + except TransportError as ex: + diagnosis = sshdiag.diagnose_exception(ex, host=host, + user=ctx.cluster_cfg.get("user"), + port=ctx.cluster_cfg.get("port", 22)) + return [Check("runner", host, "ssh", FAIL, + "%s: %s" % (diagnosis.classification, diagnosis.advice))] + + if not probe.ok and not ctx.dry_run: + detail = (probe.stderr.strip() or probe.stdout.strip() + or "exit %d with no output" % probe.returncode) + return [Check("runner", host, "reachable", FAIL, detail[:160])] + + present = set(probe.stdout.split()) + for tool in ("bash", "python3"): + checks.append(Check("runner", host, tool, OK if tool in present or ctx.dry_run else FAIL, + "present" if tool in present or ctx.dry_run else "missing")) + checks.append(Check("runner", host, "setsid", + OK if "setsid" in present or ctx.dry_run else WARN, + "present" if "setsid" in present or ctx.dry_run + else "missing; falling back to nohup + disown")) + + checks.append(_runner_ducktape_check(ctx)) + checks.append(_identity_check(ctx)) + checks.append(_runner_disk_check(ctx)) + return checks + + +def _runner_disk_check(ctx): + path = ctx.runner.expand(ctx.state_root) + result = ctx.runner.run_script( + "set -u\nmkdir -p %s 2>/dev/null || true\n" + "df -Pk %s 2>/dev/null | awk 'NR==2{printf \"%%d\", $4/1048576}'\n" + % (shlex.quote(path), shlex.quote(path)), check=False) + if ctx.dry_run: + return Check("runner", ctx.runner_host, "disk", OK, "not probed (--dry-run)") + free = _as_int(result.out) + if free is None: + return Check("runner", ctx.runner_host, "disk", WARN, + "could not read free space on %s" % path) + return Check("runner", ctx.runner_host, "disk", + WARN if free < DISK_WARN_GB else OK, "%d GB free on %s" % (free, path)) + + +def _runner_ducktape_check(ctx): + host = ctx.runner_host + pinned = pinned_ducktape_version(ctx) + python = _venv_python(ctx) + result = ctx.runner.run([python, "-c", + "import ducktape,sys;sys.stdout.write(ducktape.__version__)"], + check=False) + if ctx.dry_run: + return Check("runner", host, "ducktape", OK, "not probed (--dry-run)") + if not result.ok: + return Check("runner", host, "ducktape", FAIL, + "not importable via %s; run `ducktests-remote provision --only python`" + % python) + if pinned and result.out != pinned: + return Check("runner", host, "ducktape", WARN, + "%s installed, %s pinned in %s" % (result.out, pinned, REQUIREMENTS_RELPATH)) + return Check("runner", host, "ducktape", OK, result.out) + + +def _identity_check(ctx): + host = ctx.runner_host + identity = ctx.identity_file + if not identity: + return Check("runner", host, "identity", WARN, + "cluster.identity_file is unset; ducktape will rely on the runner's " + "ssh agent, which a detached run cannot keep") + remote = ctx.runner.expand(identity) + result = ctx.runner.run_script( + "set -e\nf=%s\n[ -f \"$f\" ] || { echo missing; exit 0; }\n" + "printf '%%s\\n' \"$(ls -l \"$f\" | cut -c1-10)\"\n" % shlex.quote(remote), + check=False) + if ctx.dry_run: + return Check("runner", host, "identity", OK, "not probed (--dry-run)") + text = result.out + if not result.ok or text == "missing": + return Check("runner", host, "identity", FAIL, + "%s does not exist on the runner. identity_file is a RUNNER-side path, " + "not a coordinator-side one; `ducktests-remote keys push` installs it." + % remote) + if len(text) >= 10 and text[4:10] != "------": + return Check("runner", host, "identity", WARN, + "%s is %s; ssh wants 0600 or stricter" % (remote, text)) + return Check("runner", host, "identity", OK, remote) + + +def _venv_python(ctx): + venv = ctx.config["runner"].get("venv") + if venv: + return posixpath.join(ctx.runner.expand(venv), "bin", "python3") + return ctx.config["runner"].get("python", "python3") + + +def pinned_ducktape_version(ctx): + """:return: the ducktape version pinned in docker/requirements.txt, or None.""" + source_root = ctx.config["run"].get("source_root") or "." + path = expand_path(posixpath.join(str(source_root).replace("\\", "/"), REQUIREMENTS_RELPATH)) + try: + with open(path, "r", encoding="utf-8") as handle: + text = handle.read() + except OSError: + return None + match = re.search(r"^ducktape==([^\s#]+)", text, re.MULTILINE) + return match.group(1) if match else None + + +# -- workers ---------------------------------------------------------------------- + + +def probe_workers(ctx, nodes): + """ + Probe every worker from the coordinator, classifying each failure. + + This is one parallel pass over the whole inventory; nothing short-circuits, because a + partial list of broken hosts cannot be turned into a single request to an + administrator. + """ + def probe(node): + transport = ctx.worker(node) + try: + result = transport.run(["true"], check=False) + diagnosis = sshdiag.diagnose(result, host=node.host, user=node.user, port=node.port) + except TransportError as ex: + diagnosis = sshdiag.diagnose_exception(ex, host=node.host, user=node.user, + port=node.port) + return HostResult(node.host, "ok" if diagnosis.ok else "failed", + diagnosis.classification, data=diagnosis) + + results = fanout(nodes, probe, jobs=ctx.jobs) + return [r.data for r in results if r.data is not None] + + +def worker_checks(ctx, nodes, versions): + """Substantive per-worker checks: java, disk, clock, stale JVMs, install dirs, sudo.""" + if ctx.dry_run: + return [Check("workers", node.host, "probe", OK, + "java, disk, clock, stale JVMs, %s (not probed: --dry-run)" + % ("install dirs for " + ", ".join(versions) if versions + else "install dirs")) + for node in nodes] + + reference = time.time() + install_root = ctx.cluster_cfg.get("install_root", "/opt") + persistent = (ctx.config["clean"]["paths"] or ["/mnt/service"])[0] + pattern = ctx.config["clean"]["process_pattern"] + + script = _WORKER_SCRIPT % { + "install_root": shlex.quote(install_root), + "persistent": shlex.quote(persistent), + "pattern": shlex.quote(pattern), + } + + def probe(node): + result = ctx.worker(node).run_script(script, check=False) + return HostResult(node.host, "ok" if result.ok else "failed", + "", detail=result.stderr, data=_parse_kv(result.stdout)) + + results = fanout(nodes, probe, jobs=ctx.jobs) + facts = {r.host: (r.data or {}) for r in results} + + checks = [] + checks += _java_checks(facts) + for host in sorted(facts): + fact = facts[host] + checks.append(_disk_from_facts(host, "install_free_gb", install_root, fact)) + checks.append(_disk_from_facts(host, "work_free_gb", persistent, fact)) + checks.append(_clock_check(host, fact, reference)) + checks.append(_stale_jvm_check(host, fact, pattern)) + checks += _privilege_checks(host, fact, ctx, install_root, persistent) + checks += _install_dir_checks(host, fact, install_root, versions) + return [c for c in checks if c is not None] + + +_WORKER_SCRIPT = """ +set -u +say() { printf '%%s=%%s\\n' "$1" "$2"; } +say epoch "$(date +%%s)" +say whoami "$(id -un 2>/dev/null || echo '?')" +say java "$(java -version 2>&1 | head -n1 | tr -d '\\r' || echo missing)" +say install_free_gb "$(df -Pk %(install_root)s 2>/dev/null | awk 'NR==2{printf "%%d", $4/1048576}')" +say install_writable "$([ -w %(install_root)s ] && echo 1 || echo 0)" +if [ -d %(persistent)s ]; then + say work_free_gb "$(df -Pk %(persistent)s 2>/dev/null | awk 'NR==2{printf "%%d", $4/1048576}')" + say work_writable "$([ -w %(persistent)s ] && echo 1 || echo 0)" +else + parent=$(dirname %(persistent)s) + say work_free_gb "$(df -Pk "$parent" 2>/dev/null | awk 'NR==2{printf "%%d", $4/1048576}')" + say work_writable "$([ -w "$parent" ] && echo 1 || echo 0)" +fi +say stale "$(pgrep -f %(pattern)s 2>/dev/null | wc -l | tr -d ' ')" +if sudo -n true 2>/dev/null; then say sudo 1; else say sudo 0; fi +say install_dirs "$(ls -1 %(install_root)s 2>/dev/null | tr '\\n' ',' )" +""" + + +def _java_checks(facts): + checks = [] + versions = {} + for host, fact in facts.items(): + java = fact.get("java", "missing") + if not java or "missing" in java or "not found" in java: + checks.append(Check("workers", host, "java", FAIL, + "java not on the non-interactive PATH; see `provision --only " + "ssh-env`")) + continue + versions.setdefault(java, []).append(host) + checks.append(Check("workers", host, "java", OK, java)) + if len(versions) > 1: + majority = max(versions, key=lambda k: len(versions[k])) + outliers = [h for k, hosts in versions.items() if k != majority for h in hosts] + checks.append(Check("workers", "-", "java-consistency", WARN, + "mixed JDKs; majority %r, outliers: %s" + % (majority, ", ".join(sorted(outliers))))) + return checks + + +def _disk_from_facts(host, key, path, fact=None): + if fact is None: + return None + free = _as_int(fact.get(key)) + if free is None: + return Check("workers", host, "disk", WARN, "could not read free space on %s" % path) + status = WARN if free < DISK_WARN_GB else OK + return Check("workers", host, "disk", status, "%d GB free on %s" % (free, path)) + + +def _clock_check(host, fact, reference): + epoch = _as_int(fact.get("epoch")) + if epoch is None: + return Check("workers", host, "clock", WARN, "could not read the clock") + skew = abs(epoch - reference) + if skew > CLOCK_FAIL_SEC: + return Check("workers", host, "clock", FAIL, "%.0fs skew; enable NTP" % skew) + if skew > CLOCK_WARN_SEC: + return Check("workers", host, "clock", WARN, "%.0fs skew" % skew) + return Check("workers", host, "clock", OK, "%.0fs skew" % skew) + + +def _stale_jvm_check(host, fact, pattern): + count = _as_int(fact.get("stale")) or 0 + if count: + return Check("workers", host, "stale-jvm", FAIL, + "%d process(es) matching %r are still running from an earlier run; " + "run `ducktests-remote clean --dry-run` then `clean`" % (count, pattern)) + return Check("workers", host, "stale-jvm", OK, "none") + + +def _privilege_checks(host, fact, ctx, install_root, persistent): + checks = [] + user = fact.get("whoami") or ctx.cluster_cfg.get("user") + checks.append(Check("workers", host, "account", OK, "connected as %s" % user)) + checks.append(Check("workers", host, "write-install", + OK if fact.get("install_writable") == "1" else WARN, + "%s %s" % (install_root, + "writable" if fact.get("install_writable") == "1" + else "NOT writable; deploy needs --sudo here"))) + checks.append(Check("workers", host, "write-work", + OK if fact.get("work_writable") == "1" else FAIL, + "%s %s" % (persistent, + "writable" if fact.get("work_writable") == "1" + else "NOT writable; tests cannot create their work dirs"))) + if fact.get("sudo") == "1": + checks.append(Check("workers", host, "sudo", OK, "passwordless")) + else: + checks.append(Check("workers", host, "sudo", WARN, + "no passwordless sudo; only the network-segmentation suites need " + "it (%s)" % ", ".join(SUDO_DEPENDENT_TESTS))) + return checks + + +def _install_dir_checks(host, fact, install_root, versions): + """ + Report the distributions found under the install root. + + ``ignitetest`` resolves a home directory as ``install_root/str(IgniteVersion(v))`` + (services/utils/path.py ``get_home_dir`` with ``IgniteAwareService.product``), and + ``IgniteVersion.__str__`` normalises the string, so ``ise--6`` maps to ``ise-6``. + Because a fork can override ``product``, a missing directory is reported as a WARN + listing what *was* found rather than as a hard failure on a guessed mapping. + """ + found = [d for d in (fact.get("install_dirs") or "").split(",") if d] + if not versions: + return [Check("workers", host, "install-root", OK, + "%d distribution(s) under %s" % (len(found), install_root))] + missing = [v for v in versions if v not in found] + if missing: + return [Check("workers", host, "install-root", WARN, + "no directory for %s under %s; found: %s" + % (", ".join(missing), install_root, ", ".join(sorted(found)) or "(none)"))] + return [Check("workers", host, "install-root", OK, + "found %s" % ", ".join(versions))] + + +def resolution_checks(ctx, nodes): + """ + Verify node-to-node name resolution from a single probe host. + + The Docker flow writes every node into ``/etc/hosts`` explicitly. On real VMs name + resolution has to already work worker-to-worker, and when it does not, tests fail + deep inside discovery with no message that points here. + """ + if len(nodes) < 2: + return [] + probe_node = nodes[0] + targets = [n.host for n in nodes if n.host != probe_node.host] + script = "for h in %s; do\n if getent hosts \"$h\" >/dev/null 2>&1; then :; " \ + "else echo \"$h\"; fi\ndone\n" % " ".join(shlex.quote(t) for t in targets) + result = ctx.worker(probe_node).run_script(script, check=False) + if ctx.dry_run: + return [Check("workers", probe_node.host, "dns", OK, "not probed (--dry-run)")] + if not result.ok: + return [Check("workers", probe_node.host, "dns", WARN, + "could not run the resolution probe")] + unresolved = [line.strip() for line in result.stdout.splitlines() if line.strip()] + if unresolved: + return [Check("workers", probe_node.host, "dns", WARN, + "cannot resolve %d peer(s) from this host: %s. Run `provision " + "--write-hosts` or fix cluster DNS." + % (len(unresolved), ", ".join(unresolved[:8])))] + return [Check("workers", probe_node.host, "dns", OK, + "resolves all %d peers" % len(targets))] + + +def runner_to_worker_checks(ctx, nodes): + """ + Verify the connection ducktape will actually make: runner to worker, with the + runner-side identity file. A coordinator that can reach a worker proves nothing + about the runner being able to. + """ + if ctx.runner_host == "local" or not nodes: + return [] + identity = ctx.runner.expand(ctx.identity_file) if ctx.identity_file else None + opts = "-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10" + if identity: + opts += " -o IdentitiesOnly=yes -i %s" % shlex.quote(identity) + lines = [] + for node in nodes: + port = "" if int(node.port) == 22 else " -p %d" % int(node.port) + lines.append("if ssh %s%s %s true >/dev/null 2>&1; then echo \"%s=1\"; " + "else echo \"%s=0\"; fi" + % (opts, port, shlex.quote(node.target), node.host, node.host)) + result = ctx.runner.run_script("\n".join(lines) + "\n", check=False) + if ctx.dry_run: + return [Check("runner->workers", ctx.runner_host, "ssh", OK, "not probed (--dry-run)")] + facts = _parse_kv(result.stdout) + checks = [] + for node in nodes: + reachable = facts.get(node.host) == "1" + checks.append(Check("runner->workers", node.host, "ssh", + OK if reachable else FAIL, + "reachable from %s" % ctx.runner_host if reachable else + "the runner cannot ssh to %s as %s with %s. This is the connection " + "ducktape makes; fix it with `ducktests-remote keys push`." + % (node.host, node.user, identity or "the default identity"))) + return checks + + +def _configured_versions(ctx): + versions = ctx.config.get("globals", {}).get("ignite_versions") + if isinstance(versions, str): + return [versions] + return list(versions or []) + + +def _parse_kv(text): + fields = {} + for line in (text or "").split("\n"): + if "=" in line: + key, value = line.split("=", 1) + fields[key.strip()] = value.strip() + return fields + + +def _as_int(value): + try: + return int(str(value).strip()) + except (TypeError, ValueError): + return None diff --git a/modules/ducktests/tests/ducktests_remote/commands/fetch.py b/modules/ducktests/tests/ducktests_remote/commands/fetch.py new file mode 100644 index 0000000000000..18f8277cf7a5d --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/fetch.py @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fetch`` - bring a run's reports back to the coordinator.""" + +import posixpath +import shlex +import tarfile +import tempfile +import uuid +from pathlib import Path + +from ducktests_remote import runs +from ducktests_remote.cli import EXIT_OK, EXIT_USAGE +from ducktests_remote.config import expand_path + +# ducktape writes these into /; see ducktape/tests/reporter.py. +DEFAULT_FILES = ("report.html", "report.txt", "report.json", "test_log.info", "session.log") + +# globals.json holds the composed secrets. It never leaves the runner. +ALWAYS_EXCLUDED = ("globals.json",) + + +def register(subparsers, common): + """Wire up the ``fetch`` subcommand.""" + parser = subparsers.add_parser( + "fetch", parents=[common], help="download a run's results", + description="Download the reports for a run. globals.json is always excluded.") + parser.add_argument("run_id", nargs="?", help="run id (default: the most recent run)") + parser.add_argument("--dest", metavar="DIR", default="./ducktests-results", + help="coordinator-side destination (default ./ducktests-results)") + parser.add_argument("--full", action="store_true", + help="download the whole results tree, not just the reports") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Fetch results. :return: the process exit code.""" + state_root = ctx.state_root_resolved() + run_id = runs.resolve_run_id(ctx.runner, state_root, ctx.args.run_id) + if not run_id: + ctx.console.error("no runs found under %s on %s" % (state_root, ctx.runner_host)) + return EXIT_USAGE + + paths = runs.RunPaths(state_root, run_id) + dest = Path(expand_path(ctx.args.dest)) / run_id + + if ctx.dry_run: + ctx.console.out("would download %s -> %s (%s), excluding %s" + % (paths.results_dir, dest, + "everything" if ctx.args.full else ", ".join(DEFAULT_FILES), + ", ".join(ALWAYS_EXCLUDED))) + return EXIT_OK + + dest.mkdir(parents=True, exist_ok=True) + # Staged inside the run directory rather than /tmp: it is known to exist, known to + # be writable by this account, and disappears with the run. + staged = paths.path(".fetch-%s.tar.gz" % uuid.uuid4().hex[:8]) + + script = _archive_script(paths, staged, full=ctx.args.full) + result = ctx.runner.run_script(script, check=False) + if not result.ok: + ctx.console.error("could not archive results on the runner: %s" + % (result.stderr.strip() or result.stdout.strip())) + return EXIT_USAGE + + with tempfile.TemporaryDirectory() as tmp: + local_archive = Path(tmp) / "results.tar.gz" + ctx.runner.download(staged, local_archive) + ctx.runner.run(["rm", "-f", "--", staged], check=False) + with tarfile.open(local_archive, "r:gz") as tar: + members = [m for m in tar.getmembers() + if posixpath.basename(m.name) not in ALWAYS_EXCLUDED] + _safe_extract(tar, members, dest) + + ctx.runner.download(paths.log_file, dest / "ducktape.log") + + ctx.console.out("results: %s" % dest.resolve()) + for report in sorted(dest.rglob("report.*")): + ctx.console.out(" %s" % report.relative_to(dest).as_posix()) + return EXIT_OK + + +def _archive_script(paths, staged, *, full): + """ + Build the tar command that stages the results on the runner. + + Archiving first and downloading one file keeps the transport free of binary + streaming, and makes the exclusion of globals.json explicit at both ends. + """ + excludes = " ".join("--exclude=%s" % shlex.quote(name) for name in ALWAYS_EXCLUDED) + # tar treats an output path containing a colon as host:path, so cd into the results + # directory and write the archive through a relative name. + out = posixpath.join("..", posixpath.basename(staged)) + common = ('set -eu\nroot=%s\n' + '[ -d "$root" ] || { echo "no results directory at $root" >&2; exit 1; }\n' + 'cd "$root"\n' % shlex.quote(paths.results_dir)) + + if full: + return common + 'tar -czf %s %s .\n' % (shlex.quote(out), excludes) + + # tar has no --include, so select the report files with find. + names = " -o ".join("-name %s" % shlex.quote(name) for name in DEFAULT_FILES) + return common + ('find . \\( %s \\) -type f -print0 | tar -czf %s %s --null -T -\n' + % (names, shlex.quote(out), excludes)) + + +def _safe_extract(tar, members, dest): + """Extract ``members`` under ``dest``, refusing anything that escapes it.""" + dest = Path(dest).resolve() + safe = [] + for member in members: + target = (dest / member.name).resolve() + if dest == target or dest in target.parents: + safe.append(member) + tar.extractall(str(dest), members=safe) # noqa: S202 - members filtered above diff --git a/modules/ducktests/tests/ducktests_remote/commands/keys.py b/modules/ducktests/tests/ducktests_remote/commands/keys.py new file mode 100644 index 0000000000000..06fb420e8d755 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/keys.py @@ -0,0 +1,135 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``keys push`` - install the runner's identity and authorise it on every worker.""" + +import argparse +import shlex +from pathlib import Path + +from ducktests_remote.cli import EXIT_OK, EXIT_TRANSPORT, EXIT_USAGE +from ducktests_remote.config import ConfigError, expand_path +from ducktests_remote.fanout import (CHANGED, HostResult, OK, any_failed, fanout, + render_table, summarise) +from ducktests_remote.transport import run_local + + +def register(subparsers, common): + """Wire up the ``keys`` subcommand.""" + parser = subparsers.add_parser( + "keys", parents=[common], help="manage the ssh key ducktape uses", + description="""Install the private key on the runner and authorise the matching +public key on every worker. + +Why this exists: ducktape connects from the runner to the workers using the identity +named in the cluster file, and a run launched with --detach outlives the coordinator's +ssh session. Agent forwarding dies with that session, so an agent-only setup produces a +run that authenticates for the first few minutes and then fails on every subsequent +connection. A real key file on the runner is the only arrangement that survives.""", + formatter_class=_RawHelp) + sub = parser.add_subparsers(dest="keys_command", metavar="") + push = sub.add_parser("push", parents=[common], + help="install the identity on the runner and authorise it on " + "the workers") + push.add_argument("--identity", metavar="PATH", + help="coordinator-side private key (default: cluster.identity_file, " + "or a generated one)") + push.add_argument("--generate", action="store_true", + help="generate a new keypair when the identity does not exist") + push.add_argument("-n", "--num-nodes", type=int, default=None, + help="only authorise on the first N inventory hosts") + push.set_defaults(handler=execute_push) + parser.set_defaults(handler=_no_action) + + +class _RawHelp(argparse.RawDescriptionHelpFormatter): + """Keeps the explanation in the help text readable.""" + + +def _no_action(ctx): + ctx.console.error("keys: expected an action, e.g. `ducktests-remote keys push`") + return EXIT_USAGE + + +def execute_push(ctx): + """Install and authorise the key. :return: the process exit code.""" + console = ctx.console + identity = Path(expand_path(ctx.args.identity or ctx.identity_file or "~/.ssh/id_rsa")) + public = Path(str(identity) + ".pub") + + if not identity.is_file(): + if not ctx.args.generate: + raise ConfigError( + "%s does not exist on this machine. Pass --generate to create a keypair, " + "or --identity to point at an existing one." % identity) + if ctx.dry_run: + console.out("[dry-run] would generate a keypair at %s" % identity) + else: + identity.parent.mkdir(parents=True, exist_ok=True) + run_local(["ssh-keygen", "-m", "PEM", "-q", "-t", "rsa", "-N", "", + "-f", str(identity)], check=True) + console.out("generated %s" % identity) + + if not public.is_file() and not ctx.dry_run: + raise ConfigError("public key %s not found next to the private key" % public) + + pubkey = public.read_text(encoding="utf-8").strip() if public.is_file() else "" + + runner_target = ctx.runner.expand(ctx.identity_file or "~/.ssh/id_rsa") + console.heading("RUNNER %s" % ctx.runner_host) + if ctx.dry_run: + console.out("[dry-run] would install %s -> %s:%s (mode 0600)" + % (identity, ctx.runner_host, runner_target)) + else: + ctx.runner.mkdirs(_parent(runner_target), mode=0o700) + ctx.runner.upload(identity, runner_target, mode=0o600) + ctx.runner.upload(public, runner_target + ".pub", mode=0o644) + console.out("installed %s (mode 0600)" % runner_target) + + console.heading("WORKERS") + results = fanout(ctx.all_nodes, lambda node: _authorize(ctx, node, pubkey), + jobs=ctx.jobs, fail_fast=getattr(ctx.args, "fail_fast", False)) + console.out(render_table(results, verbose=console.verbose)) + console.out(summarise(results)) + return EXIT_TRANSPORT if any_failed(results) else EXIT_OK + + +def _authorize(ctx, node, pubkey): + if ctx.dry_run: + return HostResult(node.host, OK, "would authorise the key for %s" % node.user) + script = """set -eu +key=%s +mkdir -p ~/.ssh +chmod 700 ~/.ssh +touch ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys +if grep -qxF "$key" ~/.ssh/authorized_keys; then + echo present +else + printf '%%s\\n' "$key" >> ~/.ssh/authorized_keys + echo added +fi +""" % shlex.quote(pubkey) + result = ctx.worker(node).run_script(script, check=False) + if not result.ok: + return HostResult(node.host, "failed", "could not write authorized_keys", + detail=result.stderr.strip()) + added = result.out == "added" + return HostResult(node.host, CHANGED if added else OK, + "authorised" if added else "already authorised") + + +def _parent(path): + return path.rsplit("/", 1)[0] or "/" diff --git a/modules/ducktests/tests/ducktests_remote/commands/logs.py b/modules/ducktests/tests/ducktests_remote/commands/logs.py new file mode 100644 index 0000000000000..06c28cb6282a9 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/logs.py @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``logs`` - print or follow a run's ``ducktape.log`` from the runner.""" + +import sys +import time + +from ducktests_remote import runs +from ducktests_remote.cli import EXIT_OK, EXIT_USAGE + +POLL_SEC = 1.5 + + +def register(subparsers, common): + """Wire up the ``logs`` subcommand.""" + parser = subparsers.add_parser( + "logs", parents=[common], help="print or follow a run's log", + description="Stream ducktape.log from the runner. Ctrl-C stops following; it " + "never touches the run.") + parser.add_argument("run_id", nargs="?", help="run id (default: the most recent run)") + parser.add_argument("-f", "--follow", action="store_true", help="keep streaming") + parser.add_argument("-n", "--lines", type=int, default=200, + help="lines of history to print first (default 200)") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Print or follow the log. :return: the process exit code.""" + state_root = ctx.state_root_resolved() + run_id = runs.resolve_run_id(ctx.runner, state_root, ctx.args.run_id) + if not run_id: + ctx.console.error("no runs found under %s on %s" % (state_root, ctx.runner_host)) + return EXIT_USAGE + + paths = runs.RunPaths(state_root, run_id) + redact = ctx.console.redactor.redact + + history = ctx.runner.run(["tail", "-n", str(ctx.args.lines), paths.log_file], check=False) + if history.stdout: + sys.stdout.write(redact(history.stdout)) + sys.stdout.flush() + + if not ctx.args.follow: + return EXIT_OK + + offset = _size(ctx, paths) + 1 + try: + while True: + chunk = ctx.runner.run(["tail", "-c", "+%d" % offset, paths.log_file], check=False) + if chunk.stdout: + offset += len(chunk.stdout.encode("utf-8")) + sys.stdout.write(redact(chunk.stdout)) + sys.stdout.flush() + state = runs.read_state(ctx.runner, paths) + if state.exit_code is not None: + ctx.console.out("") + ctx.console.out("run %s %s (ducktape exit %s)" + % (run_id, state.state, state.exit_code)) + return EXIT_OK + time.sleep(POLL_SEC) + except KeyboardInterrupt: + ctx.console.out("") + ctx.console.out("Stopped following. The run is untouched; " + "`ducktests-remote stop %s` stops it." % run_id) + return EXIT_OK + + +def _size(ctx, paths): + result = ctx.runner.run(["wc", "-c", paths.log_file], check=False) + try: + return int(result.out.split()[0]) + except (IndexError, ValueError): + return 0 diff --git a/modules/ducktests/tests/ducktests_remote/commands/provision.py b/modules/ducktests/tests/ducktests_remote/commands/provision.py new file mode 100644 index 0000000000000..3ccb06dd4f659 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/provision.py @@ -0,0 +1,409 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``provision`` - bring an unconfigured VM up to the state the Docker image guarantees. + +``modules/ducktests/tests/docker/Dockerfile`` is the source of truth for what that state +is; the package list lives in :data:`ducktests_remote.config.DOCKERFILE_PACKAGES` next to +a note saying so. This is not configuration management and must not grow into it: it +covers the specific package / directory / ssh-environment set the image installs, each +step idempotent and independently selectable. +""" + +import json +import posixpath +import shlex + +from ducktests_remote.cli import EXIT_OK, EXIT_PREFLIGHT, EXIT_TRANSPORT +from ducktests_remote.commands import doctor +from ducktests_remote.config import ConfigError +from ducktests_remote.fanout import (CHANGED, FAILED, HostResult, OK, SKIPPED, any_failed, + fanout, render_table, summarise) + +STEPS = ("packages", "jdk", "python", "user", "ssh-env", "dirs", "hosts") + +HOSTS_BEGIN = "# BEGIN ducktests-remote" +HOSTS_END = "# END ducktests-remote" + + +def register(subparsers, common): + """Wire up the ``provision`` subcommand.""" + parser = subparsers.add_parser( + "provision", parents=[common], + help="bring the workers up to the state the Docker image guarantees", + description="Idempotent, per-step preparation of the worker hosts. Run it with " + "--dry-run first: it prints the exact commands per host and changes " + "nothing. Steps: " + ", ".join(STEPS)) + parser.add_argument("--only", action="append", default=[], choices=STEPS, metavar="STEP", + help="run only these steps; repeatable") + parser.add_argument("--skip", action="append", default=[], choices=STEPS, metavar="STEP", + help="skip these steps; repeatable") + parser.add_argument("--sudo", action="store_true", + help="allow steps that need root to use `sudo -n`") + parser.add_argument("--install-jdk", action="store_true", + help="let the jdk step install a JDK instead of only verifying one") + parser.add_argument("--create-user", metavar="NAME", + help="create this account (step `user`); needs --sudo") + parser.add_argument("--authorize-key", metavar="PATH", + help="public key appended to the created account's authorized_keys") + parser.add_argument("--write-hosts", action="store_true", + help="write the inventory into /etc/hosts (step `hosts`); needs --sudo") + parser.add_argument("-n", "--num-nodes", type=int, default=None, + help="only provision the first N inventory hosts") + parser.add_argument("--json", action="store_true", help="machine-readable output") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Provision the workers. :return: the process exit code.""" + nodes = ctx.all_nodes + if not nodes: + raise ConfigError("cluster.nodes is empty; there is nothing to provision") + + selected = _selected_steps(ctx.args) + ctx.console.info("steps: %s" % ", ".join(selected)) + if ctx.dry_run: + ctx.console.info("--dry-run: printing commands only") + + all_results = {} + skipped_for_sudo = [] + for step in selected: + ctx.console.heading("STEP %s" % step) + script, needs_sudo = _step_script(ctx, step, nodes) + if script is None: + ctx.console.info("nothing to do for this step") + continue + if needs_sudo and not ctx.args.sudo: + ctx.console.warn("step %r needs root; rerun with --sudo. Skipping it and " + "continuing with the rest." % step) + skipped_for_sudo.append(step) + continue + results = _run_step(ctx, nodes, step, script) + all_results[step] = results + ctx.console.out(render_table(results, verbose=ctx.console.verbose)) + ctx.console.out(summarise(results)) + + if ctx.args.json: + ctx.console.out(json.dumps( + {step: [{"host": r.host, "status": r.status, "message": r.message} for r in res] + for step, res in all_results.items()}, indent=2)) + + if skipped_for_sudo: + ctx.console.warn("skipped for lack of --sudo: %s" % ", ".join(skipped_for_sudo)) + + failed = any(any_failed(res) for res in all_results.values()) + + if not ctx.dry_run: + # Always finish with evidence rather than an assumption. + ctx.console.heading("VERIFYING") + checks, diagnoses = doctor.run_checks(ctx) + doctor.print_report(ctx, checks, diagnoses) + if doctor.has_failures(checks): + return EXIT_PREFLIGHT + + return EXIT_TRANSPORT if failed else EXIT_OK + + +def _selected_steps(args): + steps = list(args.only) if args.only else list(STEPS) + steps = [s for s in steps if s not in (args.skip or [])] + if "user" in steps and not args.create_user and not args.only: + # Most operators use their own existing account; creating one is an escape hatch. + steps.remove("user") + if "hosts" in steps and not args.write_hosts and not args.only: + steps.remove("hosts") + return steps + + +def _run_step(ctx, nodes, step, script): + def operation(node): + transport = ctx.worker(node) + if ctx.dry_run: + ctx.console.out("[dry-run] %s: step %s" % (node.host, step)) + ctx.console.detail(script) + return HostResult(node.host, SKIPPED, "dry-run") + result = transport.run_script(script, check=False) + text = result.stdout.strip() + if not result.ok: + return HostResult(node.host, FAILED, _summary_line(result), detail=text or + result.stderr.strip()) + status = CHANGED if "CHANGED" in text else OK + return HostResult(node.host, status, _summary_line(result), detail=text) + + return fanout(nodes, operation, jobs=ctx.jobs, + fail_fast=getattr(ctx.args, "fail_fast", False)) + + +def _summary_line(result): + lines = [ln for ln in (result.stdout or "").splitlines() if ln.strip()] + if lines: + return lines[-1][:100] + return (result.stderr or "").strip().splitlines()[-1][:100] if result.stderr else "" + + +def _step_script(ctx, step, nodes): + """:return: ``(script, needs_sudo)`` for one step, or ``(None, False)`` when inert.""" + if step == "packages": + return _packages_script(ctx), True + if step == "jdk": + return _jdk_script(ctx), bool(ctx.args.install_jdk) + if step == "python": + return _python_script(ctx), False + if step == "user": + if not ctx.args.create_user: + return None, False + return _user_script(ctx), True + if step == "ssh-env": + return _ssh_env_script(ctx), False + if step == "dirs": + return _dirs_script(ctx), True + if step == "hosts": + if not ctx.args.write_hosts: + return None, False + return _hosts_script(ctx, nodes), True + raise ConfigError("unknown provision step %r" % step) + + +def _packages_script(ctx): + packages = ctx.config["provision"]["packages"] + return """set -u +missing="" +for p in %(pkgs)s; do + if command -v dpkg-query >/dev/null 2>&1; then + dpkg-query -W -f='${Status}' "$p" 2>/dev/null | grep -q "install ok installed" \\ + || missing="$missing $p" + elif command -v rpm >/dev/null 2>&1; then + rpm -q "$p" >/dev/null 2>&1 || missing="$missing $p" + else + echo "unsupported package manager: neither dpkg nor rpm found" >&2 + exit 2 + fi +done +if [ -z "$missing" ]; then echo "all %(count)d packages present"; exit 0; fi +echo "CHANGED installing:$missing" +if command -v apt-get >/dev/null 2>&1; then + sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq + sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $missing +elif command -v dnf >/dev/null 2>&1; then + sudo -n dnf install -y -q $missing +elif command -v yum >/dev/null 2>&1; then + sudo -n yum install -y -q $missing +else + echo "unsupported package manager: no apt-get, dnf or yum" >&2 + exit 2 +fi +echo "CHANGED installed:$missing" +""" % {"pkgs": " ".join(shlex.quote(p) for p in packages), "count": len(packages)} + + +def _jdk_script(ctx): + """ + Verify the JDK, and only install one when explicitly asked. + + Where a JDK comes from is site specific - a distro package, a Temurin tarball, an + internal mirror - so guessing would be worse than reporting. + """ + major = int(ctx.config["provision"]["jdk_major"]) + install = """ +if command -v apt-get >/dev/null 2>&1; then + echo "CHANGED installing openjdk-%(major)d-jdk" + sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq + sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq openjdk-%(major)d-jdk +elif command -v dnf >/dev/null 2>&1; then + echo "CHANGED installing java-%(major)d-openjdk-devel" + sudo -n dnf install -y -q java-%(major)d-openjdk-devel +else + echo "no supported package manager for an automatic JDK install" >&2 + exit 2 +fi +""" % {"major": major} if ctx.args.install_jdk else """ +echo "java %(major)d not found; install it yourself or rerun with --only jdk --install-jdk" +exit 1 +""" % {"major": major} + + return """set -u +if command -v java >/dev/null 2>&1; then + v=$(java -version 2>&1 | head -n1) + case "$v" in + *\\"%(major)d*|*\\"1.%(major)d*) echo "ok: $v"; exit 0;; + *) echo "WARN unexpected JDK: $v (expected major %(major)d)"; exit 0;; + esac +fi +%(install)s +java -version 2>&1 | head -n1 +""" % {"major": major, "install": install} + + +def _python_script(ctx): + """ + Ensure a usable Python on the host. + + The workers do not need Python: ducktape drives them over plain SSH and runs no + Python there. Only the runner needs the venv, and ``run`` creates it on demand. + This step therefore verifies rather than installs, and says so. + """ + return """set -u +if command -v python3 >/dev/null 2>&1; then + echo "ok: $(python3 --version 2>&1)" +else + echo "python3 missing. Workers do not need it - ducktape drives them over plain ssh -" + echo "but nothing here will install it either." +fi +exit 0 +""" + + +def _user_script(ctx): + name = ctx.args.create_user + key = "" + if ctx.args.authorize_key: + with open(ctx.args.authorize_key, "r", encoding="utf-8") as handle: + key = handle.read().strip() + return """set -eu +user=%(user)s +key=%(key)s +if id -u "$user" >/dev/null 2>&1; then + echo "account $user already exists" +else + echo "CHANGED creating $user" + sudo -n useradd -m -s /bin/bash "$user" +fi +if [ -n "$key" ]; then + home=$(getent passwd "$user" | cut -d: -f6) + sudo -n mkdir -p "$home/.ssh" + if sudo -n grep -qxF "$key" "$home/.ssh/authorized_keys" 2>/dev/null; then + echo "key already authorised" + else + echo "CHANGED authorising key" + printf '%%s\\n' "$key" | sudo -n tee -a "$home/.ssh/authorized_keys" >/dev/null + fi + sudo -n chown -R "$user" "$home/.ssh" + sudo -n chmod 700 "$home/.ssh" + sudo -n chmod 600 "$home/.ssh/authorized_keys" +fi +echo done +""" % {"user": shlex.quote(name), "key": shlex.quote(key)} + + +def _ssh_env_script(ctx): + """ + Put JAVA_HOME and PATH into ``~/.ssh/environment``. + + This is the step that is easiest to forget and hardest to diagnose. ducktape runs + every command over *non-interactive* ssh, where ``~/.profile`` is not sourced, so a + ``java`` that works when you log in by hand is simply absent during a test run. The + Dockerfile solves it with ``PermitUserEnvironment yes`` plus ``~/.ssh/environment``; + the same fix applies here, and the step ends by proving it non-interactively rather + than trusting the edit. + """ + extra = ctx.config["provision"].get("ssh_env_path_extra") or [] + extra_path = "".join(":%s" % p for p in extra) + return """set -u +mkdir -p ~/.ssh +chmod 700 ~/.ssh +jh="${JAVA_HOME:-}" +if [ -z "$jh" ] && command -v java >/dev/null 2>&1; then + jh=$(dirname "$(dirname "$(readlink -f "$(command -v java)")")") +fi +if [ -z "$jh" ]; then echo "cannot determine JAVA_HOME; run the jdk step first" >&2; exit 1; fi +want_path="PATH=$PATH:$jh/bin%(extra)s" +want_home="JAVA_HOME=$jh" +changed=0 +touch ~/.ssh/environment +chmod 600 ~/.ssh/environment +for line in "$want_path" "$want_home" "LANG=C.UTF-8"; do + key=${line%%%%=*} + if grep -q "^$key=" ~/.ssh/environment 2>/dev/null; then + current=$(grep "^$key=" ~/.ssh/environment | head -n1) + [ "$current" = "$line" ] && continue + grep -v "^$key=" ~/.ssh/environment > ~/.ssh/environment.tmp || true + mv ~/.ssh/environment.tmp ~/.ssh/environment + fi + printf '%%s\\n' "$line" >> ~/.ssh/environment + changed=1 +done +chmod 600 ~/.ssh/environment +if [ "$changed" -eq 1 ]; then echo "CHANGED wrote ~/.ssh/environment"; else echo "up to date"; fi +if ! grep -qi '^ *PermitUserEnvironment *yes' /etc/ssh/sshd_config 2>/dev/null; then + echo "NOTE sshd has no 'PermitUserEnvironment yes'; ~/.ssh/environment will be ignored." + echo "NOTE ask your administrator for it, or make sure java is on the default PATH." +fi +echo "verify: $(command -v java || echo 'java not on this shell PATH')" +""" % {"extra": extra_path} + + +def _dirs_script(ctx): + """ + Create the directories the services write into. + + ``ignitetest``'s ``PathAware`` builds every path under ``persistent_root``, default + ``/mnt/service`` (services/utils/path.py), and the Dockerfile chowns ``/mnt``, + ``/var/log`` and ``/opt`` to the test account. + """ + dirs = list(ctx.config["provision"]["dirs"]) + [ctx.cluster_cfg.get("install_root", "/opt")] + owner = ctx.cluster_cfg.get("user") + return """set -u +changed=0 +for d in %(dirs)s; do + if [ -d "$d" ]; then + [ -w "$d" ] || { sudo -n chown -R %(owner)s "$d" && changed=1; } + else + sudo -n mkdir -p "$d" && sudo -n chown -R %(owner)s "$d" && changed=1 + fi +done +if [ "$changed" -eq 1 ]; then echo "CHANGED prepared %(count)d directories" +else echo "%(count)d directories already usable"; fi +""" % {"dirs": " ".join(shlex.quote(d) for d in dirs), + "owner": shlex.quote(str(owner)), "count": len(dirs)} + + +def _hosts_script(ctx, nodes): + """ + Write the inventory into ``/etc/hosts`` between explicit markers. + + Only the block between the markers is ever rewritten. This is the escape hatch for + clusters whose DNS does not resolve node-to-node, mirroring what ``ducker_up`` does + for the Docker network; it is not a substitute for working DNS. + """ + entries = [] + for node in nodes: + entries.append("%s %s" % (node.ip or node.host, node.host)) + block = "\n".join(entries) + return """set -eu +block=%(block)s +tmp=$(mktemp) +awk 'BEGIN{skip=0} + /^# BEGIN ducktests-remote$/{skip=1; next} + /^# END ducktests-remote$/{skip=0; next} + skip==0{print}' /etc/hosts > "$tmp" +{ + printf '%%s\\n' "%(begin)s" + printf '%%s\\n' "$block" + printf '%%s\\n' "%(end)s" +} >> "$tmp" +if cmp -s "$tmp" /etc/hosts; then + rm -f "$tmp"; echo "/etc/hosts already up to date" +else + sudo -n cp "$tmp" /etc/hosts + rm -f "$tmp" + echo "CHANGED rewrote the ducktests-remote block in /etc/hosts" +fi +""" % {"block": shlex.quote(block), "begin": HOSTS_BEGIN, "end": HOSTS_END} + + +def venv_bin(venv, name): + """:return: ``/bin/``.""" + return posixpath.join(venv, "bin", name) diff --git a/modules/ducktests/tests/ducktests_remote/commands/run.py b/modules/ducktests/tests/ducktests_remote/commands/run.py new file mode 100644 index 0000000000000..f7672112c7dc4 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/run.py @@ -0,0 +1,503 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``run`` - compose the artifacts, launch ducktape detached, and follow its log.""" + +import json +import os +import platform +import posixpath +import shlex +import socket +import sys +import time +from pathlib import Path + +from ducktests_remote import __version__, cluster as cluster_mod, globals_builder, runs +from ducktests_remote.cli import (EXIT_OK, EXIT_PREFLIGHT, EXIT_TESTS_FAILED, Console) +from ducktests_remote.commands import doctor +from ducktests_remote.config import ConfigError, expand_path + +DEFAULT_EXCLUDES = [".git", "target", "results", "__pycache__", "*.pyc", ".idea", + "venv", ".venv", "*.egg-info", ".tox", ".pytest_cache"] + +IGNITE_IGNORE_FILE = ".ducktestsignore" + +FOLLOW_POLL_SEC = 1.5 +DOUBLE_INTERRUPT_SEC = 3.0 + + +def register(subparsers, common): + """Wire up the ``run`` subcommand.""" + parser = subparsers.add_parser( + "run", parents=[common], help="launch a ducktape run on the cluster", + description="Compose the configuration, generate cluster.json / globals.json / " + "run.sh on the runner, and launch ducktape detached.") + parser.add_argument("test_paths", nargs="*", metavar="TEST_PATH", + help="test paths, as ducktape understands them") + parser.add_argument("-t", "--tc-path", action="append", default=[], metavar="PATH", + help="test path; repeatable, equivalent to a positional argument") + parser.add_argument("-g", "--global", action="append", default=[], dest="globals_kv", + metavar="KEY=VALUE", + help="dotted globals override, e.g. -g ssl.enabled=true") + parser.add_argument("-p", "--param", action="append", default=[], dest="params_kv", + metavar="KEY=VALUE", help="dotted --parameters override") + parser.add_argument("--globals-json", metavar="JSON", + help="raw globals base layer; paste an existing Jenkins blob here") + parser.add_argument("--globals-file", metavar="PATH", help="raw globals base layer from file") + parser.add_argument("--params-json", metavar="JSON", help="raw --parameters base layer") + parser.add_argument("--cluster-file", metavar="PATH", + help="use this coordinator-side cluster file verbatim instead of " + "generating one from the inventory") + parser.add_argument("-n", "--num-nodes", type=int, default=None, metavar="N", + help="use the first N inventory hosts (default: all)") + parser.add_argument("--source-root", metavar="PATH", + help="directory synced to the runner (default: the current directory)") + parser.add_argument("--no-sync", action="store_true", + help="assume the sources are already on the runner") + parser.add_argument("--exclude", action="append", default=[], metavar="PATTERN", + help="extra sync exclusion; repeatable") + parser.add_argument("--work-dir", metavar="PATH", + help="runner-side working directory for ducktape " + "(default: the synced source root)") + parser.add_argument("--install-sources", action="store_true", + help="pip install the synced sources into the runner venv. Not needed " + "for test discovery; ducktape puts the sources on sys.path itself") + parser.add_argument("--repeat", type=int, default=None, metavar="N") + parser.add_argument("--max-parallel", type=int, default=None, metavar="N") + parser.add_argument("--test-runner-timeout", type=int, default=None, metavar="MS") + parser.add_argument("--results-root", metavar="PATH", + help="runner-side results root (default: /results)") + parser.add_argument("--skip-preflight", action="store_true", help="skip the doctor checks") + group = parser.add_mutually_exclusive_group() + group.add_argument("--follow", dest="follow", action="store_true", default=True, + help="stream the log until the run ends (default)") + group.add_argument("--detach", dest="follow", action="store_false", + help="print the run id and exit immediately") + parser.set_defaults(handler=execute) + + +def execute(ctx): # pylint: disable=too-many-return-statements + """Launch a run. :return: the process exit code.""" + args = ctx.args + console = ctx.console + + test_paths = list(args.test_paths) + list(args.tc_path) + if not test_paths: + raise ConfigError("no tests given; pass a path positionally or with -t/--tc-path") + + composed, params = _compose_payloads(ctx) + + nodes = ctx.nodes + cluster_payload, cluster_source, cluster_text = _cluster_payload(ctx, nodes) + _warn_about_topology(ctx, nodes, test_paths) + + if not args.skip_preflight and not ctx.dry_run: + console.heading("PREFLIGHT") + checks, diagnoses = doctor.run_checks(ctx, _versions(composed)) + doctor.print_report(ctx, checks, diagnoses) + if doctor.has_failures(checks): + console.error("preflight failed; fix the FAIL rows above or pass --skip-preflight") + return EXIT_PREFLIGHT + + run_id = runs.new_run_id(_coordinator_user()) + state_root = ctx.runner.expand(ctx.state_root) + paths = runs.RunPaths(state_root, run_id) + + source_root = _source_root(ctx) + work_dir = _work_dir(ctx, paths, source_root) + results_root = args.results_root or paths.results_dir + + run_sh = runs.render_run_script( + version=__version__, timestamp=runs.utc_now_iso(), + author="%s@%s" % (_coordinator_user(), socket.gethostname()), + work_dir=work_dir, results_root=results_root, + cluster_file=paths.cluster_file, globals_file=paths.globals_file, + test_paths=test_paths, venv=_venv_path(ctx), + parameters_file=paths.parameters_file if params else None, + repeat=args.repeat, max_parallel=args.max_parallel, + test_runner_timeout=args.test_runner_timeout, + extra_args=getattr(args, "passthrough", [])) + + _print_artifacts(ctx, paths, cluster_payload, composed, params, run_sh, cluster_source) + + if ctx.dry_run: + console.out("") + console.info("--dry-run: nothing was created, uploaded or started.") + return EXIT_OK + + console.heading("PREPARING %s" % run_id) + ctx.runner.mkdirs(paths.run_dir) + ctx.runner.mkdirs(paths.results_dir) + + if not args.no_sync: + _sync_sources(ctx, source_root, paths) + _ensure_venv(ctx, work_dir) + if args.install_sources: + _install_sources(ctx, work_dir) + + ctx.runner.write_file(cluster_text or cluster_mod.dumps(cluster_payload), + paths.cluster_file) + ctx.runner.write_file(globals_builder.dumps(composed), paths.globals_file, mode=0o600) + if params: + ctx.runner.write_file(globals_builder.dumps(params), paths.parameters_file, mode=0o600) + ctx.runner.write_file(run_sh, paths.run_script, mode=0o755) + ctx.runner.write_file(runs.render_launch_script(paths), paths.launch_script, mode=0o755) + + runs.write_meta(ctx.runner, paths, + _meta(ctx, run_id, test_paths, nodes, work_dir, results_root)) + _update_latest_link(ctx, paths) + + pid = ctx.runner.run_script(runs.render_detach_script(paths)).check().out + console.out("") + console.out("run id : %s" % run_id) + console.out("runner : %s (pid %s)" % (ctx.runner_host, pid)) + console.out("run dir: %s" % paths.run_dir) + console.out("log : %s" % paths.log_file) + + if not args.follow: + _print_reattach(console, run_id) + return EXIT_OK + + return _follow(ctx, paths, run_id) + + +# -- composition ------------------------------------------------------------------- + + +def _compose_payloads(ctx): + args = ctx.args + redactor = ctx.console.redactor + + layers = [] + raw = globals_builder.load_raw_layer(args.globals_json, args.globals_file) + if raw is not None: + layers.append(("--globals-json/--globals-file", raw)) + layers.append(("config globals", ctx.config.get("globals") or {})) + composed, _ = globals_builder.build(layers, args.globals_kv, redactor=redactor) + + param_layers = [] + raw_params = globals_builder.load_raw_layer(args.params_json, None) + if raw_params is not None: + param_layers.append(("--params-json", raw_params)) + param_layers.append(("config parameters", ctx.config.get("parameters") or {})) + params, _ = globals_builder.build(param_layers, args.params_kv, redactor=redactor) + + return composed, params + + +def _cluster_payload(ctx, nodes): + """ + :return: ``(parsed, source, verbatim_text)``. + + ``--cluster-file`` is uploaded byte for byte - it is the migration path from a + hand-written file such as the current ``49_cluster.json``, and rewriting it would + silently drop anything the CLI does not model. It is still parsed here so that a + broken file fails now rather than inside ducktape. + """ + if ctx.args.cluster_file: + path = Path(expand_path(ctx.args.cluster_file)) + if not path.is_file(): + raise ConfigError("cluster file not found: %s" % path) + text = path.read_text(encoding="utf-8") + try: + parsed = json.loads(text) + except ValueError as ex: + raise ConfigError("%s is not valid JSON: %s" % (path, ex)) from ex + if not parsed.get("nodes"): + raise ConfigError("%s has no 'nodes' entry" % path) + return parsed, str(path), text + identity = ctx.runner.expand(ctx.identity_file) if ctx.identity_file else None + return cluster_mod.cluster_json(nodes, identity_file=identity), "inventory", None + + +def _warn_about_topology(ctx, nodes, test_paths): + console = ctx.console + if ctx.runner_host not in ("local", None): + if any(node.host == ctx.runner_host for node in nodes): + console.warn("the runner %s is also listed as a worker. The Docker flow reserves " + "ducker01 for ducktape; on a real cluster that is your call, but " + "ducktape will schedule Ignite nodes onto the machine it runs on." + % ctx.runner_host) + if len(nodes) < 3 and len(test_paths) > 0: + console.warn("only %d worker%s in the inventory. Most ignitetest suites declare " + "@cluster(num_nodes=...) well above that and will be skipped as " + "un-runnable." % (len(nodes), "" if len(nodes) == 1 else "s")) + + +def _versions(composed): + versions = composed.get("ignite_versions") + if isinstance(versions, str): + return [versions] + return list(versions or []) + + +# -- runner-side preparation -------------------------------------------------------- + + +def _source_root(ctx): + configured = ctx.args.source_root or ctx.config["run"].get("source_root") + return Path(expand_path(configured) or os.getcwd()).resolve() + + +def _work_dir(ctx, paths, source_root): + configured = ctx.args.work_dir or ctx.config["run"].get("work_dir") + if configured: + return ctx.runner.expand(configured) + if ctx.args.no_sync: + # Nothing was uploaded, so the sources must already be where the operator says. + return ctx.runner.expand(str(source_root).replace(os.sep, "/")) + return paths.src_dir + + +def _excludes(ctx, source_root): + if ctx.args.exclude: + return list(ctx.args.exclude) + ignore_file = Path(source_root) / IGNITE_IGNORE_FILE + if ignore_file.is_file(): + return [line.strip() for line in ignore_file.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.strip().startswith("#")] + return list(DEFAULT_EXCLUDES) + list(ctx.config["run"].get("exclude") or []) + + +def _sync_sources(ctx, source_root, paths): + excludes = _excludes(ctx, source_root) + size_mb = _payload_size_mb(source_root, excludes) + limit = int(ctx.config["run"]["max_payload_mb"]) + ctx.console.info("syncing %s (%.1f MB) -> %s:%s" + % (source_root, size_mb, ctx.runner_host, paths.src_dir)) + if size_mb > limit: + raise ConfigError( + "source payload is %.1f MB, above the %d MB limit. That almost always means a " + "build directory leaked into the payload - check --exclude / %s. Distributions " + "belong in `ducktests-remote deploy`, never in the source sync." + % (size_mb, limit, IGNITE_IGNORE_FILE)) + ctx.runner.upload_dir(source_root, paths.src_dir, excludes=excludes) + + +def _payload_size_mb(source_root, excludes): + from ducktests_remote.transport import is_excluded # pylint: disable=import-outside-toplevel + total = 0 + root = Path(source_root) + for entry in root.rglob("*"): + if entry.is_dir() or entry.is_symlink(): + continue + if is_excluded(entry.relative_to(root), excludes): + continue + try: + total += entry.stat().st_size + except OSError: + continue + return total / (1024.0 * 1024.0) + + +def _venv_path(ctx): + """ + :return: the runner-side venv path, or None when ducktape is expected on PATH. + + The environment is controlled rather than assumed: an explicitly configured path is + used as given, otherwise the default under the state root is used and created when + it does not exist yet. + """ + configured = ctx.config["runner"].get("venv") + if configured: + return ctx.runner.expand(configured) + if not ctx.config["runner"].get("create_venv", True): + return None + return posixpath.join(ctx.runner.expand(ctx.state_root), "venv") + + +def _ensure_venv(ctx, work_dir): + """Create the runner venv and install the pinned requirements when it is missing.""" + venv = _venv_path(ctx) + if not venv: + return + python = ctx.config["runner"].get("python", "python3") + requirements = ctx.config["runner"].get("requirements") or posixpath.join( + work_dir, "modules", "ducktests", "tests", "docker", "requirements.txt") + index = ctx.config["provision"].get("pip_index_url") + index_arg = "--index-url %s" % shlex.quote(index) if index else "" + + script = """set -eu +venv=%(venv)s +req=%(req)s +if [ ! -x "$venv/bin/python3" ]; then + echo "creating venv at $venv" + %(python)s -m venv "$venv" + created=1 +fi +if ! "$venv/bin/python3" -c 'import ducktape' 2>/dev/null; then + if [ -f "$req" ]; then + echo "installing $req into $venv" + "$venv/bin/pip3" install --disable-pip-version-check %(index)s -r "$req" + else + echo "MISSING_REQUIREMENTS $req" >&2 + exit 3 + fi +fi +"$venv/bin/python3" -c 'import ducktape;print("ducktape " + ducktape.__version__)' +""" % {"venv": shlex.quote(venv), "req": shlex.quote(requirements), + "python": shlex.quote(python), "index": index_arg} + + result = ctx.runner.run_script(script, check=False) + if not result.ok: + raise ConfigError( + "could not prepare the runner venv at %s:\n%s\nEither point runner.venv at an " + "existing environment or make %s reachable on the runner." + % (venv, (result.stderr or result.stdout).strip(), requirements)) + ctx.console.info(result.out.splitlines()[-1] if result.out else "venv ready") + + +def _install_sources(ctx, work_dir): + venv = _venv_path(ctx) + pip = posixpath.join(venv, "bin", "pip3") if venv else "pip3" + tests_dir = posixpath.join(work_dir, "modules", "ducktests", "tests") + ctx.console.info("installing sources from %s" % tests_dir) + ctx.runner.run([pip, "install", "--disable-pip-version-check", "-e", tests_dir]).check() + + +def _update_latest_link(ctx, paths): + ctx.runner.run_script("set -eu\nln -sfn %s %s\n" + % (shlex.quote(paths.run_dir), shlex.quote(paths.latest_link)), + check=False) + + +def _meta(ctx, run_id, test_paths, nodes, work_dir, results_root): + from ducktests_remote.config import redacted_summary # pylint: disable=import-outside-toplevel + return { + "run_id": run_id, + "cli_version": __version__, + "coordinator": {"user": _coordinator_user(), "host": socket.gethostname(), + "platform": platform.platform()}, + "runner": ctx.runner_host, + "cluster": ctx.cluster_cfg.get("name"), + "nodes": [n.host for n in nodes], + "test_paths": list(test_paths), + "work_dir": work_dir, + "results_root": results_root, + "started_at": runs.utc_now_iso(), + "started_epoch": time.time(), + "config": ctx.console.redactor.redact_structure(redacted_summary(ctx.config)), + } + + +# -- output ------------------------------------------------------------------------ + + +def _print_artifacts(ctx, paths, cluster_payload, composed, params, run_sh, cluster_source): + console = ctx.console + if not (ctx.dry_run or console.verbose): + return + console.heading("cluster.json (from %s) -> %s" % (cluster_source, paths.cluster_file)) + console.out(cluster_mod.dumps(cluster_payload).rstrip()) + console.heading("globals.json (redacted) -> %s [mode 0600]" % paths.globals_file) + console.out(globals_builder.dumps( + console.redactor.redact_structure(composed)).rstrip()) + if params: + console.heading("parameters.json -> %s" % paths.parameters_file) + console.out(globals_builder.dumps( + console.redactor.redact_structure(params)).rstrip()) + console.heading("run.sh -> %s" % paths.run_script) + console.out(run_sh.rstrip()) + + +def _print_reattach(console: Console, run_id): + console.out("") + console.out("The run continues on the runner. To come back to it:") + console.out(" ducktests-remote logs %s -f" % run_id) + console.out(" ducktests-remote status %s" % run_id) + console.out("To stop it and clean the workers:") + console.out(" ducktests-remote stop %s" % run_id) + + +# -- follow ------------------------------------------------------------------------ + + +def _follow(ctx, paths, run_id): + """ + Stream the runner's log until the process ends. + + Ctrl-C detaches; it never kills the run. Losing a three hour run to a reflex is not + a recoverable mistake, so stopping takes a deliberate second interrupt. + """ + console = ctx.console + console.heading("FOLLOWING %s (Ctrl-C detaches, it does not stop the run)" % run_id) + offset = 1 + last_interrupt = 0.0 + + while True: + try: + offset = _drain(ctx, paths, offset, console) + + state = runs.read_state(ctx.runner, paths) + if state.exit_code is not None: + # The process wrote its exit code between the two reads above, so drain + # once more or the last lines of the session report are lost. + _drain(ctx, paths, offset, console) + return _finish(ctx, paths, run_id, state) + time.sleep(FOLLOW_POLL_SEC) + except KeyboardInterrupt: + now = time.monotonic() + if now - last_interrupt < DOUBLE_INTERRUPT_SEC: + return _offer_stop(ctx, paths, run_id) + last_interrupt = now + console.out("") + console.out("Detaching. The run keeps going on %s." % ctx.runner_host) + _print_reattach(console, run_id) + console.out("") + console.out("Press Ctrl-C again within %ds to stop the run instead." + % int(DOUBLE_INTERRUPT_SEC)) + + +def _drain(ctx, paths, offset, console): + """Print whatever the log has gained since ``offset``. :return: the new offset.""" + chunk = ctx.runner.run(["tail", "-c", "+%d" % offset, paths.log_file], check=False) + if chunk.stdout: + offset += len(chunk.stdout.encode("utf-8")) + sys.stdout.write(console.redactor.redact(chunk.stdout)) + sys.stdout.flush() + return offset + + +def _offer_stop(ctx, paths, run_id): + console = ctx.console + console.out("") + if not sys.stdin.isatty(): + console.out("Detached (no terminal to confirm on). Run `ducktests-remote stop %s` " + "to stop it." % run_id) + return EXIT_OK + try: + answer = input("Stop run %s and clean the workers? [y/N] " % run_id) + except (EOFError, KeyboardInterrupt): + answer = "" + if answer.strip().lower() in ("y", "yes"): + from ducktests_remote.commands import stop # pylint: disable=import-outside-toplevel + return stop.stop_run(ctx, paths, timeout=60, kill=False, clean=True) + console.out("Left running. `ducktests-remote logs %s -f` to reattach." % run_id) + return EXIT_OK + + +def _finish(ctx, paths, run_id, state): + console = ctx.console + console.out("") + console.out("run %s %s after %s (ducktape exit %s)" + % (run_id, state.state, runs.format_duration(state.elapsed), state.exit_code)) + console.out("results: %s" % posixpath.join(paths.results_dir, "latest")) + console.out("fetch : ducktests-remote fetch %s" % run_id) + return EXIT_OK if state.exit_code == 0 else EXIT_TESTS_FAILED + + +def _coordinator_user(): + return os.environ.get("USER") or os.environ.get("USERNAME") or "unknown" diff --git a/modules/ducktests/tests/ducktests_remote/commands/status.py b/modules/ducktests/tests/ducktests_remote/commands/status.py new file mode 100644 index 0000000000000..453e085a35d3b --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/status.py @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``status`` - what a run is doing, or a table of every run on the runner.""" + +import json + +from ducktests_remote import runs +from ducktests_remote.cli import EXIT_OK, EXIT_USAGE + + +def register(subparsers, common): + """Wire up the ``status`` subcommand.""" + parser = subparsers.add_parser( + "status", parents=[common], help="show the state of a run", + description="With no run id, report the most recent run on the runner. Run state " + "lives on the runner, so this works from any coordinator.") + parser.add_argument("run_id", nargs="?", help="run id (default: the most recent run)") + parser.add_argument("--all", action="store_true", help="table of every run, newest first") + parser.add_argument("--json", action="store_true", help="machine-readable output") + parser.add_argument("-n", "--lines", type=int, default=15, + help="lines of log tail to show (default 15)") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Print run status. :return: the process exit code.""" + state_root = ctx.state_root_resolved() + + if ctx.args.all: + return _print_all(ctx, state_root) + + run_id = runs.resolve_run_id(ctx.runner, state_root, ctx.args.run_id) + if not run_id: + ctx.console.error("no runs found under %s on %s" % (state_root, ctx.runner_host)) + return EXIT_USAGE + + paths = runs.RunPaths(state_root, run_id) + state = runs.read_state(ctx.runner, paths) + + if ctx.args.json: + ctx.console.out(json.dumps(_as_dict(paths, state), indent=2, sort_keys=True)) + return EXIT_OK + + console = ctx.console + meta = state.meta + console.out("run id : %s" % state.run_id) + console.out("state : %s" % console.paint(state.state, _style(state.state))) + console.out("runner : %s" % ctx.runner_host) + console.out("pid : %s" % (state.pid if state.pid is not None else "-")) + console.out("started : %s" % (state.started_at or "-")) + console.out("elapsed : %s" % runs.format_duration(state.elapsed)) + console.out("exit code : %s" % (state.exit_code if state.exit_code is not None else "-")) + console.out("tests : %s" % ", ".join(meta.get("test_paths") or []) or "-") + console.out("cluster : %s (%d node(s))" % (meta.get("cluster") or "-", + len(meta.get("nodes") or []))) + console.out("run dir : %s" % paths.run_dir) + console.out("results : %s" % (meta.get("results_root") or paths.results_dir)) + + tail = ctx.runner.run(["tail", "-n", str(ctx.args.lines), paths.log_file], check=False) + if tail.stdout.strip(): + console.heading("last %d log lines" % ctx.args.lines) + console.out(tail.stdout.rstrip()) + return EXIT_OK + + +def _print_all(ctx, state_root): + run_ids = runs.list_run_ids(ctx.runner, state_root) + if not run_ids: + ctx.console.out("no runs under %s on %s" % (state_root, ctx.runner_host)) + return EXIT_OK + + rows = [] + for run_id in run_ids: + paths = runs.RunPaths(state_root, run_id) + state = runs.read_state(ctx.runner, paths) + rows.append((run_id, state.state, + runs.format_duration(state.elapsed), + "-" if state.exit_code is None else str(state.exit_code), + ", ".join(state.meta.get("test_paths") or [])[:60])) + + if ctx.args.json: + ctx.console.out(json.dumps( + [dict(zip(("run_id", "state", "elapsed", "exit_code", "tests"), row)) + for row in rows], indent=2)) + return EXIT_OK + + headers = ("RUN ID", "STATE", "ELAPSED", "EXIT", "TESTS") + widths = [max(len(str(r[i])) for r in ([headers] + rows)) for i in range(5)] + ctx.console.out(" ".join(h.ljust(widths[i]) for i, h in enumerate(headers)).rstrip()) + ctx.console.out(" ".join("-" * widths[i] for i in range(5))) + for row in rows: + ctx.console.out(" ".join(str(c).ljust(widths[i]) for i, c in enumerate(row)).rstrip()) + return EXIT_OK + + +def _as_dict(paths, state): + return {"run_id": state.run_id, "state": state.state, "pid": state.pid, + "exit_code": state.exit_code, "run_dir": paths.run_dir, + "elapsed_sec": state.elapsed, "meta": state.meta} + + +def _style(state): + return {runs.RUNNING: "blue", runs.FINISHED: "green", runs.FAILED: "red", + runs.STOPPED: "yellow"}.get(state, "dim") diff --git a/modules/ducktests/tests/ducktests_remote/commands/stop.py b/modules/ducktests/tests/ducktests_remote/commands/stop.py new file mode 100644 index 0000000000000..4cd43c2c96b3e --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/stop.py @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``stop`` - terminate a detached run and, by default, clean the workers behind it.""" + +import shlex + +from ducktests_remote import runs +from ducktests_remote.cli import EXIT_OK, EXIT_USAGE + + +def register(subparsers, common): + """Wire up the ``stop`` subcommand.""" + parser = subparsers.add_parser( + "stop", parents=[common], help="stop a run and clean up after it", + description="SIGTERM the run's process group, wait, then optionally SIGKILL. " + "Cleans the workers afterwards unless --no-clean.") + parser.add_argument("run_id", nargs="?", help="run id (default: the most recent run)") + parser.add_argument("--kill", action="store_true", + help="SIGKILL survivors after the timeout") + parser.add_argument("--timeout", type=int, default=60, + help="seconds to wait for a graceful exit (default 60)") + parser.add_argument("--no-clean", dest="clean", action="store_false", default=True, + help="leave the workers alone; the JVMs will be someone else's problem") + parser.add_argument("-n", "--num-nodes", type=int, default=None, + help="limit the follow-up clean to the first N inventory hosts") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Stop a run. :return: the process exit code.""" + state_root = ctx.state_root_resolved() + run_id = runs.resolve_run_id(ctx.runner, state_root, ctx.args.run_id) + if not run_id: + ctx.console.error("no runs found under %s on %s" % (state_root, ctx.runner_host)) + return EXIT_USAGE + paths = runs.RunPaths(state_root, run_id) + return stop_run(ctx, paths, timeout=ctx.args.timeout, kill=ctx.args.kill, + clean=ctx.args.clean) + + +def stop_run(ctx, paths, *, timeout=60, kill=False, clean=True): + """Stop the run described by ``paths``. :return: the process exit code.""" + console = ctx.console + state = runs.read_state(ctx.runner, paths) + + if state.exit_code is not None: + console.info("run %s already ended (%s, exit %s)" + % (paths.run_id, state.state, state.exit_code)) + else: + console.info("stopping %s (pid %s, pgid %s)" % (paths.run_id, state.pid, state.pgid)) + result = ctx.runner.run_script(_stop_script(paths, timeout, kill), check=False) + console.detail(result.stdout.strip()) + if not result.ok and not ctx.dry_run: + console.warn("stop script exited %d: %s" % (result.returncode, + result.stderr.strip()[:200])) + + if clean: + console.heading("CLEANING WORKERS") + from ducktests_remote.commands import clean as clean_cmd # pylint: disable=C0415 + clean_cmd.clean_hosts(ctx, ctx.all_nodes, dry_run=ctx.dry_run) + + console.out("stopped %s" % paths.run_id) + return EXIT_OK + + +def _stop_script(paths, timeout, kill): + return """set -u +rd=%(rd)s +touch "$rd/stopped" +pid=$(cat "$rd/pid" 2>/dev/null || true) +pgid=$(cat "$rd/pgid" 2>/dev/null || true) +target="" +if [ -n "$pgid" ]; then target="-$pgid"; elif [ -n "$pid" ]; then target="$pid"; fi +if [ -z "$target" ]; then echo "no pid recorded"; exit 0; fi +kill -TERM -- "$target" 2>/dev/null || true +i=0 +while [ "$i" -lt %(timeout)d ]; do + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then sleep 1; i=$((i+1)); else break; fi +done +if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + if [ %(kill)d -eq 1 ]; then + echo "still alive after %(timeout)ds, sending SIGKILL" + kill -KILL -- "$target" 2>/dev/null || true + else + echo "still alive after %(timeout)ds; rerun with --kill" + fi +else + echo "exited within %(timeout)ds" +fi +[ -f "$rd/exit_code" ] || echo 143 > "$rd/exit_code" +""" % {"rd": shlex.quote(paths.run_dir), "timeout": int(timeout), "kill": 1 if kill else 0} diff --git a/modules/ducktests/tests/ducktests_remote/config.py b/modules/ducktests/tests/ducktests_remote/config.py new file mode 100644 index 0000000000000..eeaeef09e1357 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/config.py @@ -0,0 +1,371 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Configuration discovery, layering and validation. + +Layers, later overriding earlier, dicts deep-merged and lists replaced wholesale: + +1. built-in defaults +2. ``~/.ducktests-remote/config.yaml`` +3. ``--config FILE`` (repeatable, in order) +4. ``--profile NAME`` (repeatable, in order) +5. ``DTR_*`` environment variables +6. explicit command-line flags +7. ``-g KEY=VALUE`` dotted overrides (globals only, applied by :mod:`globals_builder`) +""" + +import copy +import difflib +import getpass +import json +import os +from pathlib import Path + +import yaml + +USER_CONFIG_DIR = Path("~/.ducktests-remote").expanduser() +USER_CONFIG_FILE = USER_CONFIG_DIR / "config.yaml" +PACKAGE_EXAMPLES = Path(__file__).resolve().parent / "examples" + +# Pinned requirements the runner venv must satisfy. Read from the repo rather than +# hardcoded so the ducktape pin has exactly one source of truth. +REQUIREMENTS_RELPATH = "modules/ducktests/tests/docker/requirements.txt" + +# System utilities, derived from modules/ducktests/tests/docker/Dockerfile section 2. +# The Dockerfile is the source of truth; when the two disagree, the Dockerfile wins. +# Image-only entries (openssh-server, vim, mc, build-essential, cmake, libfuse-dev...) +# are deliberately not replicated: a real VM already has an sshd and no build toolchain +# is needed to *run* tests. +DOCKERFILE_PACKAGES = [ + "sudo", + "netcat-traditional", + "iptables", + "rsync", + "unzip", + "wget", + "curl", + "jq", + "coreutils", + "net-tools", +] + +# ignitetest shells out to `sudo iptables` only from IgniteAwareService.drop_network +# (services/utils/ignite_aware.py). Exactly two suites reach it. +SUDO_DEPENDENT_TESTS = ( + "ignitetest/tests/discovery_test.py", + "ignitetest/tests/cellular_affinity_test.py", +) + +# ignitetest services launch these main classes; see services/ignite.py, +# services/ignite_app.py, services/utils/cdc/ignite_cdc.py and .../kafka_to_ignite.py. +IGNITE_MAIN_CLASSES = ( + "org.apache.ignite.startup.cmdline.CommandLineStartup", + "org.apache.ignite.startup.cmdline.CdcCommandLineStartup", + "org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService", + "org.apache.ignite.cdc.kafka.KafkaToIgniteCommandLineStartup", +) + +DEFAULTS = { + "profiles_dir": str(USER_CONFIG_DIR / "profiles"), + "cluster": { + "name": "default", + "identity_file": "~/.ssh/id_rsa", + "user": None, # resolved to the coordinator's $USER, never to "ducker" + "port": 22, + "install_root": "/opt", + "runner": "local", + "state_root": "~/.ducktests-remote", + "nodes": [], + "extra_hosts": [], + }, + "runner": { + # Deterministic environment: use venv when given, else the default under the + # state root, creating and populating it when missing. + "venv": None, + "python": "python3", + "create_venv": True, + "requirements": None, # defaults to /modules/.../requirements.txt + }, + "run": { + "source_root": None, + "work_dir": None, + "exclude": [], + "max_payload_mb": 200, + # ducktape's loader walks up from each test file while __init__.py exists and + # appends the resulting top-level directory to sys.path + # (ducktape/tests/loader.py::_add_top_level_dirs_to_sys_path), so the synced + # sources are importable without being installed. Only the third-party + # requirements have to be in the venv. Left as an opt-in escape hatch. + "install_sources": False, + }, + "deploy": { + "dist_dir": "./dist", + "install_root": None, # defaults to cluster.install_root + "sudo": False, # /opt is writable by the ordinary user on these clusters + "owner": None, + "staging_dir": "/tmp/ducktests-remote-staging", + "checksum": False, + }, + "clean": { + # pgrep -f patterns; see IGNITE_MAIN_CLASSES above. + "process_pattern": "org.apache.ignite", + # persistent_root default from ignitetest/services/utils/path.py. + "paths": ["/mnt/service"], + "allowed_roots": ["/mnt", "/tmp", "/var/tmp"], + }, + "provision": { + "packages": list(DOCKERFILE_PACKAGES), + "jdk_major": 17, + "install_jdk": False, + "pip_index_url": None, + "ssh_env_path_extra": [], + "dirs": ["/mnt/service"], + }, + "ssh": { + "connect_timeout": 15, + }, + "jobs": 16, + "globals": {}, + "parameters": {}, +} + +_FREE_FORM_SECTIONS = ("globals", "parameters") + + +class ConfigError(Exception): + """Raised for anything the operator can fix by editing a file or a flag (exit 1).""" + + +def deep_merge(base, overlay): + """ + Merge ``overlay`` into a copy of ``base``. + + Dicts merge recursively; every other type, lists included, replaces outright. + Lists concatenating across profiles would make it impossible to *shrink* a list in a + later layer, which is exactly what an override is for. + """ + if not isinstance(base, dict) or not isinstance(overlay, dict): + return copy.deepcopy(overlay) + merged = copy.deepcopy(base) + for key, value in overlay.items(): + if key in merged and isinstance(merged[key], dict) and isinstance(value, dict): + merged[key] = deep_merge(merged[key], value) + else: + merged[key] = copy.deepcopy(value) + return merged + + +def parse_document(text, source=""): + """ + Parse YAML or JSON, choosing by content rather than by file extension. + + :param text: document body. + :param source: name used in error messages. + """ + stripped = text.lstrip() + try: + if stripped.startswith(("{", "[")): + return json.loads(text) + return yaml.safe_load(text) + except (ValueError, yaml.YAMLError) as ex: + raise ConfigError("%s: not valid YAML or JSON: %s" % (source, ex)) from ex + + +def load_document(path): + """:return: the parsed contents of ``path``.""" + path = Path(path).expanduser() + if not path.is_file(): + raise ConfigError("config file not found: %s" % path) + data = parse_document(path.read_text(encoding="utf-8"), str(path)) + if data is None: + return {} + if not isinstance(data, dict): + raise ConfigError("%s: top level must be a mapping, found %s" % (path, type(data).__name__)) + return data + + +def resolve_profile(name, profiles_dir): + """ + Locate a profile by name. + + Search order: ``/.yaml`` (then ``.yml``/``.json``), a + coordinator-relative path, and finally the profiles shipped in ``examples/``. + """ + candidates = [] + for base in (Path(profiles_dir).expanduser(), Path.cwd()): + for suffix in (".yaml", ".yml", ".json", ""): + candidates.append(base / (name + suffix)) + for suffix in (".yaml", ".yml", ".json"): + candidates.append(PACKAGE_EXAMPLES / ("profile-%s%s" % (name, suffix))) + candidates.append(PACKAGE_EXAMPLES / (name + suffix)) + for candidate in candidates: + if candidate.is_file(): + return candidate + raise ConfigError("profile %r not found; looked in %s" % ( + name, ", ".join(sorted({str(c.parent) for c in candidates})))) + + +def validate(config, reference=None, path=""): + """ + Reject unknown keys with a "did you mean" hint. + + A typo silently ignored in a config that drives a three hour run is expensive; this + turns it into an immediate, named failure. + """ + reference = DEFAULTS if reference is None else reference + for key, value in config.items(): + where = "%s.%s" % (path, key) if path else key + if key not in reference: + hint = difflib.get_close_matches(key, list(reference), n=1) + suffix = "; did you mean %r?" % hint[0] if hint else "" + raise ConfigError("unknown config key %r%s" % (where, suffix)) + if where in _FREE_FORM_SECTIONS: + continue + if isinstance(value, dict) and isinstance(reference[key], dict) and reference[key]: + validate(value, reference[key], where) + + +def env_overrides(environ=None): + """ + Build a config overlay from ``DTR_*`` environment variables. + + ``DTR_CLUSTER__RUNNER=build-vm-01`` sets ``cluster.runner``. A double underscore + separates path segments so single underscores stay usable inside key names + (``DTR_RUN__MAX_PAYLOAD_MB``). A handful of single-segment aliases cover the flags + operators reach for most. + + Any other ``DTR_*`` variable is left alone. Profiles interpolate secrets with + ``${env:...}``, and those variables are frequently named ``DTR_SOMETHING`` too; + treating every one of them as a config path would turn a password into a config + error. The ``__`` form is still validated, so a typo there is caught. + """ + environ = os.environ if environ is None else environ + aliases = {"DTR_RUNNER": "cluster.runner", "DTR_JOBS": "jobs", + "DTR_CLUSTER": "cluster.name", "DTR_STATE_ROOT": "cluster.state_root", + "DTR_INSTALL_ROOT": "cluster.install_root", "DTR_USER": "cluster.user"} + overlay = {} + for name, raw in sorted(environ.items()): + if not name.startswith("DTR_") or name in ("DTR_CONFIG", "DTR_PROFILE"): + continue + if name in aliases: + dotted = aliases[name] + elif "__" in name[len("DTR_"):]: + dotted = name[len("DTR_"):].lower().replace("__", ".") + else: + continue + set_dotted(overlay, dotted, coerce_scalar(raw)) + return overlay + + +def coerce_scalar(raw): + """:return: ``raw`` parsed as JSON when possible, otherwise the string itself.""" + try: + return json.loads(raw) + except (TypeError, ValueError): + return raw + + +def set_dotted(target, dotted, value): + """Set ``a.b.c`` inside a nested dict, creating intermediate dicts as needed.""" + parts = dotted.split(".") + node = target + for part in parts[:-1]: + node = node.setdefault(part, {}) + if not isinstance(node, dict): + raise ConfigError("cannot set %r: %r is not a mapping" % (dotted, part)) + node[parts[-1]] = value + return target + + +def get_dotted(source, dotted, default=None): + """:return: the value at ``a.b.c`` inside a nested dict, or ``default``.""" + node = source + for part in dotted.split("."): + if not isinstance(node, dict) or part not in node: + return default + node = node[part] + return node + + +def load_config(config_files=(), profiles=(), overrides=None, environ=None, + user_config=USER_CONFIG_FILE): + """ + Compose the effective configuration from every layer. + + :param config_files: ``--config`` paths, applied in order. + :param profiles: ``--profile`` names, applied in order. + :param overrides: overlay built from explicit command-line flags. + :param environ: environment mapping, defaults to ``os.environ``. + :param user_config: path to the per-user config file. + :return: the merged, validated configuration. + """ + config = copy.deepcopy(DEFAULTS) + sources = [] + + user_config = Path(user_config).expanduser() if user_config else None + if user_config and user_config.is_file(): + config = _layer(config, load_document(user_config), str(user_config)) + sources.append(str(user_config)) + + for path in config_files: + config = _layer(config, load_document(path), str(path)) + sources.append(str(path)) + + for name in profiles: + path = resolve_profile(name, config.get("profiles_dir", DEFAULTS["profiles_dir"])) + config = _layer(config, load_document(path), str(path)) + sources.append(str(path)) + + config = _layer(config, env_overrides(environ), "environment") + + if overrides: + config = _layer(config, overrides, "command line") + + config["cluster"]["user"] = config["cluster"]["user"] or _current_user() + config["_sources"] = sources + return config + + +def _layer(config, overlay, source): + if not overlay: + return config + try: + validate(overlay) + except ConfigError as ex: + raise ConfigError("%s: %s" % (source, ex)) from ex + return deep_merge(config, overlay) + + +def _current_user(): + try: + return getpass.getuser() + except (KeyError, OSError): # no passwd entry, e.g. some containers + return os.environ.get("USER") or os.environ.get("USERNAME") or "root" + + +def expand_path(path, default=None): + """:return: ``path`` with ``~`` expanded, or ``default`` when path is falsy.""" + if not path: + return default + return os.path.expanduser(str(path)) + + +def redacted_summary(config): + """:return: a config copy safe to write into ``meta.json`` (globals stripped).""" + summary = copy.deepcopy({k: v for k, v in config.items() if not k.startswith("_")}) + summary["globals"] = "" + summary["parameters"] = "" + return summary diff --git a/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml b/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml new file mode 100644 index 0000000000000..ec545b8cd4615 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Example cluster inventory. Copy to ~/.ducktests-remote/config.yaml and edit. +# Everything here is a placeholder: this file lives in a public repository, so it +# contains no real hostnames, addresses, accounts or secrets. + +cluster: + name: example + + # SSH account used on every worker. Defaults to your own $USER when omitted. + # There is no dedicated test account on a real VM cluster unless your site created + # one; nothing in this tool assumes one exists. + user: ${env:DTR_SSH_USER} + + # Path to the private key AS THE RUNNER SEES IT, not as this machine sees it. + # `ducktests-remote keys push` installs it there. + identity_file: ~/.ssh/id_rsa + + port: 22 + + # Where distributions live on the workers. ignitetest resolves a home directory as + # /, product being str(IgniteVersion(version)). + install_root: /opt + + # Host that runs the ducktape process. "local" runs it on this machine. + runner: ${env:DTR_RUNNER} + + # Run state (run dirs, logs, the venv) lives here on the runner. + state_root: ~/.ducktests-remote + + # Any number of workers: 2, 12, 49. Ranges expand, and a bare string is shorthand + # for {host: ...}. + nodes: + - host: worker[01-12].example.invalid + - host: worker13.example.invalid + ip: 203.0.113.13 # optional -> externally_routable_ip + - host: worker14.example.invalid + user: someone-else # per-host override for mixed clusters + + # Hosts that provision/deploy/clean/doctor should also target, but that ducktape + # must not schedule Ignite nodes onto - typically the runner itself. + extra_hosts: [] + +runner: + # Leave unset to use /venv, created on first run with the pinned + # requirements from modules/ducktests/tests/docker/requirements.txt. + venv: null + python: python3 + create_venv: true + +deploy: + dist_dir: ./dist + +jobs: 16 diff --git a/modules/ducktests/tests/ducktests_remote/examples/profile-ise-perf.yaml b/modules/ducktests/tests/ducktests_remote/examples/profile-ise-perf.yaml new file mode 100644 index 0000000000000..3f811a378986e --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/examples/profile-ise-perf.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A fork profile: what the long Jenkins --globals blob becomes once it is split up. +# Every value that would identify a real environment is an ${env:...} placeholder, +# because this file is committed to a public Apache repository. +# +# export ISE_USER=... ISE_PASSWORD=... +# ducktests-remote --profile ise-perf run +# +# ${file:PATH} is also available and reads a trimmed single-line file, which is what +# you want for a password kept outside the environment. + +globals: + project: ${env:DTR_PROJECT} + + ignite_versions: + - ${env:DTR_IGNITE_VERSION} + + ssl: + enabled: true + + authentication: + enabled: true + username: ${env:ISE_USER} + password: ${env:ISE_PASSWORD} + + # Fork spec override; resolve_spec() in ignitetest/services/utils/ignite_spec.py + # reads these from globals to swap in the fork's node startup. + NodeSpec: ${env:DTR_NODE_SPEC} + + # ignitetest defaults: persistent_root /mnt/service, install_root /opt. + # Override them here when the site layout differs. + # persistent_root: /data/ducktests + # install_root: /opt + +# Injected into the tests themselves, mapped to ducktape's --parameters. +# Only emitted when non-empty, matching run_tests.sh. +parameters: {} diff --git a/modules/ducktests/tests/ducktests_remote/examples/profile-smoke.yaml b/modules/ducktests/tests/ducktests_remote/examples/profile-smoke.yaml new file mode 100644 index 0000000000000..20aa05a7b0a62 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/examples/profile-smoke.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Smallest useful profile: the smoke suite against the locally built master. +# +# ducktests-remote --profile smoke run ./modules/ducktests/tests/ignitetest/tests/smoke_test.py +# +globals: + ignite_versions: ["dev"] diff --git a/modules/ducktests/tests/ducktests_remote/fanout.py b/modules/ducktests/tests/ducktests_remote/fanout.py new file mode 100644 index 0000000000000..dcd0d24d7ddfb --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/fanout.py @@ -0,0 +1,153 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Parallel per-host execution and the table it prints. + +The batch never aborts on the first failure unless asked to. When an operator has to +send a single request to whoever administers the machines, a partial list of broken +hosts is worse than useless. +""" + +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from typing import Any, Optional + +OK = "ok" +CHANGED = "changed" +SKIPPED = "skipped" +WARN = "warn" +FAILED = "failed" + +_ORDER = {FAILED: 0, WARN: 1, CHANGED: 2, SKIPPED: 3, OK: 4} + + +@dataclass +class HostResult: + """Outcome of one per-host operation.""" + + host: str + status: str = OK + message: str = "" + detail: str = "" + duration: float = 0.0 + data: Optional[Any] = field(default=None) + + @property + def ok(self): + """:return: True unless the operation failed.""" + return self.status != FAILED + + +class FanoutAborted(Exception): + """Raised when ``--fail-fast`` cut a batch short.""" + + +def fanout(hosts, operation, *, jobs=16, fail_fast=False): + """ + Apply ``operation(host)`` across ``hosts`` in a bounded thread pool. + + :param operation: callable returning a :class:`HostResult` (or a ``(status, message)`` + tuple, or None for plain success). + :param jobs: pool size. + :param fail_fast: stop scheduling further hosts after the first failure. + :return: results in inventory order. + """ + hosts = list(hosts) + if not hosts: + return [] + results = [None] * len(hosts) + stop = {"flag": False} + + def wrapped(index, host): + if stop["flag"]: + return HostResult(_host_name(host), SKIPPED, "skipped after earlier failure") + started = time.monotonic() + try: + outcome = operation(host) + except Exception as ex: # noqa: BLE001 - per-host isolation is the whole point + outcome = HostResult(_host_name(host), FAILED, _short(ex), detail=repr(ex)) + outcome = _normalise(outcome, host) + outcome.duration = time.monotonic() - started + if fail_fast and outcome.status == FAILED: + stop["flag"] = True + results[index] = outcome + return outcome + + with ThreadPoolExecutor(max_workers=max(1, int(jobs))) as pool: + list(pool.map(lambda pair: wrapped(*pair), list(enumerate(hosts)))) + + return [r for r in results if r is not None] + + +def _normalise(outcome, host): + name = _host_name(host) + if outcome is None: + return HostResult(name, OK) + if isinstance(outcome, HostResult): + return outcome + if isinstance(outcome, tuple): + status, message = (list(outcome) + [""])[:2] + return HostResult(name, status, message) + return HostResult(name, OK, str(outcome)) + + +def _host_name(host): + return getattr(host, "host", None) or str(host) + + +def _short(ex): + text = str(ex).strip().splitlines() + return text[0][:160] if text else ex.__class__.__name__ + + +def render_table(results, *, verbose=False, headers=("HOST", "STATUS", "TIME", "DETAIL")): + """:return: an aligned table of fan-out results, failures first within equal status.""" + if not results: + return "(no hosts)" + rows = [(r.host, r.status, "%.1fs" % r.duration, r.message or "") for r in results] + widths = [max(len(str(row[i])) for row in ([headers] + rows)) for i in range(4)] + lines = [" ".join(str(h).ljust(widths[i]) for i, h in enumerate(headers)).rstrip()] + lines.append(" ".join("-" * widths[i] for i in range(4))) + for row in rows: + lines.append(" ".join(str(c).ljust(widths[i]) for i, c in enumerate(row)).rstrip()) + if verbose: + for result in results: + if result.detail: + lines.append("") + lines.append("--- %s ---" % result.host) + lines.append(result.detail.rstrip()) + else: + for result in results: + if result.status == FAILED and result.detail: + lines.append("") + lines.append("--- %s ---" % result.host) + lines.append(result.detail.rstrip()) + return "\n".join(lines) + + +def summarise(results): + """:return: a compact ``9 ok, 2 failed`` style summary line.""" + counts = {} + for result in results: + counts[result.status] = counts.get(result.status, 0) + 1 + ordered = sorted(counts.items(), key=lambda kv: _ORDER.get(kv[0], 9)) + return ", ".join("%d %s" % (count, status) for status, count in ordered) or "nothing to do" + + +def any_failed(results): + """:return: True when at least one host failed.""" + return any(r.status == FAILED for r in results) diff --git a/modules/ducktests/tests/ducktests_remote/globals_builder.py b/modules/ducktests/tests/ducktests_remote/globals_builder.py new file mode 100644 index 0000000000000..7b70df8d397b7 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/globals_builder.py @@ -0,0 +1,199 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Composition of the ``--globals`` payload, and the redactor that keeps its secrets out of +everything the CLI prints. + +The Jenkins one-liner this replaces carries a kilobyte of JSON with a password and a set +of internal IPs on the command line, where it lands in shell history, process listings +and build logs. Here the same content is layered from profiles, resolved from the +environment at the last moment, and written straight to a 0600 file on the runner. +""" + +import json +import os +import re +from pathlib import Path + +from ducktests_remote.config import ConfigError, deep_merge, set_dotted + +_PLACEHOLDER = re.compile(r"\$\{(env|file):([^}]+)\}") + + +class Redactor: + """ + Replaces resolved secret *values* with ``***`` in anything the CLI emits. + + Keying on values rather than on key names is what makes this reliable: a password + that leaks into an unrelated field, a rendered command line or a traceback is still + caught. Key-name matching is only a fallback for values we never resolved ourselves. + """ + + MASK = "***" + SENSITIVE_KEYS = ("password", "passwd", "secret", "token", "keystore_pass", "truststore_pass") + + def __init__(self): + self._values = set() + + def add(self, value): + """Register a resolved secret value.""" + if isinstance(value, str) and len(value.strip()) >= 3: + self._values.add(value.strip()) + + @property + def values(self): + """:return: the registered secret values.""" + return frozenset(self._values) + + def redact(self, text): + """:return: ``text`` with every registered secret replaced by ``***``.""" + if text is None: + return None + result = str(text) + for secret in sorted(self._values, key=len, reverse=True): + result = result.replace(secret, self.MASK) + return result + + def redact_structure(self, data): + """:return: a copy of ``data`` with secret values, and sensitive keys, masked.""" + if isinstance(data, dict): + out = {} + for key, value in data.items(): + if isinstance(key, str) and key.lower() in self.SENSITIVE_KEYS: + out[key] = self.MASK + else: + out[key] = self.redact_structure(value) + return out + if isinstance(data, list): + return [self.redact_structure(item) for item in data] + if isinstance(data, str): + return self.redact(data) + return data + + +def interpolate(value, redactor, environ=None, source=""): + """ + Resolve ``${env:NAME}`` and ``${file:PATH}`` placeholders throughout a structure. + + A missing variable or file is a hard error naming both the placeholder and the file + it came from. Substituting an empty string instead would produce a run that fails + three hours later with an authentication error nobody can trace back to here. + """ + environ = os.environ if environ is None else environ + + if isinstance(value, dict): + return {k: interpolate(v, redactor, environ, source) for k, v in value.items()} + if isinstance(value, list): + return [interpolate(v, redactor, environ, source) for v in value] + if not isinstance(value, str): + return value + + resolved = value + for match in _PLACEHOLDER.finditer(value): + kind, ref = match.group(1), match.group(2).strip() + if kind == "env": + if ref not in environ: + raise ConfigError( + "%s: environment variable %r referenced by ${env:%s} is not set" + % (source, ref, ref)) + replacement = environ[ref] + else: + path = Path(os.path.expanduser(ref)) + if not path.is_file(): + raise ConfigError( + "%s: file %s referenced by ${file:%s} does not exist" % (source, path, ref)) + replacement = path.read_text(encoding="utf-8").strip() + redactor.add(replacement) + resolved = resolved.replace(match.group(0), replacement) + return resolved + + +def parse_kv_override(item): + """ + Parse a ``-g a.b.c=value`` argument. + + Values are parsed as JSON when they parse, so ``ssl.enabled=true`` yields a boolean + and ``project=ise`` yields a string. This mirrors ``_extend_json`` in + ``docker/run_tests.sh``, with nesting added. + """ + if "=" not in item: + raise ConfigError("expected KEY=VALUE, found %r" % item) + key, raw = item.split("=", 1) + key = key.strip() + if not key: + raise ConfigError("empty key in %r" % item) + try: + value = json.loads(raw) + except ValueError: + value = raw + return key, value + + +def build(base_layers=(), overrides=(), redactor=None, environ=None): + """ + Compose the final globals mapping. + + :param base_layers: ``(source_name, mapping)`` pairs, later layers winning. + :param overrides: raw ``KEY=VALUE`` strings from ``-g``. + :param redactor: :class:`Redactor` collecting resolved secrets; created when omitted. + :param environ: environment mapping used for ``${env:}``. + :return: ``(globals_dict, redactor)``. + """ + redactor = Redactor() if redactor is None else redactor + composed = {} + for source, layer in base_layers: + if not layer: + continue + if not isinstance(layer, dict): + raise ConfigError("%s: globals must be a mapping, found %s" + % (source, type(layer).__name__)) + composed = deep_merge(composed, interpolate(layer, redactor, environ, source)) + + for item in overrides: + key, value = parse_kv_override(item) + set_dotted(composed, key, interpolate(value, redactor, environ, "-g %s" % key)) + + return composed, redactor + + +def load_raw_layer(json_text=None, json_file=None): + """ + Parse a raw base layer from ``--globals-json`` / ``--globals-file``. + + This is the migration path: paste the existing Jenkins blob verbatim, get a working + run, then split it into YAML profiles one key at a time. + """ + if json_text and json_file: + raise ConfigError("pass only one of --globals-json / --globals-file") + if json_file: + path = Path(os.path.expanduser(json_file)) + if not path.is_file(): + raise ConfigError("globals file not found: %s" % path) + json_text = path.read_text(encoding="utf-8") + if not json_text: + return None + try: + data = json.loads(json_text) + except ValueError as ex: + raise ConfigError("globals is not valid JSON: %s" % ex) from ex + if not isinstance(data, dict): + raise ConfigError("globals must be a JSON object, found %s" % type(data).__name__) + return data + + +def dumps(data): + """:return: canonical JSON for a globals/parameters payload.""" + return json.dumps(data, indent=2, sort_keys=True) + "\n" diff --git a/modules/ducktests/tests/ducktests_remote/runs.py b/modules/ducktests/tests/ducktests_remote/runs.py new file mode 100644 index 0000000000000..8d73a6ffdd1f9 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/runs.py @@ -0,0 +1,357 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Run directory layout and state, all of it living on the runner. + +Nothing about a run is kept on the coordinator. That is what lets any coordinator +inspect, follow or stop a run that a different one started - including after the machine +that launched it has been closed and put in a bag. +""" + +import json +import os +import posixpath +import re +import secrets +import shlex +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import List, Optional + +RUN_ID_RE = re.compile(r"^[A-Za-z0-9._-]+-\d{8}-\d{6}-[0-9a-f]{4}$") + +RUNNING = "running" +FINISHED = "finished" +FAILED = "failed" +STOPPED = "stopped" +UNKNOWN = "unknown" + + +def new_run_id(user, now=None, entropy=None): + """ + :return: a run id of the form ``max-20260727-141233-9f2a``. + + The user and timestamp make it readable in a directory listing; the four hex + characters keep two runs started in the same second apart. + """ + now = now or datetime.now() + safe_user = re.sub(r"[^A-Za-z0-9._-]", "_", str(user or "unknown")) or "unknown" + suffix = entropy if entropy is not None else secrets.token_hex(2) + return "%s-%s-%s" % (safe_user, now.strftime("%Y%m%d-%H%M%S"), suffix) + + +def is_run_id(value): + """:return: True when ``value`` looks like a run id.""" + return bool(value) and bool(RUN_ID_RE.match(value)) + + +@dataclass +class RunPaths: + """Absolute, runner-side paths for one run.""" + + state_root: str + run_id: str + + @property + def runs_root(self): + """:return: ``/runs``.""" + return posixpath.join(self.state_root, "runs") + + @property + def run_dir(self): + """:return: ``/runs/``.""" + return posixpath.join(self.runs_root, self.run_id) + + @property + def latest_link(self): + """:return: the ``latest`` symlink beside the run directories.""" + return posixpath.join(self.runs_root, "latest") + + @property + def src_dir(self): + """:return: where this run's sources are synced.""" + return posixpath.join(self.state_root, "src", self.run_id) + + def path(self, *parts): + """:return: a path inside the run directory.""" + return posixpath.join(self.run_dir, *parts) + + meta = property(lambda self: self.path("meta.json")) + cluster_file = property(lambda self: self.path("cluster.json")) + globals_file = property(lambda self: self.path("globals.json")) + parameters_file = property(lambda self: self.path("parameters.json")) + run_script = property(lambda self: self.path("run.sh")) + launch_script = property(lambda self: self.path("launch.sh")) + pid_file = property(lambda self: self.path("pid")) + pgid_file = property(lambda self: self.path("pgid")) + exit_code_file = property(lambda self: self.path("exit_code")) + stopped_file = property(lambda self: self.path("stopped")) + log_file = property(lambda self: self.path("ducktape.log")) + results_dir = property(lambda self: self.path("results")) + + +@dataclass +class RunState: + """Everything ``status`` needs, derived from files on the runner.""" + + run_id: str + state: str = UNKNOWN + pid: Optional[int] = None + pgid: Optional[int] = None + exit_code: Optional[int] = None + meta: dict = field(default_factory=dict) + + @property + def started_at(self): + """:return: ISO timestamp recorded at launch, or None.""" + return self.meta.get("started_at") + + @property + def elapsed(self): + """:return: seconds since launch, or None when the start time is unknown.""" + started = self.meta.get("started_epoch") + if not started: + return None + end = self.meta.get("finished_epoch") or time.time() + return max(0.0, float(end) - float(started)) + + +def derive_state(*, pid_alive, exit_code, stopped): + """ + Turn the three observable facts into a state. + + ``exit_code`` present wins over liveness: the file is written last, so a process that + has already exited is never reported as running just because a pid got reused. + """ + if exit_code is not None: + if stopped: + return STOPPED + return FINISHED if exit_code == 0 else FAILED + if pid_alive: + return RUNNING + if stopped: + return STOPPED + return UNKNOWN + + +def read_state(transport, paths: RunPaths) -> RunState: + """Read one run's state from the runner in a single round trip.""" + script = _STATE_SCRIPT % {"run_dir": shlex.quote(paths.run_dir)} + result = transport.run_script(script, check=False) + fields = _parse_kv(result.stdout) + + pid = _as_int(fields.get("pid")) + pgid = _as_int(fields.get("pgid")) + exit_code = _as_int(fields.get("exit_code")) + meta = {} + if fields.get("meta"): + try: + meta = json.loads(fields["meta"]) + except ValueError: + meta = {} + state = derive_state(pid_alive=fields.get("alive") == "1", + exit_code=exit_code, + stopped=fields.get("stopped") == "1") + return RunState(paths.run_id, state, pid, pgid, exit_code, meta) + + +_STATE_SCRIPT = """ +set -u +rd=%(run_dir)s +say() { printf '%%s=%%s\\n' "$1" "$2"; } +pid="" +[ -f "$rd/pid" ] && pid=$(cat "$rd/pid" 2>/dev/null || true) +say pid "$pid" +[ -f "$rd/pgid" ] && say pgid "$(cat "$rd/pgid" 2>/dev/null || true)" +[ -f "$rd/exit_code" ] && say exit_code "$(cat "$rd/exit_code" 2>/dev/null || true)" +[ -f "$rd/stopped" ] && say stopped 1 +if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then say alive 1; else say alive 0; fi +if [ -f "$rd/meta.json" ]; then printf 'meta=%%s\\n' "$(tr -d '\\n' < "$rd/meta.json")"; fi +""" + + +def list_run_ids(transport, state_root) -> List[str]: + """:return: run ids present on the runner, newest first.""" + runs_root = posixpath.join(state_root, "runs") + result = transport.run(["ls", "-1", runs_root], check=False) + if not result.ok: + return [] + ids = [line.strip() for line in result.stdout.splitlines() if is_run_id(line.strip())] + return sorted(ids, reverse=True) + + +def resolve_run_id(transport, state_root, run_id=None): + """:return: ``run_id`` when given, otherwise the most recent run on the runner.""" + if run_id: + return run_id + ids = list_run_ids(transport, state_root) + if not ids: + return None + return ids[0] + + +def write_meta(transport, paths: RunPaths, meta): + """Write ``meta.json`` into the run directory.""" + transport.write_file(json.dumps(meta, indent=2, sort_keys=True) + "\n", paths.meta) + + +def utc_now_iso(): + """:return: the current time as an ISO-8601 UTC string.""" + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def format_duration(seconds): + """:return: ``1h 04m 12s`` style duration, or ``-`` for None.""" + if seconds is None: + return "-" + seconds = int(seconds) + hours, rest = divmod(seconds, 3600) + minutes, secs = divmod(rest, 60) + if hours: + return "%dh %02dm %02ds" % (hours, minutes, secs) + if minutes: + return "%dm %02ds" % (minutes, secs) + return "%ds" % secs + + +def _parse_kv(text): + fields = {} + for line in (text or "").split("\n"): + if "=" in line: + key, value = line.split("=", 1) + fields[key.strip()] = value.strip() + return fields + + +def _as_int(value): + try: + return int(str(value).strip()) + except (TypeError, ValueError): + return None + + +def default_state_root(state_root): + """:return: the state root with ``~`` left alone (it is expanded on the runner).""" + return state_root or "~/.ducktests-remote" + + +def local_fetch_dir(base, run_id): + """:return: the coordinator-side directory ``fetch`` writes into.""" + return os.path.join(str(base), run_id) + + +TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates") + + +def render_template(name, mapping): + """Substitute ``{{key}}`` placeholders in a packaged template.""" + with open(os.path.join(TEMPLATE_DIR, name), "r", encoding="utf-8") as handle: + text = handle.read() + for key, value in mapping.items(): + text = text.replace("{{%s}}" % key, str(value)) + left = re.search(r"\{\{(\w+)\}\}", text) + if left: + raise KeyError("template %s has an unfilled placeholder %r" % (name, left.group(1))) + return text + + +def render_run_script(*, version, timestamp, author, work_dir, results_root, cluster_file, + globals_file, test_paths, venv=None, parameters_file=None, + repeat=None, max_parallel=None, test_runner_timeout=None, + extra_args=()): + """ + Render the ``run.sh`` that the runner executes. + + ducktape 0.13 accepts a *file path* for ``--globals`` (``command_line/main.py`` + checks ``os.path.isfile`` before parsing the argument as JSON), so the composed + blob is referenced by path and never crosses a shell command line. + + Every interpolated value is shell-quoted, which is why paths with spaces and JSON + with braces survive the round trip. + """ + extra = [] + if parameters_file: + extra.append("--parameters %s" % shlex.quote(str(parameters_file))) + if repeat: + extra.append("--repeat %d" % int(repeat)) + if max_parallel: + extra.append("--max-parallel %d" % int(max_parallel)) + if test_runner_timeout: + extra.append("--test-runner-timeout %d" % int(test_runner_timeout)) + for arg in extra_args: + extra.append(shlex.quote(str(arg))) + + extra_lines = "".join(" \\\n %s" % item for item in extra) + + if venv: + activate = shlex.quote(posixpath.join(str(venv), "bin", "activate")) + venv_activate = ("# activate the runner venv; `set +u` because older activate\n" + "# scripts read unset variables\nset +u\n" + ". %s\nset -u\n" % activate) + else: + venv_activate = "# no venv configured: ducktape is expected on PATH\n" + + return render_template("run.sh.tmpl", { + "version": version, + "timestamp": timestamp, + "author": author, + "work_dir": shlex.quote(str(work_dir)), + "venv_activate": venv_activate, + "results_root": shlex.quote(str(results_root)), + "cluster_file": shlex.quote(str(cluster_file)), + "globals_file": shlex.quote(str(globals_file)), + "extra_lines": extra_lines, + "test_paths": " ".join(shlex.quote(str(p)) for p in test_paths), + }) + + +def render_launch_script(paths: RunPaths): + """ + Render the wrapper that detaches the run and records its exit code. + + The run is detached from second zero rather than "detached later": the failure worth + surviving is the coordinator going away, and a run that is only detachable after the + fact does not survive it. ``setsid`` puts ducktape in its own session so a dropped + SSH connection cannot SIGHUP it and leave Ignite JVMs alive on every worker; where + ``setsid`` is missing we fall back to ``nohup`` plus ``disown``. + """ + run_dir = shlex.quote(paths.run_dir) + return """#!/usr/bin/env bash +# Generated by ducktests-remote. Waits on run.sh and records its exit code. +rd=%s +ps -o pgid= -p $$ 2>/dev/null | tr -d ' ' > "$rd/pgid" || true +bash "$rd/run.sh" >> "$rd/ducktape.log" 2>&1 < /dev/null +echo $? > "$rd/exit_code" +""" % run_dir + + +def render_detach_script(paths: RunPaths): + """Render the snippet that starts ``launch.sh`` detached and records its pid.""" + run_dir = shlex.quote(paths.run_dir) + return """set -eu +rd=%s +cd "$rd" +: > "$rd/ducktape.log" +if command -v setsid >/dev/null 2>&1; then + setsid nohup bash "$rd/launch.sh" > /dev/null 2>&1 < /dev/null & +else + nohup bash "$rd/launch.sh" > /dev/null 2>&1 < /dev/null & + disown 2>/dev/null || true +fi +echo $! > "$rd/pid" +cat "$rd/pid" +""" % run_dir diff --git a/modules/ducktests/tests/ducktests_remote/sshdiag.py b/modules/ducktests/tests/ducktests_remote/sshdiag.py new file mode 100644 index 0000000000000..2037571e5f7e6 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/sshdiag.py @@ -0,0 +1,271 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +SSH failure classification and the "what to ask your administrator" block. + +The common first-time experience is not a subtle bug, it is *"nobody has added me to +these machines yet"*. A raw ``Permission denied (publickey)`` repeated twelve times +tells the operator nothing about what to ask for, so every SSH failure is classified and +mapped to a concrete next action, and the whole inventory is always probed before +anything is reported. + +The patterns below are derived from OpenSSH's own message strings. They have not been +replayed against every target distribution's build, so treat the table as data: adding a +distro-specific string is a one-line change and ``checks/check_remote_sshdiag.py`` is +table-driven over recorded samples. +""" + +import re +import shutil +from dataclasses import dataclass +from typing import List, Optional + +from ducktests_remote.transport import Result, TransportError, run_local + +OK = "ok" +UNRESOLVED = "unresolved" +NO_SSHD = "no-sshd" +UNREACHABLE = "unreachable" +NO_ACCESS = "no-access" +NO_USER = "no-user" +HOSTKEY = "hostkey" +NO_SUDO = "no-sudo" +UNKNOWN = "unknown" + +# Ordered: the first pattern that matches wins, so the specific ones come first. +PATTERNS = ( + (HOSTKEY, r"REMOTE HOST IDENTIFICATION HAS CHANGED|Host key verification failed|" + r"host key .* has changed"), + (UNRESOLVED, r"Could not resolve hostname|Name or service not known|" + r"nodename nor servname provided|Temporary failure in name resolution|" + r"Name does not resolve"), + (NO_SSHD, r"Connection refused|port \d+: Connection refused"), + (UNREACHABLE, r"Connection timed out|Operation timed out|No route to host|" + r"Network is unreachable|Host is unreachable|" + r"kex_exchange_identification: Connection closed"), + (NO_USER, r"Invalid user|no such user|Please login as the user|" + r"This account is currently not available"), + (NO_ACCESS, r"Permission denied \(publickey|Permission denied \(.*publickey|" + r"Too many authentication failures|no matching host key type found|" + r"Authentication failed"), + (NO_SUDO, r"sudo: a (password|terminal) is required|sudo: no tty present|" + r"a terminal is required to read the password|" + r"is not in the sudoers file"), +) + +_ADVICE = { + UNRESOLVED: "hostname does not resolve from here. Check VPN/DNS, or put the IP in " + "cluster.nodes[].ip.", + NO_SSHD: "host answers but nothing is listening on port {port}; sshd is down or on " + "another port.", + UNREACHABLE: "no network path to {host}:{port}. Firewall, routing or the host is down.", + NO_ACCESS: "your key is not authorised for user {user!r} on this host.", + NO_USER: "account {user!r} does not exist on this host.", + HOSTKEY: "the host key changed. Inspect it, then remove the stale line yourself; this " + "tool will never do it for you.", + NO_SUDO: "passwordless sudo is missing for {user!r}.", + UNKNOWN: "unrecognised ssh failure; rerun with -v for the full stderr.", +} + + +@dataclass +class SshDiagnosis: + """One host's SSH state.""" + + host: str + classification: str = OK + user: Optional[str] = None + port: int = 22 + stderr: str = "" + returncode: int = 0 + + @property + def ok(self): + """:return: True when the connection succeeded.""" + return self.classification == OK + + @property + def advice(self): + """:return: a one-line, host-specific next action.""" + template = _ADVICE.get(self.classification, _ADVICE[UNKNOWN]) + return template.format(host=self.host, user=self.user, port=self.port) + + +def classify(returncode, stderr): + """ + Map an ssh exit status plus stderr onto one of the classification constants. + + :param returncode: process exit status; 0 always means ``ok``. + :param stderr: captured stderr, matched case-insensitively. + """ + if returncode == 0: + return OK + text = stderr or "" + for name, pattern in PATTERNS: + if re.search(pattern, text, re.IGNORECASE): + return name + return UNKNOWN + + +def diagnose(result: Result, *, host, user=None, port=22): + """:return: an :class:`SshDiagnosis` built from a command :class:`Result`.""" + return SshDiagnosis(host=host, + classification=classify(result.returncode, result.stderr), + user=user, port=port, + stderr=(result.stderr or "").strip(), + returncode=result.returncode) + + +def diagnose_exception(ex, *, host, user=None, port=22): + """:return: an :class:`SshDiagnosis` built from a raised transport error.""" + stderr = str(ex) + result = getattr(ex, "result", None) + if isinstance(result, Result): + return diagnose(result, host=host, user=user, port=port) + return SshDiagnosis(host=host, classification=classify(255, stderr), user=user, port=port, + stderr=stderr, returncode=255) + + +def key_fingerprint(identity_file): + """ + :return: the fingerprint of the public key being offered, or None. + + The administrator needs this to confirm they are authorising the right key, and the + operator usually cannot produce it from memory. + """ + if not identity_file: + return None + keygen = shutil.which("ssh-keygen") + if not keygen: + return None + for candidate in (str(identity_file) + ".pub", str(identity_file)): + try: + result = run_local([keygen, "-l", "-f", candidate]) + except TransportError: + continue + if result.ok and result.out: + return result.out + return None + + +def public_key(identity_file): + """:return: the contents of ``.pub`` when readable, else None.""" + if not identity_file: + return None + try: + with open(str(identity_file) + ".pub", "r", encoding="utf-8") as handle: + return handle.read().strip() + except OSError: + return None + + +def summarise(diagnoses: List[SshDiagnosis]): + """:return: ``9 ok, 2 no-access, 1 no-user``.""" + counts = {} + for diagnosis in diagnoses: + counts[diagnosis.classification] = counts.get(diagnosis.classification, 0) + 1 + ordered = sorted(counts.items(), key=lambda kv: (kv[0] != OK, kv[0])) + return ", ".join("%d %s" % (count, name) for name, count in ordered) + + +def admin_request_block(diagnoses: List[SshDiagnosis], *, user, identity_file=None): + """ + Render the copy-pasteable block the operator forwards to whoever owns the machines. + + This block is the point of the whole classification exercise; the table above is + plumbing. It names the hosts, the account, the key, and the exact line to append. + """ + failures = [d for d in diagnoses if not d.ok] + if not failures: + return "" + + by_class = {} + for diagnosis in failures: + by_class.setdefault(diagnosis.classification, []).append(diagnosis.host) + + lines = ["", "=" * 72, + "WHAT TO ASK YOUR ADMINISTRATOR", + "=" * 72, + "%d of %d hosts are not usable. Summary: %s" + % (len(failures), len(diagnoses), summarise(diagnoses)), + ""] + + fingerprint = key_fingerprint(identity_file) + pubkey = public_key(identity_file) + + if NO_ACCESS in by_class: + hosts = by_class[NO_ACCESS] + lines += ["Please authorise my SSH key for the account %r on these %d host(s):" + % (user, len(hosts)), + _host_list(hosts), ""] + if fingerprint: + lines += [" key fingerprint offered: %s" % fingerprint] + lines += [" append this line to ~%s/.ssh/authorized_keys on each host:" % user, + " %s" % (pubkey or "" % (identity_file or "your key")), + ""] + + if NO_USER in by_class: + hosts = by_class[NO_USER] + working = [d.host for d in diagnoses if d.ok] + lines += ["The account %r does not exist on these %d host(s):" % (user, len(hosts)), + _host_list(hosts)] + if working: + lines += [" (it does exist on: %s)" % ", ".join(sorted(working)[:10])] + lines += [" please create it, or tell me which account to use instead.", ""] + + if NO_SSHD in by_class: + lines += ["sshd is not listening on these host(s):", + _host_list(by_class[NO_SSHD]), ""] + + if UNREACHABLE in by_class: + lines += ["No network path to these host(s) on port %d; please check the firewall:" + % (failures[0].port or 22), + _host_list(by_class[UNREACHABLE]), ""] + + if UNRESOLVED in by_class: + lines += ["These hostnames do not resolve from my machine. Either grant DNS/VPN " + "access, or give me their IP addresses:", + _host_list(by_class[UNRESOLVED]), ""] + + if NO_SUDO in by_class: + lines += ["Passwordless sudo is missing for %r on these host(s):" % user, + _host_list(by_class[NO_SUDO]), + " It is needed only by the network-segmentation suites", + " (ignitetest/tests/discovery_test.py, " + "ignitetest/tests/cellular_affinity_test.py), which call", + " `sudo iptables` from IgniteAwareService.drop_network. Every other test " + "runs unprivileged.", + " Required line: %s ALL=(ALL) NOPASSWD: /usr/sbin/iptables, " + "/usr/sbin/iptables-save, /usr/sbin/iptables-restore" % user, + ""] + + if HOSTKEY in by_class: + lines += ["Host key mismatch on these host(s) - verify before removing anything:", + _host_list(by_class[HOSTKEY])] + for host in by_class[HOSTKEY]: + lines.append(" ssh-keygen -R %s" % host) + lines.append("") + + if UNKNOWN in by_class: + lines += ["Unclassified ssh failures (rerun with -v for the full output):", + _host_list(by_class[UNKNOWN]), ""] + + lines.append("=" * 72) + return "\n".join(lines) + + +def _host_list(hosts): + return "\n".join(" %s" % host for host in sorted(hosts)) diff --git a/modules/ducktests/tests/ducktests_remote/templates/run.sh.tmpl b/modules/ducktests/tests/ducktests_remote/templates/run.sh.tmpl new file mode 100644 index 0000000000000..a59fd9580bc39 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/templates/run.sh.tmpl @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated by ducktests-remote {{version}} at {{timestamp}} by {{author}}. +# Do not edit: rerun `ducktests-remote run` instead. You can, however, ssh to this +# host and execute this exact script to reproduce the run by hand. +set -euo pipefail + +cd {{work_dir}} +{{venv_activate}} +exec ducktape \ + --results-root {{results_root}} \ + --cluster-file {{cluster_file}} \ + --globals {{globals_file}}{{extra_lines}} \ + {{test_paths}} diff --git a/modules/ducktests/tests/ducktests_remote/transport.py b/modules/ducktests/tests/ducktests_remote/transport.py new file mode 100644 index 0000000000000..2f13bdcf9c4bc --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/transport.py @@ -0,0 +1,510 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Command transport. + +Every command the CLI runs, locally or remotely, goes through a :class:`Transport`. +Nothing above this boundary may shell out to ``ssh`` directly, so ``--runner local`` and +``--runner some-host`` exercise identical code paths. +""" + +import abc +import os +import posixpath +import shlex +import shutil +import subprocess +import tarfile +import tempfile +import uuid +from dataclasses import dataclass, field +from fnmatch import fnmatch +from pathlib import Path +from typing import List, Optional, Sequence, Union + +# OpenSSH on Windows has no ControlMaster support; multiplexing options make it fail hard. +_MULTIPLEXING_SUPPORTED = os.name != "nt" + +DEFAULT_SSH_OPTIONS = ( + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ServerAliveInterval=30", +) + + +class TransportError(Exception): + """Raised when a transport-level operation fails (exit code 5 territory).""" + + def __init__(self, message, result=None): + super().__init__(message) + self.result = result + + +@dataclass +class Result: + """Outcome of a single command.""" + + argv: List[str] + returncode: int + stdout: str = "" + stderr: str = "" + host: str = "local" + + @property + def ok(self): + """:return: True when the command exited successfully.""" + return self.returncode == 0 + + @property + def out(self): + """:return: stdout with surrounding whitespace removed.""" + return self.stdout.strip() + + def check(self): + """Raise :class:`TransportError` unless the command succeeded.""" + if not self.ok: + raise TransportError( + "command failed on %s (exit %d): %s\n%s" + % (self.host, self.returncode, shlex.join(self.argv), self.stderr.strip()), + self) + return self + + +@dataclass +class Transport(abc.ABC): # pylint: disable=too-many-instance-attributes + """Runs commands and moves files on exactly one host.""" + + name: str = "local" + dry_run: bool = False + verbose: bool = False + printer: Optional[object] = None + _home: Optional[str] = field(default=None, init=False, repr=False) + + def home(self): + """:return: the home directory of the account this transport connects as.""" + if self._home is None: + if self.dry_run: + # Nothing is executed in a dry run, so the remote home is unknown; keep + # the tilde so the preview stays readable rather than inventing a path. + self._home = "~" + else: + result = self.run(["printenv", "HOME"], check=False) + self._home = result.out or "/root" + return self._home + + def expand(self, path): + """ + Expand a leading ``~`` against the *remote* home directory. + + Paths are shell-quoted before they reach the remote side, so a literal tilde + would never be expanded by the remote shell. Resolving it here, once per + connection, keeps ``~/.ducktests-remote`` working without giving up quoting. + """ + text = str(path) + if text == "~": + return self.home() + if text.startswith("~/"): + return posixpath.join(self.home(), text[2:]) + return text + + def _echo(self, message): + if self.printer is not None: + self.printer(message) + + def _trace(self, argv): + if self.dry_run: + self._echo("[dry-run] %s$ %s" % (self.name, shlex.join(argv))) + elif self.verbose: + self._echo("+ %s$ %s" % (self.name, shlex.join(argv))) + + @abc.abstractmethod + def run(self, argv, *, check=True, timeout=None, input=None): # noqa: A002 - matches spec + """Run ``argv`` (a list, never a string) and return a :class:`Result`.""" + + def run_script(self, script, **kw): + """ + Feed a shell script to ``bash -s`` over stdin. + + Long quoted one-liners are the single largest source of remote-execution bugs; + anything longer than a couple of words belongs here instead. + """ + return self.run(["bash", "-s"], input=script, **kw) + + @abc.abstractmethod + def upload(self, local_path, remote_path, *, mode=None): + """Copy a single local file to ``remote_path``.""" + + @abc.abstractmethod + def download(self, remote_path, local_path): + """Copy a single remote file to ``local_path``.""" + + @abc.abstractmethod + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False): + """Copy a directory tree; ``local_dir`` contents land inside ``remote_dir``.""" + + def exists(self, remote_path): + """:return: True when ``remote_path`` exists on this host.""" + if self.dry_run: + return True + return self.run(["test", "-e", remote_path], check=False).ok + + def mkdirs(self, remote_path, *, mode=None): + """Create ``remote_path`` and any missing parents.""" + self.run(["mkdir", "-p", remote_path]).check() + if mode is not None: + self.run(["chmod", "%o" % mode, remote_path]).check() + + def write_file(self, content, remote_path, *, mode=None): + """ + Write ``content`` to ``remote_path`` without going through a shell quote. + + Written as bytes with explicit LF endings: a coordinator on Windows would + otherwise translate every newline to CRLF, and a shell script with carriage + returns fails on the runner with a message that names the wrong line. + """ + with tempfile.TemporaryDirectory() as tmp: + local = Path(tmp) / "payload" + local.write_bytes(content.replace("\r\n", "\n").encode("utf-8")) + self.upload(local, remote_path, mode=mode) + + def read_file(self, remote_path, *, missing_ok=True): + """:return: contents of ``remote_path``, or None when it does not exist.""" + result = self.run(["cat", remote_path], check=False) + if result.ok: + return result.stdout + if missing_ok: + return None + return result.check().stdout + + +class LocalTransport(Transport): + """Runs everything in the coordinator's own filesystem.""" + + def __init__(self, **kw): + super().__init__(name="local", **kw) + + def run(self, argv, *, check=True, timeout=None, input=None): # noqa: A002 - matches spec + argv = [str(a) for a in argv] + self._trace(argv) + if self.dry_run: + return Result(argv, 0, host=self.name) + return _spawn(argv, host=self.name, check=check, timeout=timeout, input=input) + + def upload(self, local_path, remote_path, *, mode=None): + self._trace(["cp", str(local_path), remote_path]) + if self.dry_run: + return + Path(remote_path).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(str(local_path), remote_path) + if mode is not None: + os.chmod(remote_path, mode) + + def download(self, remote_path, local_path): + self._trace(["cp", remote_path, str(local_path)]) + if self.dry_run: + return + Path(local_path).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(remote_path, str(local_path)) + + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False): + self._trace(["cp", "-r", str(local_dir), remote_dir]) + if self.dry_run: + return + if delete and Path(remote_dir).exists(): + shutil.rmtree(remote_dir) + ignore = shutil.ignore_patterns(*[_exclude_to_glob(e) for e in excludes]) if excludes else None + shutil.copytree(str(local_dir), remote_dir, dirs_exist_ok=True, ignore=ignore, symlinks=True) + + +@dataclass +class SshTransport(Transport): + """ + Runs commands on a remote host through the *system* ``ssh``/``scp`` binaries. + + paramiko is deliberately not used: the system client brings ``~/.ssh/config``, + ``ProxyJump``, agent and Kerberos support along for free, and it is the same client + an engineer would use by hand when reproducing a failure. + """ + + user: Optional[str] = None + port: int = 22 + identity_file: Optional[str] = None + connect_timeout: int = 15 + control_dir: Optional[str] = None + _rsync: Optional[bool] = field(default=None, init=False, repr=False) + + def __post_init__(self): + if self.control_dir is None: + self.control_dir = tempfile.gettempdir() + + @property + def target(self): + """:return: ``user@host`` or plain host when no user is configured.""" + return "%s@%s" % (self.user, self.name) if self.user else self.name + + def ssh_options(self, *, for_scp=False): + """:return: the base ssh option list shared by every connection this class makes.""" + opts = list(DEFAULT_SSH_OPTIONS) + opts += ["-o", "ConnectTimeout=%d" % self.connect_timeout] + if _MULTIPLEXING_SUPPORTED: + # doctor and deploy open many connections per host; multiplexing turns each + # extra one into a no-cost channel on the first connection. + opts += ["-o", "ControlMaster=auto", + "-o", "ControlPersist=60s", + "-o", "ControlPath=%s" % os.path.join(self.control_dir, "dtr-%r@%h:%p")] + if self.identity_file: + opts += ["-o", "IdentitiesOnly=yes", "-i", os.path.expanduser(self.identity_file)] + if self.port and int(self.port) != 22: + opts += ["-P" if for_scp else "-p", str(self.port)] + return opts + + def run(self, argv, *, check=True, timeout=None, input=None): # noqa: A002 - matches spec + argv = [str(a) for a in argv] + remote = shlex.join(argv) + full = ["ssh"] + self.ssh_options() + ["-T", self.target, remote] + self._trace(argv) + if self.dry_run: + return Result(argv, 0, host=self.name) + result = _spawn(full, host=self.name, check=check, timeout=timeout, input=input) + return Result(argv, result.returncode, result.stdout, result.stderr, self.name) + + def upload(self, local_path, remote_path, *, mode=None): + self._trace(["scp", str(local_path), "%s:%s" % (self.target, remote_path)]) + if self.dry_run: + return + remote = "%s:%s" % (self.target, remote_path) + argv = ["scp"] + self.ssh_options(for_scp=True) + [str(local_path), remote] + _spawn(argv, host=self.name, check=True) + if mode is not None: + self.run(["chmod", "%o" % mode, remote_path]).check() + + def download(self, remote_path, local_path): + self._trace(["scp", "%s:%s" % (self.target, remote_path), str(local_path)]) + if self.dry_run: + return + Path(local_path).parent.mkdir(parents=True, exist_ok=True) + remote = "%s:%s" % (self.target, remote_path) + argv = ["scp"] + self.ssh_options(for_scp=True) + [remote, str(local_path)] + _spawn(argv, host=self.name, check=True) + + def has_rsync(self): + """:return: True when rsync is usable on both ends. Probed once, then cached.""" + if self._rsync is None: + if self.dry_run: + self._rsync = False + else: + local = shutil.which("rsync") is not None + remote = local and self.run(["command", "-v", "rsync"], check=False).ok + self._rsync = bool(remote) + return self._rsync + + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False): + local_dir = str(local_dir) + self._trace(["rsync", local_dir, "%s:%s" % (self.target, remote_dir)]) + if self.dry_run: + return + self.mkdirs(remote_dir) + if self.has_rsync(): + argv = ["rsync", "-az", "-e", shlex.join(["ssh"] + self.ssh_options())] + if delete: + argv.append("--delete") + for pattern in excludes: + argv += ["--exclude", pattern] + argv += [local_dir.rstrip("/\\") + "/", "%s:%s/" % (self.target, remote_dir)] + _spawn(argv, host=self.name, check=True) + return + # rsync missing on one of the ends: fall back to a tar stream through scp. + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / "payload.tar.gz" + make_tarball(local_dir, archive, excludes=excludes) + staged = "/tmp/dtr-upload-%s.tar.gz" % uuid.uuid4().hex[:8] + self.upload(archive, staged) + script = "set -eu\nmkdir -p %s\n" % shlex.quote(remote_dir) + if delete: + script += "rm -rf -- %s/*\n" % shlex.quote(remote_dir) + script += "tar -xzf %s -C %s\nrm -f -- %s\n" % ( + shlex.quote(staged), shlex.quote(remote_dir), shlex.quote(staged)) + self.run_script(script).check() + + +@dataclass +class ProxiedTransport(Transport): + """ + Runs commands on a host by hopping through another transport. + + Two uses: probing a worker over the connection *ducktape itself* will make (from the + runner, with the runner-side identity file), and ``deploy --via HOST``, where a + payload is uploaded once and then fanned out from a machine that is close to the + workers instead of once per host from a laptop. + """ + + via: Optional[Transport] = None + user: Optional[str] = None + port: int = 22 + identity_file: Optional[str] = None + connect_timeout: int = 15 + staging_dir: str = "/tmp" + + @property + def target(self): + """:return: ``user@host`` as seen from the intermediate host.""" + return "%s@%s" % (self.user, self.name) if self.user else self.name + + def ssh_options(self, *, for_scp=False): + """:return: ssh options for the second hop, evaluated on the intermediate host.""" + opts = list(DEFAULT_SSH_OPTIONS) + opts += ["-o", "ConnectTimeout=%d" % self.connect_timeout] + if self.identity_file: + opts += ["-o", "IdentitiesOnly=yes", "-i", self.identity_file] + if self.port and int(self.port) != 22: + opts += ["-P" if for_scp else "-p", str(self.port)] + return opts + + def run(self, argv, *, check=True, timeout=None, input=None): # noqa: A002 - matches spec + argv = [str(a) for a in argv] + hop = ["ssh"] + self.ssh_options() + ["-T", self.target, shlex.join(argv)] + result = self.via.run(hop, check=False, timeout=timeout, input=input) + result = Result(argv, result.returncode, result.stdout, result.stderr, self.name) + return result.check() if check else result + + def _staged(self, name): + return posixpath.join(self.staging_dir, "dtr-%s-%s" % (uuid.uuid4().hex[:8], name)) + + def upload(self, local_path, remote_path, *, mode=None): + staged = self._staged(Path(local_path).name) + self.via.upload(local_path, staged) + argv = ["scp"] + self.ssh_options(for_scp=True) + [staged, + "%s:%s" % (self.target, remote_path)] + try: + self.via.run(argv).check() + finally: + self.via.run(["rm", "-f", "--", staged], check=False) + if mode is not None: + self.run(["chmod", "%o" % mode, remote_path]).check() + + def download(self, remote_path, local_path): + staged = self._staged(posixpath.basename(remote_path)) + argv = ["scp"] + self.ssh_options(for_scp=True) + ["%s:%s" % (self.target, remote_path), + staged] + self.via.run(argv).check() + try: + self.via.download(staged, local_path) + finally: + self.via.run(["rm", "-f", "--", staged], check=False) + + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False): + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / "payload.tar.gz" + make_tarball(local_dir, archive, excludes=excludes) + staged = self._staged("payload.tar.gz") + self.via.upload(archive, staged) + try: + self.push_archive(staged, remote_dir, delete=delete) + finally: + self.via.run(["rm", "-f", "--", staged], check=False) + + def push_archive(self, staged_on_via, remote_dir, *, delete=False): + """Send an archive that already sits on the intermediate host to this host.""" + remote_tmp = self._staged("payload.tar.gz") + argv = ["scp"] + self.ssh_options(for_scp=True) + [ + staged_on_via, "%s:%s" % (self.target, remote_tmp)] + self.via.run(argv).check() + script = "set -eu\nmkdir -p %s\n" % shlex.quote(remote_dir) + if delete: + script += "rm -rf -- %s/*\n" % shlex.quote(remote_dir) + script += "tar -xzf %s -C %s\nrm -f -- %s\n" % ( + shlex.quote(remote_tmp), shlex.quote(remote_dir), shlex.quote(remote_tmp)) + self.run_script(script).check() + + +def make_tarball(source_dir, archive_path, *, excludes=()): + """Create a gzip tarball of the *contents* of ``source_dir``, honouring rsync-ish excludes.""" + source = Path(source_dir) + with tarfile.open(archive_path, "w:gz") as tar: + for entry in sorted(source.rglob("*")): + if entry.is_dir(): + continue + rel = entry.relative_to(source) + if is_excluded(rel, excludes): + continue + tar.add(str(entry), arcname=rel.as_posix()) + + +def is_excluded(rel_path, excludes): + """ + :return: True when ``rel_path`` matches one of the rsync-style ``excludes``. + + A pattern matches a whole relative path, any prefix of it, or any single path + component, so ``.git/``, ``*.pyc`` and ``target/`` all behave as expected. + """ + rel = Path(rel_path) + posix = rel.as_posix() + for raw in excludes: + pattern = raw.rstrip("/") + if not pattern: + continue + if fnmatch(posix, pattern) or fnmatch(posix, pattern + "/*"): + return True + if any(fnmatch(part, pattern) for part in rel.parts): + return True + return False + + +def _exclude_to_glob(exclude): + return exclude.rstrip("/") or exclude + + +def _spawn(argv, *, host, check=True, timeout=None, input=None): # noqa: A002 - matches spec + payload = input.encode("utf-8") if isinstance(input, str) else input + try: + completed = subprocess.run( # noqa: S603 - argv is always a list, never a shell string + argv, input=payload, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=timeout, check=False) + except FileNotFoundError as ex: + raise TransportError("%s: %s" % (argv[0], ex)) from ex + except subprocess.TimeoutExpired as ex: + raise TransportError("timed out after %ss on %s: %s" % (timeout, host, shlex.join(argv))) from ex + result = Result(argv, completed.returncode, + _decode(completed.stdout), _decode(completed.stderr), host) + if check: + result.check() + return result + + +def _decode(raw): + if raw is None: + return "" + return raw.decode("utf-8", errors="replace") + + +def run_local(argv, *, check=False, timeout=None, input=None): # noqa: A002 - matches spec + """Run a command on the coordinator itself, outside any transport (probes only).""" + return _spawn([str(a) for a in argv], host="local", check=check, timeout=timeout, input=input) + + +def build_transport(host: str, *, user=None, port=22, identity_file=None, connect_timeout=15, + dry_run=False, verbose=False, printer=None) -> Transport: + """:return: a :class:`LocalTransport` for ``local``, otherwise an :class:`SshTransport`.""" + if host in (None, "", "local", "localhost"): + return LocalTransport(dry_run=dry_run, verbose=verbose, printer=printer) + return SshTransport(name=host, user=user, port=int(port or 22), identity_file=identity_file, + connect_timeout=connect_timeout, dry_run=dry_run, verbose=verbose, + printer=printer) + + +def quote_all(values: Sequence[Union[str, Path]]) -> str: + """:return: the values shell-quoted and space joined.""" + return " ".join(shlex.quote(str(v)) for v in values) diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py index 06d8799077657..2c3ded900f311 100644 --- a/modules/ducktests/tests/setup.py +++ b/modules/ducktests/tests/setup.py @@ -31,4 +31,11 @@ packages=find_packages(exclude=["ignitetest.tests", "ignitetest.tests.*"]), include_package_data=True, install_requires=open('docker/requirements.txt').read(), + entry_points={ + "console_scripts": [ + # Coordinator-side CLI for running ducktests on a real VM cluster. + # It never imports ducktape; see ducktests_remote/README.md. + "ducktests-remote = ducktests_remote.cli:main", + ], + }, tests_require=["pytest==6.2.5"]) From 965b6a2cff712ac42ddb4de179b35913070aedd5 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 28 Jul 2026 15:00:20 +0300 Subject: [PATCH 2/9] Make the pip index and the workers' JDK configurable in ducktests-remote A runner inside a corporate network cannot reach PyPI, and a worker rarely carries the JDK the tests need on its non-interactive PATH. Both were assumptions the CLI made silently. pip: new `pip` config section (index_url, extra_index_url, trusted_host, timeout, retries, cert) rendered into command-line flags by pipconf and applied to every pip command the CLI runs. `run --install-sources` reached PyPI regardless of the configured index before this; it no longer does. `provision.pip_index_url` moves to `pip.index_url`. java: new `java` config section and a resolution ladder in java.py, shared by `provision --only jdk`, `provision --only ssh-env` and `doctor` - an explicit java.home, else the JVM already on PATH when its major matches, else a JDK under java.search_paths, else java.archive delivered from the coordinator reusing deploy's staging and atomic swap. The selected JDK is written to both ~/.ssh/environment and a marked block above the interactivity guard in ~/.bashrc, then verified over a fresh non-interactive session, because that is the JVM ignite.sh, the Kafka service, jmxterm and jvm_utils.java_version actually get. doctor fails preflight on a major that does not match. `provision.jdk_major` moves to `java.major`. ${env:} and ${file:} placeholders are now resolved in every config section, not only in globals; examples/cluster.yaml has always advertised this, but the values were passed through literally. --- .../tests/ducktests_remote/README.md | 132 +++- .../checks/check_remote_config.py | 57 ++ .../checks/check_remote_deploy.py | 4 +- .../checks/check_remote_java.py | 403 ++++++++++++ .../checks/check_remote_pip.py | 147 +++++ .../ducktests/tests/ducktests_remote/cli.py | 15 +- .../tests/ducktests_remote/commands/deploy.py | 26 +- .../tests/ducktests_remote/commands/doctor.py | 70 ++- .../ducktests_remote/commands/provision.py | 341 +++++++--- .../tests/ducktests_remote/commands/run.py | 27 +- .../tests/ducktests_remote/config.py | 108 +++- .../ducktests_remote/examples/cluster.yaml | 24 + .../tests/ducktests_remote/globals_builder.py | 44 +- .../ducktests/tests/ducktests_remote/java.py | 589 ++++++++++++++++++ .../tests/ducktests_remote/pipconf.py | 159 +++++ 15 files changed, 1975 insertions(+), 171 deletions(-) create mode 100644 modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py create mode 100644 modules/ducktests/tests/ducktests_remote/checks/check_remote_pip.py create mode 100644 modules/ducktests/tests/ducktests_remote/java.py create mode 100644 modules/ducktests/tests/ducktests_remote/pipconf.py diff --git a/modules/ducktests/tests/ducktests_remote/README.md b/modules/ducktests/tests/ducktests_remote/README.md index ff214c1604c4c..98c94638b0ce7 100644 --- a/modules/ducktests/tests/ducktests_remote/README.md +++ b/modules/ducktests/tests/ducktests_remote/README.md @@ -47,6 +47,44 @@ Runtime dependencies: the standard library and `PyYAML`. The CLI deliberately ** imports ducktape** — it drives ducktape on the runner, so it stays installable on a coordinator that has none. There is a unit check that fails if that ever changes. +### When PyPI is not reachable + +Only the runner ever runs pip: it creates the venv from `docker/requirements.txt`, and +`run --install-sources` installs the synced sources into it. Workers need no Python at +all. Point pip at your mirror once, in the config: + +```yaml +pip: + index_url: https://nexus.corp/repository/pypi/simple + extra_index_url: [https://nexus.corp/repository/pypi-internal/simple] + trusted_host: [nexus.corp] # plain http, or a certificate you cannot fix + cert: /etc/pki/tls/certs/corp-ca.pem + timeout: 60 + retries: 5 +``` + +or per invocation: `--pip-index-url`, `--pip-extra-index-url`, `--pip-trusted-host`, +`--pip-timeout`, `--pip-cert`. The flags become literal pip arguments — `--dry-run` shows +them — and they apply to *every* pip command the CLI runs, including the build +dependencies of `--install-sources`. + +`cert` is a **runner-side** path, exactly like `identity_file`: it is opened by pip on the +runner, so a file that exists on your laptop proves nothing. + +An index URL usually carries a token. Write it as `${env:NEXUS_URL}` and it is resolved on +the coordinator at launch and masked in everything the CLI prints; credentials typed +directly into a config file are masked too, on output, but they are then sitting in a file. + +The coordinator's own install obeys the same flags, in pip's own spelling: + +```bash +pip install --index-url https://nexus.corp/repository/pypi/simple -e . +``` + +`provision --only packages` uses apt/dnf and is *not* covered by any of this. Which OS +repositories a VM talks to is set when the image is built; this tool does not pretend to +manage it. + ## Quickstart Every command accepts `--dry-run`, and `--dry-run` is genuinely side-effect free: it @@ -147,7 +185,8 @@ globals: Then `ducktests-remote --profile ise-perf run -t ./isetest/perftests/`. Compare the two with `--dry-run` until the rendered `globals.json` matches, and delete the blob. -`${env:NAME}` and `${file:PATH}` are resolved on the coordinator at launch. A missing +`${env:NAME}` and `${file:PATH}` are resolved on the coordinator at launch, in `globals` +and in every other section alike — `cluster.user`, `pip.index_url`, `java.home`. A missing variable is a hard error naming the variable and the file it came from — never an empty string, never a run that fails on authentication three hours later. @@ -228,10 +267,10 @@ ducktests-remote provision --sudo --write-hosts | Step | What it does | | --- | --- | | `packages` | Installs the Dockerfile's system utilities. Detects apt/dnf/yum; an unknown package manager is a clear failure, not a guess. Needs `--sudo`. | -| `jdk` | Verifies `java -version` matches the expected major. Installing is opt-in (`--install-jdk`) because where a JDK comes from is site-specific. | +| `jdk` | Resolves a JDK of `java.major` per host and, when none is there, delivers `java.archive`. See "Choosing the JDK". | | `python` | Verifies only. Workers do not need Python — ducktape drives them over plain SSH. The runner's venv is created by `run`. | | `user` | `--create-user NAME` plus `--authorize-key`. Not run by default; most operators use their own account. Needs `--sudo`. | -| `ssh-env` | Writes `PATH`/`JAVA_HOME` into `~/.ssh/environment`. **The one that is easiest to forget.** | +| `ssh-env` | Points the workers' non-interactive `PATH`/`JAVA_HOME` at that JDK, then proves it. **The one that is easiest to forget.** | | `dirs` | Creates and chowns `/mnt/service` and the install root. Needs `--sudo`. | | `hosts` | `--write-hosts` rewrites only the block between `# BEGIN ducktests-remote` and `# END ducktests-remote` in `/etc/hosts`. Needs `--sudo`. | @@ -245,8 +284,85 @@ the `doctor` checks, so it ends with evidence rather than an assumption. ducktape runs every command over **non-interactive** SSH, where `~/.profile` is not sourced. A `java` that works fine when you log in by hand is simply absent during a test run, and the failure surfaces as an unrelated timeout. The Dockerfile solves this with -`PermitUserEnvironment yes` plus `~/.ssh/environment`; this step does the same and then -proves it by running `java -version` non-interactively. +`PermitUserEnvironment yes` plus `~/.ssh/environment`; this step does the same, adds a +`~/.bashrc` fallback, and then proves it by running `java -version` non-interactively. + +## Choosing the JDK + +### Why `PATH` matters more than `JAVA_HOME` + +`ignitetest` reaches a JVM four different ways, and only one of them respects `JAVA_HOME`: + +| Consumer | Mechanism | +| --- | --- | +| `ignite.sh`, via `IgniteSpec.envs()` | honours `JAVA_HOME` | +| `jvm_utils.java_version()` → `java -version` | bare `java`, so `PATH` | +| `services/kafka/kafka.py` → `nohup java …` | bare `java`, so `PATH` | +| `jmx_utils` → `java -jar jmxterm.jar` | bare `java`, so `PATH` | + +Setting `JAVA_HOME` therefore changes what `ignite.sh` uses and nothing else. Both are +set, and what a fresh non-interactive session actually gets is then verified rather than +assumed. + +### Configuration + +```yaml +java: + major: 17 # derived from the Dockerfile's eclipse-temurin:17 + home: /opt/jdk-17.0.11 # optional: use exactly this, no search + search_paths: [/opt, /usr/lib/jvm, /usr/java] + archive: ~/jdk/OpenJDK17U-jdk_x64_linux_hotspot.tar.gz + install_root: /opt # defaults to cluster.install_root + ssh_environment: true + bashrc: true +``` + +`provision --only jdk` resolves one JDK per host, in this order: + +1. **`java.home`** — verified, and a host that does not have it is a failure naming that + host. Explicit means explicit; falling back would defeat the point of saying it. +2. **the JVM already on the non-interactive `PATH`**, when its major matches. Nothing is + installed on a VM that is already correct. +3. **a JDK under `search_paths`** — `/opt/jdk-17.0.11`, `/usr/lib/jvm/java-17-openjdk`. + Highest patch level wins, compared numerically, so `17.0.11` beats `17.0.9`. +4. **`java.archive`**, delivered from the coordinator to the hosts that got this far — + and only to those. A `.tar.gz`, `.tgz`, `.tar` or an unpacked directory; a single + top-level directory is stripped, so a stock Temurin tarball lands as + `/opt/jdk-17.0.11+9`. Bad archives (no `bin/java`, a macOS build with `Contents/Home`, + a zip) fail on the coordinator, before anything is copied to twelve machines. +5. otherwise a failure listing every JDK that *was* found. `--install-jdk` (with `--sudo`) + adds the distribution's own package as a last rung. + +Delivery reuses `deploy`: staging plus an atomic swap, and a `.ducktests-java.json` +manifest so a host that already has the JDK is skipped. `--force` re-delivers. + +```bash +ducktests-remote provision --dry-run --only jdk # what each host would resolve to +ducktests-remote provision --only jdk --only ssh-env # resolve, install, then point PATH at it +ducktests-remote provision --only jdk --java-archive ~/jdk/temurin17.tar.gz +``` + +`provision --only ssh-env` runs the same ladder, so it can be used on its own and still +points at the JDK you asked for rather than at whatever `java` happens to be first. + +### Making it stick + +Both files are written, from one resolved value, in one step: + +- **`~/.ssh/environment`** — what the Dockerfile does. Silently ignored unless sshd carries + `PermitUserEnvironment yes`; `provision` says so when it does not. +- **`~/.bashrc`** — a marked block at the **top** of the file, above the + `case $- in *i*) ;; *) return;; esac` guard the stock file opens with. That guard exists + precisely because bash *does* source `~/.bashrc` for non-interactive ssh commands. It + does nothing when the account's login shell is not bash. + +Then a fresh connection is opened and asked what it got. That answer is the result: if it +is still the wrong JVM, the step fails there, not three hours into a run. + +`doctor` judges the same thing — what a non-interactive session gets — and a major that +does not match `java.major` is a **FAIL**, which stops `run` at preflight (exit 2). A +matching version from a JDK other than an explicitly configured `java.home` is a WARN: the +tests will run, but the pin is not in effect. ## Privileges the tests actually need @@ -349,8 +465,10 @@ nothing. `doctor` checks it on the runner and reports mode; `keys push` installs lifetime of your session. A run that lasts hours outlives it, and every subsequent worker connection then fails. Use a real key file on the runner. -**`java: command not found` deep inside a test.** Non-interactive SSH does not source -`~/.profile`. Run `provision --only ssh-env`. +**`java: command not found`, or the wrong JDK, deep inside a test.** Non-interactive SSH +does not source `~/.profile`. Run `provision --only jdk --only ssh-env`. If it still +reports the wrong JVM afterwards, sshd is ignoring `~/.ssh/environment` *and* the login +shell is not bash; set `java.home` to a JDK the site already puts on the default `PATH`. **Discovery failures with no useful message.** Workers that cannot resolve each other's hostnames fail inside discovery. `doctor` runs an N-way resolution probe; `provision diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_config.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_config.py index c310f0ddd40e7..715628d9b9c80 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_config.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_config.py @@ -139,3 +139,60 @@ def check_missing_config_file_is_reported(self): with pytest.raises(ConfigError) as ex: load_config(config_files=["/nonexistent/nope.yaml"], environ={}, user_config=None) assert "not found" in str(ex.value) + + +class CheckRenamedKeys: + """Keys that moved out of `provision` when pip and java got sections of their own.""" + + def check_the_old_pip_index_key_is_rejected_with_a_hint(self): + with pytest.raises(ConfigError) as ex: + validate({"provision": {"pip_index_url": "https://nexus.invalid/simple"}}) + assert "provision.pip_index_url" in str(ex.value) + + def check_the_old_jdk_major_key_is_rejected(self): + with pytest.raises(ConfigError) as ex: + validate({"provision": {"jdk_major": 17}}) + assert "provision.jdk_major" in str(ex.value) + + def check_the_new_sections_validate(self): + validate({"pip": {"index_url": "https://nexus.invalid/simple", "timeout": 60}, + "java": {"major": 17, "home": "/opt/jdk-17", "bashrc": False}}) + + def check_a_typo_in_the_new_sections_is_caught(self): + with pytest.raises(ConfigError) as ex: + validate({"java": {"majr": 17}}) + assert "java.majr" in str(ex.value) and "major" in str(ex.value) + + +class CheckInterpolation: + """``${env:}`` / ``${file:}`` outside the globals section.""" + + def check_a_placeholder_in_a_config_section_is_resolved(self): + # examples/cluster.yaml has always advertised this; before pip.* needed it, the + # placeholder was passed through literally and ssh went looking for a host named + # "${env:DTR_RUNNER}". + config = load_config(environ={"DTR_TEST_RUNNER": "build-vm-01"}, user_config=None, + overrides={"cluster": {"runner": "${env:DTR_TEST_RUNNER}"}}) + assert config["cluster"]["runner"] == "build-vm-01" + + def check_a_missing_variable_names_the_variable(self): + with pytest.raises(ConfigError) as ex: + load_config(environ={}, user_config=None, + overrides={"pip": {"index_url": "${env:DTR_ABSENT_INDEX}"}}) + assert "DTR_ABSENT_INDEX" in str(ex.value) + + def check_a_resolved_value_is_registered_for_redaction(self): + from ducktests_remote.globals_builder import Redactor # noqa: PLC0415 + + redactor = Redactor() + load_config(environ={"DTR_TEST_INDEX": "https://bob:s3cret@nexus.invalid/simple"}, + user_config=None, redactor=redactor, + overrides={"pip": {"index_url": "${env:DTR_TEST_INDEX}"}}) + assert redactor.redact("index is https://bob:s3cret@nexus.invalid/simple") \ + == "index is ***" + + def check_globals_are_left_for_the_globals_builder(self): + # Resolved there, per layer, so the error can name the profile it came from. + config = load_config(environ={}, user_config=None, + overrides={"globals": {"password": "${env:DTR_ABSENT}"}}) + assert config["globals"]["password"] == "${env:DTR_ABSENT}" diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py index 2913d815fcbf6..f890d9e4c849a 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py @@ -91,14 +91,14 @@ def check_manifest_filename_is_stable(self): assert deploy.MANIFEST_NAME == ".ducktests-deploy.json" def check_swap_removes_the_old_tree_only_after_the_move(self, tmp_path): - script = deploy._swap_script("/opt/.x.tmp.1", "/opt/x", False, None) # noqa: SLF001 + script = deploy.swap_script("/opt/.x.tmp.1", "/opt/x", False, None) move_index = script.index('mv -- "$staging" "$target"') remove_index = script.index('rm -rf -- "$old"') assert move_index < remove_index, \ "a half-copied distribution that looks present is worse than an absent one" def check_sudo_prefixes_every_privileged_command(self): - script = deploy._swap_script("/opt/.x.tmp.1", "/opt/x", True, "max") # noqa: SLF001 + script = deploy.swap_script("/opt/.x.tmp.1", "/opt/x", True, "max") assert script.count("sudo -n ") >= 3 assert "chown -R max" in script diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py new file mode 100644 index 0000000000000..b1421edda6841 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py @@ -0,0 +1,403 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for JDK resolution, the environment files, and the archive inspection.""" + +import io +import json +import tarfile + +import pytest + +from fake_transport import FakeTransport + +from ducktests_remote import java +from ducktests_remote.cli import Console, Context +from ducktests_remote.cluster import Node +from ducktests_remote.commands import doctor, provision +from ducktests_remote.config import ConfigError, load_config + +PROBE = """java_requested=17 +java=openjdk version "11.0.19" 2023-04-18 +java_path=/usr/bin/java +java_path_version=11.0.19 +java_path_major=11 +java_path_real=/usr/lib/jvm/java-11-openjdk/bin/java +java_env_home= +java_candidates=/usr/lib/jvm/java-11-openjdk:11:11.0.19,/opt/jdk-17.0.9:17:17.0.9,\ +/opt/jdk-17.0.11:17:17.0.11 +""" + +MATCHING = """java_requested=17 +java=openjdk version "17.0.11" 2024-04-16 +java_path=/usr/bin/java +java_path_version=17.0.11 +java_path_major=17 +java_path_real=/opt/jdk-17.0.11/bin/java +java_env_home=/opt/jdk-17.0.11 +java_candidates=/opt/jdk-17.0.11:17:17.0.11 +""" + + +def _cfg(**kw): + kw.setdefault("major", 17) + kw.setdefault("search_paths", ["/opt", "/usr/lib/jvm"]) + return java.JavaConfig(**kw) + + +class CheckVersionParsing: + """Mirrors ignitetest's jvm_utils.java_major_version.""" + + @pytest.mark.parametrize("version,major", [ + ("1.8.0_292", 8), ("11.0.19", 11), ("17.0.11+9", 17), ("21", 21), ("11-ea", 11), + ("", None), ("garbage", None), + ]) + def check_major(self, version, major): + assert java.major_of(version) == major + + def check_patch_levels_order_numerically(self): + assert java.version_key("17.0.9") < java.version_key("17.0.11") + + +class CheckResolution: + """The ladder, exercised against recorded probe output.""" + + def check_the_newest_matching_jdk_is_chosen(self): + res = java.parse_probe("w1", PROBE, _cfg()) + assert res.home == "/opt/jdk-17.0.11", "17.0.9 sorts after 17.0.11 as a string" + assert res.source == java.SEARCH + assert res.path_matches(17) is False, "PATH still points at Java 11" + + def check_a_matching_java_on_path_is_used_as_is(self): + res = java.parse_probe("w1", MATCHING, _cfg()) + assert res.source == java.CURRENT + assert res.home == "/opt/jdk-17.0.11" + assert res.path_matches(17) and res.home_in_effect + + def check_an_explicit_home_short_circuits_the_search(self): + res = java.parse_probe("w1", PROBE, _cfg(home="/opt/jdk-17.0.9")) + assert res.home == "/opt/jdk-17.0.9" and res.source == java.EXPLICIT + + def check_an_explicit_home_that_is_absent_selects_nothing(self): + # Deliberately not a fallback: an explicit java.home that silently resolves to a + # different JVM is worse than a failure naming the host. + assert not java.parse_probe("w1", PROBE, _cfg(home="/opt/nope")).selected + + def check_no_matching_jdk_selects_nothing_but_reports_what_is_there(self): + res = java.parse_probe("w1", PROBE, _cfg(major=21)) + assert not res.selected + assert ("/opt/jdk-17.0.11", 17, "17.0.11") in res.candidates + + def check_no_requested_version_accepts_whatever_is_there(self): + res = java.parse_probe("w1", PROBE, _cfg(major=None)) + assert res.home == "/usr/lib/jvm/java-11-openjdk" and res.source == java.CURRENT + + def check_a_symlinked_java_still_counts_as_the_selected_home(self): + res = java.parse_probe("w1", MATCHING, _cfg()) + assert res.home_in_effect, "/usr/bin/java resolves into the selected home" + + def check_an_empty_probe_is_survivable(self): + res = java.parse_probe("w1", "", _cfg()) + assert not res.selected and res.path_major is None + + +class CheckDiscoveryScript: + """The generated shell has to be safe to run from doctor.""" + + def check_paths_are_quoted(self): + script = java.discovery_script(_cfg(search_paths=["/opt/my jdks"], + home="/opt/vendor jdk")) + assert "'/opt/my jdks'" in script and "explicit='/opt/vendor jdk'" in script + + def check_it_never_writes_anything(self): + script = java.discovery_script(_cfg()) + for mutation in ("mkdir", "rm ", "mv ", "install", ">>", "chmod"): + assert mutation not in script, "discovery must be read-only: %r" % mutation + + def check_it_always_exits_zero(self): + # One unusable host must not abort the fan-out; the status is in the fields. + assert java.discovery_script(_cfg()).rstrip().endswith("exit 0") + + +class CheckEnvScript: + """Both environment files, written from one resolved value.""" + + def check_both_files_are_written(self): + script = java.env_script(_cfg(), "/opt/jdk-17") + assert "~/.ssh/environment" in script and "~/.bashrc" in script + + def check_each_file_can_be_switched_off(self): + # Composed in Python rather than guarded at runtime, so --dry-run shows only what + # the step will really do - and so a disabled section cannot print a note about + # itself as the fallback. + without_bashrc = java.env_script(_cfg(bashrc=False), "/opt/jdk-17") + assert "~/.bashrc" not in without_bashrc + assert "java.bashrc is off" in without_bashrc, "and the note says so" + without_ssh = java.env_script(_cfg(ssh_environment=False), "/opt/jdk-17") + assert "~/.ssh/environment" not in without_ssh and "~/.bashrc" in without_ssh + + def check_switching_both_off_is_refused(self): + with pytest.raises(ConfigError): + java.env_script(_cfg(ssh_environment=False, bashrc=False), "/opt/jdk-17") + + def check_the_bashrc_block_goes_above_the_interactivity_guard(self): + # The stock ~/.bashrc returns early for non-interactive shells, which is exactly + # the case ducktape runs in, so appending would be writing to /dev/null. + script = java.env_script(_cfg(), "/opt/jdk-17") + assert script.index(java.BLOCK_BEGIN) < script.index("awk 'BEGIN{skip=0}"), \ + "the block is emitted before the existing file is appended to it" + + def check_the_jdk_is_prepended_to_path_not_appended(self): + assert 'want_path="PATH=$jh/bin:$base_path' in java.env_script(_cfg(), "/opt/jdk-17") + + def check_the_path_is_not_grown_on_every_run(self): + # A host that honours ~/.ssh/environment feeds the composed PATH straight back in. + script = java.env_script(_cfg(), "/opt/jdk-17", ["/opt/venv/bin"]) + assert 'strip="$jh/bin:/opt/venv/bin"' in script + + def check_it_refuses_a_home_without_java(self): + assert '[ -x "$jh/bin/java" ]' in java.env_script(_cfg(), "/opt/not-a-jdk") + + +class CheckVerifyScript: + """What a fresh session gets is the authority, so it must be parseable.""" + + def check_it_reports_the_fields_the_resolver_reads(self): + script = java.verify_script() + for key in ("java_path", "java_path_version", "java_path_major", "java_env_home"): + assert "say %s " % key in script + + def check_its_output_round_trips(self): + verified = """java=openjdk version "17.0.11" 2024-04-16 +java_path=/usr/bin/java +java_path_version=17.0.11 +java_path_major=17 +java_path_real=/opt/jdk-17.0.11/bin/java +""" + assert java.parse_probe("w1", verified, _cfg()).path_matches(17) + + +def _tarball(tmp_path, names, top="jdk-17.0.11+9"): + path = tmp_path / "jdk.tar.gz" + with tarfile.open(path, "w:gz") as tar: + for name in names: + full = "%s/%s" % (top, name) if top else name + info = tarfile.TarInfo(full) + info.size = 4 + tar.addfile(info, io.BytesIO(b"data")) + return path + + +class CheckArchivePlan: + """A bad archive must fail on the coordinator, before it is copied anywhere.""" + + def check_a_single_top_level_directory_is_stripped(self, tmp_path): + plan = java.archive_plan(_tarball(tmp_path, ["bin/java", "lib/modules"])) + assert plan.strip == 1 and plan.top_level == "jdk-17.0.11+9" + assert plan.name == "jdk-17.0.11+9", "the target directory takes the JDK's own name" + assert plan.bytes == 8 + + def check_an_archive_without_bin_java_is_refused(self, tmp_path): + # A macOS build has Contents/Home in between and is worth catching here rather + # than on twelve hosts at once. + with pytest.raises(ConfigError) as ex: + java.archive_plan(_tarball(tmp_path, ["Contents/Home/bin/java"])) + assert "bin/java" in str(ex.value) + + def check_a_flat_archive_is_not_stripped(self, tmp_path): + plan = java.archive_plan(_tarball(tmp_path, ["bin/java", "lib/modules"], top="")) + assert plan.strip == 0 and plan.top_level is None + assert plan.name == "jdk" + + def check_a_directory_source_is_accepted(self, tmp_path): + home = tmp_path / "jdk-17" + (home / "bin").mkdir(parents=True) + (home / "bin" / "java").write_text("#!/bin/sh\n", encoding="utf-8") + plan = java.archive_plan(str(home)) + assert plan.kind == "dir" and plan.name == "jdk-17" + + def check_a_directory_without_bin_java_is_refused(self, tmp_path): + (tmp_path / "empty").mkdir() + with pytest.raises(ConfigError): + java.archive_plan(str(tmp_path / "empty")) + + def check_a_zip_names_the_supported_formats(self, tmp_path): + path = tmp_path / "jdk.zip" + path.write_bytes(b"PK\x03\x04") + with pytest.raises(ConfigError) as ex: + java.archive_plan(str(path)) + assert ".tar.gz" in str(ex.value) + + def check_a_missing_archive_names_the_path(self, tmp_path): + with pytest.raises(ConfigError) as ex: + java.archive_plan(str(tmp_path / "absent.tar.gz")) + assert "absent.tar.gz" in str(ex.value) + + def check_the_target_directory_can_be_renamed(self, tmp_path): + plan = java.archive_plan(_tarball(tmp_path, ["bin/java"])) + assert java.target_dir(_cfg(install_root="/opt", name="jdk-17"), plan) == "/opt/jdk-17" + assert java.target_dir(_cfg(install_root="/opt"), plan) == "/opt/jdk-17.0.11+9" + + +def _facts(text): + return {line.split("=", 1)[0]: line.split("=", 1)[1] + for line in text.strip().splitlines() if "=" in line} + + +def _context(probe_output, **java_cfg): + """A context whose single worker answers the JDK probe with ``probe_output``.""" + class Args: # pylint: disable=too-few-public-methods + """Minimal stand-in for the parsed command line.""" + + args = Args() + args.dry_run = False + args.sudo = False + args.force = False + args.install_jdk = False + args.num_nodes = None + + config = load_config(user_config=None) + config["java"].update(java_cfg) + config["cluster"]["nodes"] = [{"host": "w1"}] + + ctx = Context(config, args, Console(color=False)) + fake = FakeTransport(name="w1") + fake.when("java_requested", probe_output) + ctx._workers["w1"] = fake # noqa: SLF001 - the point of the fake + return ctx, fake + + +NODE = Node(host="w1", user="max", port=22, identity_file=None) + + +class CheckDelivery: + """`provision --only jdk`, on one host, with no network and no processes.""" + + def check_a_host_that_already_matches_is_left_alone(self): + ctx, fake = _context(MATCHING) + result = provision._jdk_on_host( # noqa: SLF001 + ctx, NODE, java.config_of(ctx), None, java.discovery_script(java.config_of(ctx))) + assert result.status == provision.OK + assert not fake.uploads, "nothing may be sent to a host that is already correct" + + def check_a_host_without_a_match_gets_the_archive(self, tmp_path): + archive = _tarball(tmp_path, ["bin/java", "lib/modules"]) + ctx, fake = _context(PROBE, major=21, archive=str(archive), install_root="/opt") + cfg = java.config_of(ctx) + plan = java.archive_plan(cfg.archive) + result = provision._jdk_on_host(ctx, NODE, cfg, plan, # noqa: SLF001 + java.discovery_script(cfg)) + + assert result.status == provision.CHANGED + assert fake.uploads, "the archive has to reach the host" + scripts = "\n".join(fake.scripts) + assert "--strip-components=1" in scripts, "a Temurin tarball has one top-level dir" + assert "/opt/jdk-17.0.11+9" in scripts, "the swap targets the JDK's own name" + manifest = [body for path, (body, _) in fake.files.items() + if path.endswith(provision.JAVA_MANIFEST_NAME)] + assert manifest and json.loads(manifest[0])["hash"] + + def check_a_host_that_already_has_that_archive_is_skipped(self, tmp_path): + archive = _tarball(tmp_path, ["bin/java"]) + ctx, fake = _context(PROBE, major=21, archive=str(archive), install_root="/opt") + cfg = java.config_of(ctx) + plan = java.archive_plan(cfg.archive) + digest = provision._tar_manifest(plan)["hash"] # noqa: SLF001 + fake.when("cat", json.dumps({"hash": digest})) + + result = provision._jdk_on_host(ctx, NODE, cfg, plan, # noqa: SLF001 + java.discovery_script(cfg)) + assert result.status == provision.OK and not fake.uploads + + def check_no_archive_and_no_match_fails_with_the_config_keys(self): + ctx, _ = _context(PROBE, major=21) + cfg = java.config_of(ctx) + result = provision._jdk_on_host(ctx, NODE, cfg, None, # noqa: SLF001 + java.discovery_script(cfg)) + assert result.status == provision.FAILED + assert "java.archive" in result.message and "java.home" in result.message + + def check_a_missing_explicit_home_is_never_papered_over(self, tmp_path): + # Even with an archive available: java.home means that JDK, not a similar one. + archive = _tarball(tmp_path, ["bin/java"]) + ctx, fake = _context(PROBE, home="/opt/vendor-jdk", archive=str(archive)) + cfg = java.config_of(ctx) + result = provision._jdk_on_host(ctx, NODE, cfg, # noqa: SLF001 + java.archive_plan(cfg.archive), + java.discovery_script(cfg)) + assert result.status == provision.FAILED and "/opt/vendor-jdk" in result.message + assert not fake.uploads + + +class CheckSshEnvStep: + """The verification, not the edit, decides the outcome.""" + + def check_a_session_that_still_gets_the_wrong_jvm_fails(self): + ctx, fake = _context(PROBE) # PATH java is 11, selected is 17 + fake.when("say java_path", PROBE) # ... and stays 11 after the write + results = provision._run_ssh_env_step(ctx, [NODE]) # noqa: SLF001 + assert results[0].status == provision.FAILED + assert "still gets" in results[0].message + + def check_a_session_that_gets_the_right_jvm_passes(self): + ctx, fake = _context(PROBE) + # Only the verify script matches this needle: the discovery script is answered by + # the earlier `java_requested` response, so this is the session *after* the write. + fake.when("say java_path", MATCHING) + results = provision._run_ssh_env_step(ctx, [NODE]) # noqa: SLF001 + assert results[0].status in (provision.OK, provision.CHANGED) + assert any("~/.bashrc" in script for script in fake.scripts) + assert any("~/.ssh/environment" in script for script in fake.scripts) + + def check_nothing_to_point_at_fails_before_writing(self): + ctx, fake = _context(PROBE, major=21) + results = provision._run_ssh_env_step(ctx, [NODE]) # noqa: SLF001 + assert results[0].status == provision.FAILED + assert "provision --only jdk" in results[0].message + assert not any("~/.bashrc" in s for s in fake.scripts) + + +class CheckDoctorVerdicts: + """The preflight judges what a non-interactive session gets, nothing else.""" + + def check_a_matching_jdk_passes(self): + checks = doctor._java_checks({"w1": _facts(MATCHING)}, _cfg()) # noqa: SLF001 + assert [c.status for c in checks] == [doctor.OK] + + def check_a_wrong_version_fails_and_names_the_remedy(self): + checks = doctor._java_checks({"w1": _facts(PROBE)}, _cfg()) # noqa: SLF001 + assert checks[0].status == doctor.FAIL + assert "/opt/jdk-17.0.11" in checks[0].message and "provision" in checks[0].message + + def check_a_wrong_version_with_nothing_installed_names_the_config_keys(self): + checks = doctor._java_checks({"w1": _facts(PROBE)}, _cfg(major=21)) # noqa: SLF001 + assert checks[0].status == doctor.FAIL + assert "java.archive" in checks[0].message + + def check_a_missing_java_fails(self): + checks = doctor._java_checks({"w1": {}}, _cfg()) # noqa: SLF001 + assert checks[0].status == doctor.FAIL and "PATH" in checks[0].message + + def check_an_explicit_home_that_is_not_in_effect_only_warns(self): + # Right version, wrong JDK: the tests will still run, so this is not a failure. + checks = doctor._java_checks( # noqa: SLF001 + {"w1": _facts(MATCHING)}, _cfg(home="/opt/jdk-17-vendor")) + assert checks[0].status == doctor.WARN and "java.home" in checks[0].message + + def check_mixed_jdks_are_reported_once_for_the_cluster(self): + facts = {"w1": _facts(MATCHING), "w2": _facts(MATCHING)} + facts["w2"]["java_path_version"] = "17.0.2" + checks = doctor._java_checks(facts, _cfg()) # noqa: SLF001 + assert [c for c in checks if c.name == "java-consistency"] diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_pip.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_pip.py new file mode 100644 index 0000000000000..1dd6452ea450e --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_pip.py @@ -0,0 +1,147 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for the pip index configuration and for the commands that carry it.""" + +import pytest + +from fake_transport import FakeTransport + +from ducktests_remote import pipconf +from ducktests_remote.cli import Console, Context +from ducktests_remote.commands import run +from ducktests_remote.config import ConfigError, load_config + + +def _config(**pip): + config = load_config(user_config=None) + config["pip"].update(pip) + return config + + +class CheckArguments: + """What ends up on the pip command line.""" + + def check_nothing_configured_adds_no_arguments(self): + # The rendered command has to stay byte for byte what it was before this section + # existed, or every golden-string check in here becomes a lie. + assert pipconf.pip_args(_config()) == [] + assert pipconf.pip_args_str(_config()) == "" + + def check_full_configuration_in_a_fixed_order(self): + config = _config(index_url="https://nexus.invalid/simple", + extra_index_url=["https://nexus.invalid/internal"], + trusted_host=["nexus.invalid"], timeout=60, retries=5, + cert="/etc/pki/corp.pem") + assert pipconf.pip_args(config) == [ + "--index-url", "https://nexus.invalid/simple", + "--extra-index-url", "https://nexus.invalid/internal", + "--trusted-host", "nexus.invalid", + "--timeout", "60", + "--retries", "5", + "--cert", "/etc/pki/corp.pem", + ] + + def check_a_bare_string_is_accepted_where_a_list_is_expected(self): + config = _config(extra_index_url="https://a.invalid/s", trusted_host="a.invalid") + assert pipconf.pip_args(config) == ["--extra-index-url", "https://a.invalid/s", + "--trusted-host", "a.invalid"] + + def check_repeatable_values_are_repeated(self): + config = _config(trusted_host=["a.invalid", "b.invalid"]) + assert pipconf.pip_args(config).count("--trusted-host") == 2 + + def check_blank_entries_are_dropped(self): + assert pipconf.pip_args(_config(index_url=" ", trusted_host=["", " "])) == [] + + def check_arguments_are_quoted_for_a_script(self): + rendered = pipconf.pip_args_str(_config(cert="/etc/pki/corp ca.pem")) + assert "'/etc/pki/corp ca.pem'" in rendered + + def check_a_bad_timeout_names_the_key(self): + with pytest.raises(ConfigError) as ex: + pipconf.pip_args(_config(timeout="soon")) + assert "pip.timeout" in str(ex.value) + + def check_a_negative_timeout_is_rejected(self): + with pytest.raises(ConfigError): + pipconf.pip_args(_config(timeout=0)) + + def check_a_non_string_index_names_the_key(self): + with pytest.raises(ConfigError) as ex: + pipconf.pip_args(_config(index_url=["https://a.invalid"])) + assert "pip.index_url" in str(ex.value) + + +class CheckDescription: + """The one-line summary doctor prints.""" + + def check_credentials_never_reach_the_terminal(self): + # This URL came straight from a config file, so the value-keyed redactor has + # never seen it; masking the userinfo is the only thing standing in the way. + line = pipconf.describe(_config(index_url="https://bob:s3cret@nexus.invalid/simple")) + assert "s3cret" not in line and "bob" not in line + assert "https://***@nexus.invalid/simple" in line + + def check_no_configuration_says_pypi(self): + assert "PyPI" in pipconf.describe(_config()) + + def check_every_configured_value_is_mentioned(self): + line = pipconf.describe(_config(index_url="https://nexus.invalid/simple", + trusted_host="nexus.invalid", timeout=60, + retries=5, cert="/etc/pki/corp.pem")) + for expected in ("nexus.invalid", "trusted", "60", "5", "corp.pem"): + assert expected in line + + +def _context(config, **args): + class Args: # pylint: disable=too-few-public-methods + """Minimal stand-in for the parsed command line.""" + + parsed = Args() + parsed.dry_run = False + for key, value in args.items(): + setattr(parsed, key, value) + ctx = Context(config, parsed, Console(color=False)) + ctx._runner = FakeTransport() # noqa: SLF001 - the point of the fake + return ctx + + +class CheckCallSites: + """Both pip invocations have to carry the configuration, not just the first.""" + + def check_the_venv_install_carries_the_index(self): + config = _config(index_url="https://nexus.invalid/simple", trusted_host="nexus.invalid") + ctx = _context(config) + ctx.runner.when("import ducktape", "ducktape 0.13.0") + run._ensure_venv(ctx, "/work") # noqa: SLF001 - internal by design + script = "\n".join(ctx.runner.scripts) + assert "--index-url https://nexus.invalid/simple" in script + assert "--trusted-host nexus.invalid" in script + + def check_install_sources_carries_the_index_too(self): + # It did not, before pip.* existed: `--install-sources` went to PyPI whatever the + # configured index was, and the run failed minutes into a job nobody was watching. + ctx = _context(_config(index_url="https://nexus.invalid/simple")) + run._install_sources(ctx, "/work") # noqa: SLF001 - internal by design + argv = ctx.runner.commands[-1] + assert "--index-url" in argv and "https://nexus.invalid/simple" in argv + assert argv[-2:] == ["-e", "/work/modules/ducktests/tests"] + + def check_an_unconfigured_venv_script_stays_unchanged(self): + ctx = _context(_config()) + ctx.runner.when("import ducktape", "ducktape 0.13.0") + run._ensure_venv(ctx, "/work") # noqa: SLF001 - internal by design + assert "install --disable-pip-version-check -r" in "\n".join(ctx.runner.scripts) diff --git a/modules/ducktests/tests/ducktests_remote/cli.py b/modules/ducktests/tests/ducktests_remote/cli.py index ab2b2a973428d..513bb62225272 100644 --- a/modules/ducktests/tests/ducktests_remote/cli.py +++ b/modules/ducktests/tests/ducktests_remote/cli.py @@ -226,7 +226,17 @@ def _flag_overrides(args): ("state_root", "cluster.state_root"), ("dist_dir", "deploy.dist_dir"), ("source_root", "run.source_root"), - ("work_dir", "run.work_dir")): + ("work_dir", "run.work_dir"), + # Repeatable pip flags replace the configured list rather than + # extending it, matching how deep_merge treats every other list. + ("pip_index_url", "pip.index_url"), + ("pip_extra_index_url", "pip.extra_index_url"), + ("pip_trusted_host", "pip.trusted_host"), + ("pip_timeout", "pip.timeout"), + ("pip_cert", "pip.cert"), + ("java_home", "java.home"), + ("java_major", "java.major"), + ("java_archive", "java.archive")): value = getattr(args, flag, None) if value: set_dotted(overlay, dotted, value) @@ -265,7 +275,8 @@ def main(argv=None): try: config = load_config(config_files=getattr(args, "config", None) or [], profiles=getattr(args, "profile", None) or [], - overrides=_flag_overrides(args)) + overrides=_flag_overrides(args), + redactor=console.redactor) ctx = Context(config, args, console) return args.handler(ctx) except ConfigError as ex: diff --git a/modules/ducktests/tests/ducktests_remote/commands/deploy.py b/modules/ducktests/tests/ducktests_remote/commands/deploy.py index a52072892be85..91bf5dcfc0b81 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/deploy.py +++ b/modules/ducktests/tests/ducktests_remote/commands/deploy.py @@ -20,6 +20,10 @@ ``/``; the name is never interpreted, rewritten, or checked against version-parsing logic. The operator names the directories to match what the tests expect, which is also what makes fork layouts work without special cases. + +:func:`build_manifest`, :func:`prepare_script`, :func:`swap_script` and :func:`human` are +public because ``provision``'s ``jdk`` step delivers a JDK the same way and must not grow +a second copy of the staging-and-swap logic. """ import hashlib @@ -156,17 +160,17 @@ def _sha256(path): def _print_cost(ctx, plans, nodes): total = sum(m["bytes"] for _, m in plans) - per_host = _human(total) + per_host = human(total) console = ctx.console console.info("%d distribution(s), %s each, %d host(s) = %s total" - % (len(plans), per_host, len(nodes), _human(total * len(nodes)))) + % (len(plans), per_host, len(nodes), human(total * len(nodes)))) if not ctx.args.via and len(nodes) > 3 and total > 200 * 1024 * 1024: console.warn("that is %s over the wire from this machine. `--via ` uploads it once and fans out from there." - % _human(total * len(nodes))) + % human(total * len(nodes))) -def _human(size): +def human(size): value = float(size) for unit in ("B", "KB", "MB", "GB", "TB"): if value < 1024 or unit == "TB": @@ -188,7 +192,7 @@ def _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes): if ctx.dry_run: return [HostResult(node.host, SKIPPED, - "would send %s to %s" % (_human(manifest["bytes"]), target)) + "would send %s to %s" % (human(manifest["bytes"]), target)) for node in nodes] with tempfile.TemporaryDirectory() as tmp: @@ -258,10 +262,10 @@ def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, archive, identity_file=node.identity_file, staging_dir=ctx.config["deploy"]["staging_dir"], dry_run=ctx.dry_run, verbose=ctx.console.verbose) - proxied.run_script(_prepare_script(staging, ctx.args.sudo)).check() + proxied.run_script(prepare_script(staging, ctx.args.sudo)).check() proxied.push_archive(staged_on_via, staging) else: - transport.run_script(_prepare_script(staging, ctx.args.sudo)).check() + transport.run_script(prepare_script(staging, ctx.args.sudo)).check() remote_archive = "%s/.payload.tar.gz" % staging transport.upload(archive, remote_archive) transport.run_script( @@ -270,18 +274,18 @@ def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, archive, shlex.quote(remote_archive))).check() transport.write_file(manifest_body, posixpath.join(staging, MANIFEST_NAME)) - transport.run_script(_swap_script(staging, target, ctx.args.sudo, ctx.args.owner)).check() + transport.run_script(swap_script(staging, target, ctx.args.sudo, ctx.args.owner)).check() return HostResult(node.host, CHANGED, "%s files, %s" - % (manifest["files"], _human(manifest["bytes"]))) + % (manifest["files"], human(manifest["bytes"]))) -def _prepare_script(staging, use_sudo): +def prepare_script(staging, use_sudo): sudo = "sudo -n " if use_sudo else "" return "set -eu\n%(sudo)srm -rf -- %(staging)s\n%(sudo)smkdir -p %(staging)s\n" % { "sudo": sudo, "staging": shlex.quote(staging)} -def _swap_script(staging, target, use_sudo, owner): +def swap_script(staging, target, use_sudo, owner): """ Swap the freshly extracted tree into place, then delete the old one. diff --git a/modules/ducktests/tests/ducktests_remote/commands/doctor.py b/modules/ducktests/tests/ducktests_remote/commands/doctor.py index b0ad82205169f..e0e133bd8f0e2 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/doctor.py +++ b/modules/ducktests/tests/ducktests_remote/commands/doctor.py @@ -28,7 +28,7 @@ import shutil import time -from ducktests_remote import sshdiag +from ducktests_remote import java, pipconf, sshdiag from ducktests_remote.cli import EXIT_OK, EXIT_PREFLIGHT from ducktests_remote.config import (REQUIREMENTS_RELPATH, SUDO_DEPENDENT_TESTS, expand_path) @@ -244,6 +244,10 @@ def runner_checks(ctx): checks.append(_runner_ducktape_check(ctx)) checks.append(_identity_check(ctx)) checks.append(_runner_disk_check(ctx)) + # Informational: reaching the index is pip's business, and doctor does not make + # network calls on someone else's behalf to find out. + checks.append(Check("runner", host, "pip", OK, + pipconf.describe(ctx.config, ctx.console.redactor))) return checks @@ -367,11 +371,13 @@ def worker_checks(ctx, nodes, versions): install_root = ctx.cluster_cfg.get("install_root", "/opt") persistent = (ctx.config["clean"]["paths"] or ["/mnt/service"])[0] pattern = ctx.config["clean"]["process_pattern"] + java_cfg = java.config_of(ctx) script = _WORKER_SCRIPT % { "install_root": shlex.quote(install_root), "persistent": shlex.quote(persistent), "pattern": shlex.quote(pattern), + "java_probe": java.discovery_script(java_cfg), } def probe(node): @@ -383,7 +389,7 @@ def probe(node): facts = {r.host: (r.data or {}) for r in results} checks = [] - checks += _java_checks(facts) + checks += _java_checks(facts, java_cfg) for host in sorted(facts): fact = facts[host] checks.append(_disk_from_facts(host, "install_free_gb", install_root, fact)) @@ -400,7 +406,6 @@ def probe(node): say() { printf '%%s=%%s\\n' "$1" "$2"; } say epoch "$(date +%%s)" say whoami "$(id -un 2>/dev/null || echo '?')" -say java "$(java -version 2>&1 | head -n1 | tr -d '\\r' || echo missing)" say install_free_gb "$(df -Pk %(install_root)s 2>/dev/null | awk 'NR==2{printf "%%d", $4/1048576}')" say install_writable "$([ -w %(install_root)s ] && echo 1 || echo 0)" if [ -d %(persistent)s ]; then @@ -414,30 +419,65 @@ def probe(node): say stale "$(pgrep -f %(pattern)s 2>/dev/null | wc -l | tr -d ' ')" if sudo -n true 2>/dev/null; then say sudo 1; else say sudo 0; fi say install_dirs "$(ls -1 %(install_root)s 2>/dev/null | tr '\\n' ',' )" +# Last, because the JDK probe ends in `exit 0`. +%(java_probe)s """ -def _java_checks(facts): +def _java_checks(facts, java_cfg): + """ + Judge the JVM the tests will actually get. + + The question is deliberately not "is a good JDK installed somewhere" but "what does a + non-interactive ssh session get", because that is the JVM ``ignite.sh``, the Kafka + service, jmxterm and ``jvm_utils.java_version`` all end up running. A JDK of the + right version sitting unused in ``/opt`` is a remedy to suggest, not a pass. + """ checks = [] - versions = {} - for host, fact in facts.items(): - java = fact.get("java", "missing") - if not java or "missing" in java or "not found" in java: + seen = {} + for host in sorted(facts): + res = java.parse_facts(host, facts[host], java_cfg) + version = facts[host].get("java", "missing") + if res.path_major is None: checks.append(Check("workers", host, "java", FAIL, - "java not on the non-interactive PATH; see `provision --only " - "ssh-env`")) + "java not on the non-interactive PATH; run `provision " + "--only jdk --only ssh-env`")) continue - versions.setdefault(java, []).append(host) - checks.append(Check("workers", host, "java", OK, java)) - if len(versions) > 1: - majority = max(versions, key=lambda k: len(versions[k])) - outliers = [h for k, hosts in versions.items() if k != majority for h in hosts] + seen.setdefault(res.path_version or version, []).append(host) + checks.append(_java_check(host, res, java_cfg)) + + if len(seen) > 1: + majority = max(seen, key=lambda k: len(seen[k])) + outliers = [h for k, hosts in seen.items() if k != majority for h in hosts] checks.append(Check("workers", "-", "java-consistency", WARN, "mixed JDKs; majority %r, outliers: %s" % (majority, ", ".join(sorted(outliers))))) return checks +def _java_check(host, res, java_cfg): + if not res.path_matches(java_cfg.major): + if res.selected: + return Check("workers", host, "java", FAIL, + "non-interactive java is %s, requested %s; %s is present on this " + "host - run `provision --only jdk --only ssh-env`" + % (res.path_version, java_cfg.major, res.home)) + found = ", ".join("%s (Java %s)" % (home, major) for home, major, _ in res.candidates) + return Check("workers", host, "java", FAIL, + "non-interactive java is %s, requested %s, and no Java %s is " + "installed here. Found: %s. Set java.archive to deliver one, or " + "java.home to name an existing one." + % (res.path_version, java_cfg.major, java_cfg.major, found or "nothing")) + if java_cfg.home and not res.home_in_effect: + # Right version, wrong JDK: the tests will run, so this is not a failure, but an + # explicit java.home that is not in effect is always worth saying out loud. + return Check("workers", host, "java", WARN, + "non-interactive java is %s from %s, but java.home names %s" + % (res.path_version, res.path_real or res.path_java, java_cfg.home)) + return Check("workers", host, "java", OK, + "%s from %s" % (res.path_version, res.path_real or res.path_java)) + + def _disk_from_facts(host, key, path, fact=None): if fact is None: return None diff --git a/modules/ducktests/tests/ducktests_remote/commands/provision.py b/modules/ducktests/tests/ducktests_remote/commands/provision.py index 3ccb06dd4f659..204df63d5974c 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/provision.py +++ b/modules/ducktests/tests/ducktests_remote/commands/provision.py @@ -23,21 +23,29 @@ step idempotent and independently selectable. """ +import hashlib import json import posixpath import shlex +import tempfile +import uuid +from pathlib import Path +from ducktests_remote import java from ducktests_remote.cli import EXIT_OK, EXIT_PREFLIGHT, EXIT_TRANSPORT -from ducktests_remote.commands import doctor +from ducktests_remote.commands import deploy, doctor from ducktests_remote.config import ConfigError from ducktests_remote.fanout import (CHANGED, FAILED, HostResult, OK, SKIPPED, any_failed, fanout, render_table, summarise) +from ducktests_remote.transport import make_tarball STEPS = ("packages", "jdk", "python", "user", "ssh-env", "dirs", "hosts") HOSTS_BEGIN = "# BEGIN ducktests-remote" HOSTS_END = "# END ducktests-remote" +JAVA_MANIFEST_NAME = ".ducktests-java.json" + def register(subparsers, common): """Wire up the ``provision`` subcommand.""" @@ -54,7 +62,17 @@ def register(subparsers, common): parser.add_argument("--sudo", action="store_true", help="allow steps that need root to use `sudo -n`") parser.add_argument("--install-jdk", action="store_true", - help="let the jdk step install a JDK instead of only verifying one") + help="let the jdk step fall back to the distribution's own JDK " + "package when nothing else resolves; needs --sudo") + parser.add_argument("--java-home", metavar="PATH", + help="use exactly this JDK home on the workers (sets java.home)") + parser.add_argument("--java-major", type=int, default=None, metavar="N", + help="Java major version the tests need (sets java.major)") + parser.add_argument("--java-archive", metavar="PATH", + help="coordinator-side JDK tarball or directory to deliver to " + "hosts that have no matching JDK (sets java.archive)") + parser.add_argument("--force", action="store_true", + help="deliver the JDK again even when the host already has it") parser.add_argument("--create-user", metavar="NAME", help="create this account (step `user`); needs --sudo") parser.add_argument("--authorize-key", metavar="PATH", @@ -82,16 +100,21 @@ def execute(ctx): skipped_for_sudo = [] for step in selected: ctx.console.heading("STEP %s" % step) - script, needs_sudo = _step_script(ctx, step, nodes) - if script is None: - ctx.console.info("nothing to do for this step") - continue - if needs_sudo and not ctx.args.sudo: - ctx.console.warn("step %r needs root; rerun with --sudo. Skipping it and " - "continuing with the rest." % step) - skipped_for_sudo.append(step) - continue - results = _run_step(ctx, nodes, step, script) + if step in _PYTHON_DRIVEN: + # These two resolve a JDK per host before they can act, and `jdk` may have to + # upload one, so they are not expressible as a single canned script. + results = _PYTHON_DRIVEN[step](ctx, nodes) + else: + script, needs_sudo = _step_script(ctx, step, nodes) + if script is None: + ctx.console.info("nothing to do for this step") + continue + if needs_sudo and not ctx.args.sudo: + ctx.console.warn("step %r needs root; rerun with --sudo. Skipping it and " + "continuing with the rest." % step) + skipped_for_sudo.append(step) + continue + results = _run_step(ctx, nodes, step, script) all_results[step] = results ctx.console.out(render_table(results, verbose=ctx.console.verbose)) ctx.console.out(summarise(results)) @@ -158,16 +181,12 @@ def _step_script(ctx, step, nodes): """:return: ``(script, needs_sudo)`` for one step, or ``(None, False)`` when inert.""" if step == "packages": return _packages_script(ctx), True - if step == "jdk": - return _jdk_script(ctx), bool(ctx.args.install_jdk) if step == "python": return _python_script(ctx), False if step == "user": if not ctx.args.create_user: return None, False return _user_script(ctx), True - if step == "ssh-env": - return _ssh_env_script(ctx), False if step == "dirs": return _dirs_script(ctx), True if step == "hosts": @@ -209,42 +228,175 @@ def _packages_script(ctx): """ % {"pkgs": " ".join(shlex.quote(p) for p in packages), "count": len(packages)} -def _jdk_script(ctx): +def _run_jdk_step(ctx, nodes): + """ + Put a JDK of the requested major on every worker, and say which one it is. + + Rungs 1-3 of the ladder are discovery and cost one round trip per host; only the hosts + that come back empty reach rung 4 (deliver ``java.archive``) or rung 5 (fail with the + list of JDKs that *were* found). Nothing is uploaded to a host that already has what + it needs. + """ + cfg = java.config_of(ctx) + plan = java.archive_plan(cfg.archive, cfg.name) if cfg.archive else None + if plan: + target = java.target_dir(cfg, plan) + ctx.console.info("java: %s (%s) available for delivery to %s" + % (plan.path, deploy.human(plan.bytes), target)) + if len(nodes) > 3 and plan.bytes > 200 * 1024 * 1024: + ctx.console.warn("that is up to %s over the wire from this machine, if every " + "host turns out to need it." + % deploy.human(plan.bytes * len(nodes))) + + script = java.discovery_script(cfg) + + def operation(node): + if ctx.dry_run: + ctx.console.out("[dry-run] %s: probe for a Java %s JDK" + % (node.host, cfg.major or "any")) + ctx.console.detail(script) + return HostResult(node.host, SKIPPED, "dry-run") + return _jdk_on_host(ctx, node, cfg, plan, script) + + return fanout(nodes, operation, jobs=ctx.jobs, + fail_fast=getattr(ctx.args, "fail_fast", False)) + + +def _jdk_on_host(ctx, node, cfg, plan, script): + transport = ctx.worker(node) + probe = transport.run_script(script, check=False) + if not probe.ok: + return HostResult(node.host, FAILED, "could not probe for a JDK", + detail=probe.stderr.strip()) + + res = java.parse_probe(node.host, probe.stdout, cfg) + if res.selected: + return HostResult(node.host, OK, res.summary(), detail=probe.stdout.strip()) + + if cfg.home: + # An explicit java.home that is not there is a configuration error, not something + # to paper over by installing a different JDK. + return HostResult(node.host, FAILED, + "java.home %s has no usable bin/java on this host" % cfg.home, + detail=_found(res)) + + if plan is not None: + return _deliver_jdk(ctx, node, cfg, plan) + + if ctx.args.install_jdk: + return _install_jdk(ctx, node, cfg) + + return HostResult(node.host, FAILED, + "no Java %s here; set java.archive to deliver one, java.home to " + "name an existing one, or pass --install-jdk" % cfg.major, + detail=_found(res)) + + +def _found(res): + if not res.candidates: + return "no JDK found at all" + return "found: " + ", ".join("%s (Java %s)" % (home, major) + for home, major, _ in res.candidates) + + +def _deliver_jdk(ctx, node, cfg, plan): """ - Verify the JDK, and only install one when explicitly asked. + Copy the JDK to one worker, reusing ``deploy``'s staging and atomic swap. - Where a JDK comes from is site specific - a distro package, a Temurin tarball, an - internal mirror - so guessing would be worse than reporting. + A half-extracted JDK that looks present is exactly as bad as a half-extracted + distribution, which is why this does not extract in place. """ - major = int(ctx.config["provision"]["jdk_major"]) - install = """ + transport = ctx.worker(node) + target = java.target_dir(cfg, plan) + manifest = deploy.build_manifest(plan.path) if plan.kind == "dir" else _tar_manifest(plan) + + if not ctx.args.force: + existing = transport.read_file(posixpath.join(target, JAVA_MANIFEST_NAME)) + if existing: + try: + if json.loads(existing).get("hash") == manifest["hash"]: + return HostResult(node.host, OK, "%s already delivered" % target) + except ValueError: + pass + + install_root = posixpath.dirname(target) + writable = transport.run(["test", "-w", install_root], check=False).ok + if not writable and not ctx.args.sudo: + return HostResult(node.host, FAILED, + "%s is not writable by %s and --sudo was not passed" + % (install_root, node.user or "this account")) + + staging = "%s/.%s.tmp.%s" % (install_root, posixpath.basename(target), + uuid.uuid4().hex[:8]) + transport.run_script(deploy.prepare_script(staging, ctx.args.sudo)).check() + + with tempfile.TemporaryDirectory() as tmp: + if plan.kind == "dir": + archive = Path(tmp) / "jdk.tar.gz" + make_tarball(plan.path, archive) + strip = 0 + else: + archive = plan.path + strip = plan.strip + remote = "%s/.payload.tar.gz" % staging + transport.upload(archive, remote) + transport.run_script( + "set -eu\ntar -xzf %s -C %s%s\nrm -f -- %s\n" + % (shlex.quote(remote), shlex.quote(staging), + " --strip-components=%d" % strip if strip else "", + shlex.quote(remote))).check() + + check = transport.run(["test", "-x", "%s/bin/java" % staging], check=False) + if not check.ok: + transport.run(["rm", "-rf", "--", staging], check=False) + return HostResult(node.host, FAILED, + "the delivered archive has no bin/java under %s" % staging) + + transport.write_file(json.dumps(manifest, indent=2, sort_keys=True), + posixpath.join(staging, JAVA_MANIFEST_NAME)) + transport.run_script(deploy.swap_script(staging, target, ctx.args.sudo, None)).check() + return HostResult(node.host, CHANGED, "delivered %s to %s" + % (deploy.human(plan.bytes), target)) + + +def _tar_manifest(plan): + """:return: a manifest keyed on the archive itself, so redelivery is skipped.""" + stat = plan.path.stat() + digest = hashlib.sha256( + ("%s\0%d\0%d" % (plan.path.name, stat.st_size, int(stat.st_mtime))).encode("utf-8") + ).hexdigest() + return {"hash": digest, "source": plan.path.name, "bytes": plan.bytes, + "mode": "archive size+mtime"} + + +def _install_jdk(ctx, node, cfg): + """Last rung: the distribution's own JDK package. Opt-in, and it needs root.""" + if not ctx.args.sudo: + return HostResult(node.host, FAILED, + "--install-jdk needs --sudo as well") + script = """set -u +major=%(major)d if command -v apt-get >/dev/null 2>&1; then - echo "CHANGED installing openjdk-%(major)d-jdk" + echo "CHANGED installing openjdk-$major-jdk" sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq - sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq openjdk-%(major)d-jdk + sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "openjdk-$major-jdk" elif command -v dnf >/dev/null 2>&1; then - echo "CHANGED installing java-%(major)d-openjdk-devel" - sudo -n dnf install -y -q java-%(major)d-openjdk-devel + echo "CHANGED installing java-$major-openjdk-devel" + sudo -n dnf install -y -q "java-$major-openjdk-devel" +elif command -v yum >/dev/null 2>&1; then + echo "CHANGED installing java-$major-openjdk-devel" + sudo -n yum install -y -q "java-$major-openjdk-devel" else echo "no supported package manager for an automatic JDK install" >&2 exit 2 fi -""" % {"major": major} if ctx.args.install_jdk else """ -echo "java %(major)d not found; install it yourself or rerun with --only jdk --install-jdk" -exit 1 -""" % {"major": major} - - return """set -u -if command -v java >/dev/null 2>&1; then - v=$(java -version 2>&1 | head -n1) - case "$v" in - *\\"%(major)d*|*\\"1.%(major)d*) echo "ok: $v"; exit 0;; - *) echo "WARN unexpected JDK: $v (expected major %(major)d)"; exit 0;; - esac -fi -%(install)s -java -version 2>&1 | head -n1 -""" % {"major": major, "install": install} +""" % {"major": int(cfg.major or 0)} + result = ctx.worker(node).run_script(script, check=False) + if not result.ok: + return HostResult(node.host, FAILED, "could not install a JDK", + detail=(result.stderr or result.stdout).strip()) + return HostResult(node.host, CHANGED, "installed the distribution's Java %s package" + % cfg.major) def _python_script(ctx): @@ -298,51 +450,68 @@ def _user_script(ctx): """ % {"user": shlex.quote(name), "key": shlex.quote(key)} -def _ssh_env_script(ctx): +def _run_ssh_env_step(ctx, nodes): """ - Put JAVA_HOME and PATH into ``~/.ssh/environment``. - - This is the step that is easiest to forget and hardest to diagnose. ducktape runs - every command over *non-interactive* ssh, where ``~/.profile`` is not sourced, so a - ``java`` that works when you log in by hand is simply absent during a test run. The - Dockerfile solves it with ``PermitUserEnvironment yes`` plus ``~/.ssh/environment``; - the same fix applies here, and the step ends by proving it non-interactively rather - than trusting the edit. + Make the selected JDK the one a *non-interactive* ssh session gets. + + The step that is easiest to forget and hardest to diagnose. ducktape runs every + command over non-interactive ssh, where ``~/.profile`` is not sourced, so a ``java`` + that works when you log in by hand is simply absent during a test run and the failure + surfaces as an unrelated timeout. + + The JDK is resolved with the same ladder the ``jdk`` step uses - not with whatever + ``java`` happens to be first on PATH - so ``provision --only ssh-env`` on its own + still installs the JDK the operator asked for. Both ``~/.ssh/environment`` and + ``~/.bashrc`` are written, and neither is trusted: the step ends by opening a fresh + connection and reporting the JDK that one actually gets. """ + cfg = java.config_of(ctx) + discover = java.discovery_script(cfg) extra = ctx.config["provision"].get("ssh_env_path_extra") or [] - extra_path = "".join(":%s" % p for p in extra) - return """set -u -mkdir -p ~/.ssh -chmod 700 ~/.ssh -jh="${JAVA_HOME:-}" -if [ -z "$jh" ] && command -v java >/dev/null 2>&1; then - jh=$(dirname "$(dirname "$(readlink -f "$(command -v java)")")") -fi -if [ -z "$jh" ]; then echo "cannot determine JAVA_HOME; run the jdk step first" >&2; exit 1; fi -want_path="PATH=$PATH:$jh/bin%(extra)s" -want_home="JAVA_HOME=$jh" -changed=0 -touch ~/.ssh/environment -chmod 600 ~/.ssh/environment -for line in "$want_path" "$want_home" "LANG=C.UTF-8"; do - key=${line%%%%=*} - if grep -q "^$key=" ~/.ssh/environment 2>/dev/null; then - current=$(grep "^$key=" ~/.ssh/environment | head -n1) - [ "$current" = "$line" ] && continue - grep -v "^$key=" ~/.ssh/environment > ~/.ssh/environment.tmp || true - mv ~/.ssh/environment.tmp ~/.ssh/environment - fi - printf '%%s\\n' "$line" >> ~/.ssh/environment - changed=1 -done -chmod 600 ~/.ssh/environment -if [ "$changed" -eq 1 ]; then echo "CHANGED wrote ~/.ssh/environment"; else echo "up to date"; fi -if ! grep -qi '^ *PermitUserEnvironment *yes' /etc/ssh/sshd_config 2>/dev/null; then - echo "NOTE sshd has no 'PermitUserEnvironment yes'; ~/.ssh/environment will be ignored." - echo "NOTE ask your administrator for it, or make sure java is on the default PATH." -fi -echo "verify: $(command -v java || echo 'java not on this shell PATH')" -""" % {"extra": extra_path} + + def operation(node): + if ctx.dry_run: + ctx.console.out("[dry-run] %s: resolve the JDK, then write ~/.ssh/environment%s" + % (node.host, " and ~/.bashrc" if cfg.bashrc else "")) + ctx.console.detail(discover) + return HostResult(node.host, SKIPPED, "dry-run") + + transport = ctx.worker(node) + probe = transport.run_script(discover, check=False) + res = java.parse_probe(node.host, probe.stdout, cfg) + if not res.selected: + return HostResult(node.host, FAILED, + "no Java %s to point at; run `provision --only jdk` first" + % (cfg.major or "JDK"), detail=_found(res)) + + written = transport.run_script(java.env_script(cfg, res.home, extra), check=False) + if not written.ok: + return HostResult(node.host, FAILED, "could not write the environment files", + detail=(written.stderr or written.stdout).strip()) + + return _verify_ssh_env(ctx, node, cfg, res, written) + + return fanout(nodes, operation, jobs=ctx.jobs, + fail_fast=getattr(ctx.args, "fail_fast", False)) + + +def _verify_ssh_env(ctx, node, cfg, res, written): + """Ask a fresh session what it gets. That answer, not the edit, is the result.""" + check = java.parse_probe(node.host, + ctx.worker(node).run_script(java.verify_script(), check=False).stdout, + cfg) + detail = "\n".join(part for part in (written.stdout.strip(), + "verified: %s" % (check.path_java or "no java")) if part) + if not check.path_matches(cfg.major): + return HostResult(node.host, FAILED, + "a fresh non-interactive session still gets %s, not Java %s. " + "sshd may ignore ~/.ssh/environment and the login shell may not " + "be bash; set java.home on a PATH the site already provides." + % (check.path_version or "no java", cfg.major), + detail=detail) + status = CHANGED if "CHANGED" in written.stdout else OK + return HostResult(node.host, status, "non-interactive java is %s from %s" + % (check.path_version or "unknown", res.home), detail=detail) def _dirs_script(ctx): @@ -407,3 +576,7 @@ def _hosts_script(ctx, nodes): def venv_bin(venv, name): """:return: ``/bin/``.""" return posixpath.join(venv, "bin", name) + + +# Steps that need per-host decisions in Python rather than one canned script. +_PYTHON_DRIVEN = {"jdk": _run_jdk_step, "ssh-env": _run_ssh_env_step} diff --git a/modules/ducktests/tests/ducktests_remote/commands/run.py b/modules/ducktests/tests/ducktests_remote/commands/run.py index f7672112c7dc4..8a0f6647ba5a0 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/run.py +++ b/modules/ducktests/tests/ducktests_remote/commands/run.py @@ -25,7 +25,8 @@ import time from pathlib import Path -from ducktests_remote import __version__, cluster as cluster_mod, globals_builder, runs +from ducktests_remote import (__version__, cluster as cluster_mod, globals_builder, pipconf, + runs) from ducktests_remote.cli import (EXIT_OK, EXIT_PREFLIGHT, EXIT_TESTS_FAILED, Console) from ducktests_remote.commands import doctor from ducktests_remote.config import ConfigError, expand_path @@ -75,6 +76,16 @@ def register(subparsers, common): parser.add_argument("--install-sources", action="store_true", help="pip install the synced sources into the runner venv. Not needed " "for test discovery; ducktape puts the sources on sys.path itself") + parser.add_argument("--pip-index-url", metavar="URL", + help="package index for the runner venv, when PyPI is unreachable") + parser.add_argument("--pip-extra-index-url", action="append", default=None, metavar="URL", + help="additional index; repeatable, replaces the configured list") + parser.add_argument("--pip-trusted-host", action="append", default=None, metavar="HOST", + help="host whose certificate pip should not verify; repeatable") + parser.add_argument("--pip-timeout", type=int, default=None, metavar="SECONDS", + help="pip socket timeout, for a slow internal mirror") + parser.add_argument("--pip-cert", metavar="PATH", + help="CA bundle for the index, as a RUNNER-side path") parser.add_argument("--repeat", type=int, default=None, metavar="N") parser.add_argument("--max-parallel", type=int, default=None, metavar="N") parser.add_argument("--test-runner-timeout", type=int, default=None, metavar="MS") @@ -327,8 +338,7 @@ def _ensure_venv(ctx, work_dir): python = ctx.config["runner"].get("python", "python3") requirements = ctx.config["runner"].get("requirements") or posixpath.join( work_dir, "modules", "ducktests", "tests", "docker", "requirements.txt") - index = ctx.config["provision"].get("pip_index_url") - index_arg = "--index-url %s" % shlex.quote(index) if index else "" + index_arg = pipconf.pip_args_str(ctx.config) script = """set -eu venv=%(venv)s @@ -354,9 +364,11 @@ def _ensure_venv(ctx, work_dir): result = ctx.runner.run_script(script, check=False) if not result.ok: raise ConfigError( - "could not prepare the runner venv at %s:\n%s\nEither point runner.venv at an " - "existing environment or make %s reachable on the runner." - % (venv, (result.stderr or result.stdout).strip(), requirements)) + "could not prepare the runner venv at %s:\n%s\nInstalling from: %s.\nEither " + "point runner.venv at an existing environment, make %s reachable on the " + "runner, or set pip.index_url to an index the runner can reach." + % (venv, (result.stderr or result.stdout).strip(), + pipconf.describe(ctx.config, ctx.console.redactor), requirements)) ctx.console.info(result.out.splitlines()[-1] if result.out else "venv ready") @@ -365,7 +377,8 @@ def _install_sources(ctx, work_dir): pip = posixpath.join(venv, "bin", "pip3") if venv else "pip3" tests_dir = posixpath.join(work_dir, "modules", "ducktests", "tests") ctx.console.info("installing sources from %s" % tests_dir) - ctx.runner.run([pip, "install", "--disable-pip-version-check", "-e", tests_dir]).check() + ctx.runner.run([pip, "install", "--disable-pip-version-check"] + + pipconf.pip_args(ctx.config) + ["-e", tests_dir]).check() def _update_latest_link(ctx, paths): diff --git a/modules/ducktests/tests/ducktests_remote/config.py b/modules/ducktests/tests/ducktests_remote/config.py index eeaeef09e1357..40058a6bf7c87 100644 --- a/modules/ducktests/tests/ducktests_remote/config.py +++ b/modules/ducktests/tests/ducktests_remote/config.py @@ -32,6 +32,7 @@ import getpass import json import os +import re from pathlib import Path import yaml @@ -69,6 +70,15 @@ "ignitetest/tests/cellular_affinity_test.py", ) +# Where a JDK is looked for on a worker, in this order of preference within a major +# version. A search path may be a directory *of* JDK homes (/usr/lib/jvm) or a JDK home +# itself; both shapes are probed. +JAVA_SEARCH_PATHS = ["/opt", "/usr/lib/jvm", "/usr/java"] + +# modules/ducktests/tests/docker/Dockerfile pins `ARG jdk_version="eclipse-temurin:17"`. +# The Dockerfile is the source of truth here too; when the two disagree, it wins. +JAVA_MAJOR = 17 + # ignitetest services launch these main classes; see services/ignite.py, # services/ignite_app.py, services/utils/cdc/ignite_cdc.py and .../kafka_to_ignite.py. IGNITE_MAIN_CLASSES = ( @@ -128,12 +138,33 @@ }, "provision": { "packages": list(DOCKERFILE_PACKAGES), - "jdk_major": 17, "install_jdk": False, - "pip_index_url": None, "ssh_env_path_extra": [], "dirs": ["/mnt/service"], }, + # Everything pip needs to reach an index that is not PyPI. Only the runner ever runs + # pip: the venv install and `run --install-sources`. Workers are driven over plain + # ssh and never see Python. + "pip": { + "index_url": None, # --index-url + "extra_index_url": [], # --extra-index-url, repeatable; a bare string is ok + "trusted_host": [], # --trusted-host, repeatable; a bare string is ok + "timeout": None, # --timeout, seconds + "retries": None, # --retries + "cert": None, # --cert; a RUNNER-side path to a CA bundle + }, + # Which JVM the workers run the tests under. See ducktests_remote/java.py for the + # resolution ladder and for why PATH matters more here than JAVA_HOME. + "java": { + "major": JAVA_MAJOR, + "home": None, # explicit JDK home on the workers; set -> no search + "search_paths": list(JAVA_SEARCH_PATHS), + "archive": None, # coordinator-side .tar.gz/.tgz/.tar, or a directory + "install_root": None, # defaults to cluster.install_root + "name": None, # target directory name; defaults to the archive's own + "ssh_environment": True, # write ~/.ssh/environment + "bashrc": True, # write a marked block at the top of ~/.bashrc + }, "ssh": { "connect_timeout": 15, }, @@ -144,11 +175,61 @@ _FREE_FORM_SECTIONS = ("globals", "parameters") +_PLACEHOLDER = re.compile(r"\$\{(env|file):([^}]+)\}") + class ConfigError(Exception): """Raised for anything the operator can fix by editing a file or a flag (exit 1).""" +class _NullRedactor: + """Accepts resolved values and forgets them; used when no redactor was supplied.""" + + def add(self, value): + """Ignore ``value``.""" + + +def interpolate(value, redactor=None, environ=None, source=""): + """ + Resolve ``${env:NAME}`` and ``${file:PATH}`` placeholders throughout a structure. + + A missing variable or file is a hard error naming both the placeholder and the file + it came from. Substituting an empty string instead would produce a run that fails + three hours later with an authentication error nobody can trace back to here. + + Every resolved value is handed to ``redactor`` so that it is masked in everything the + CLI prints afterwards, ``--dry-run`` included. + """ + environ = os.environ if environ is None else environ + redactor = _NullRedactor() if redactor is None else redactor + + if isinstance(value, dict): + return {k: interpolate(v, redactor, environ, source) for k, v in value.items()} + if isinstance(value, list): + return [interpolate(v, redactor, environ, source) for v in value] + if not isinstance(value, str): + return value + + resolved = value + for match in _PLACEHOLDER.finditer(value): + kind, ref = match.group(1), match.group(2).strip() + if kind == "env": + if ref not in environ: + raise ConfigError( + "%s: environment variable %r referenced by ${env:%s} is not set" + % (source, ref, ref)) + replacement = environ[ref] + else: + path = Path(os.path.expanduser(ref)) + if not path.is_file(): + raise ConfigError( + "%s: file %s referenced by ${file:%s} does not exist" % (source, path, ref)) + replacement = path.read_text(encoding="utf-8").strip() + redactor.add(replacement) + resolved = resolved.replace(match.group(0), replacement) + return resolved + + def deep_merge(base, overlay): """ Merge ``overlay`` into a copy of ``base``. @@ -301,7 +382,7 @@ def get_dotted(source, dotted, default=None): def load_config(config_files=(), profiles=(), overrides=None, environ=None, - user_config=USER_CONFIG_FILE): + user_config=USER_CONFIG_FILE, redactor=None): """ Compose the effective configuration from every layer. @@ -310,6 +391,7 @@ def load_config(config_files=(), profiles=(), overrides=None, environ=None, :param overrides: overlay built from explicit command-line flags. :param environ: environment mapping, defaults to ``os.environ``. :param user_config: path to the per-user config file. + :param redactor: collects values resolved from ``${env:}`` / ``${file:}``. :return: the merged, validated configuration. """ config = copy.deepcopy(DEFAULTS) @@ -334,11 +416,31 @@ def load_config(config_files=(), profiles=(), overrides=None, environ=None, if overrides: config = _layer(config, overrides, "command line") + config = _interpolate_config(config, sources, redactor, environ) + config["cluster"]["user"] = config["cluster"]["user"] or _current_user() config["_sources"] = sources return config +def _interpolate_config(config, sources, redactor, environ): + """ + Resolve placeholders in every section except the free-form ones. + + ``globals`` and ``parameters`` are left alone here: :mod:`globals_builder` resolves + them per layer, so its errors can name the profile a placeholder came from, and doing + it twice would re-scan an already resolved secret for placeholders of its own. + """ + source = "config (%s)" % (", ".join(sources) if sources else "built-in defaults") + resolved = {} + for key, value in config.items(): + if key in _FREE_FORM_SECTIONS: + resolved[key] = value + else: + resolved[key] = interpolate(value, redactor, environ, source) + return resolved + + def _layer(config, overlay, source): if not overlay: return config diff --git a/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml b/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml index ec545b8cd4615..7b9e26ba57ec9 100644 --- a/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml +++ b/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml @@ -61,6 +61,30 @@ runner: python: python3 create_venv: true +# Where the runner installs those requirements from. Only the runner ever runs pip; +# workers are driven over plain ssh and need no Python. Leave the whole section out to +# use PyPI. A token in the URL belongs in the environment, not in this file - and +# anything resolved from ${env:} is masked in everything the CLI prints. +pip: + # index_url: ${env:DTR_PIP_INDEX_URL} # a missing variable is a hard error, never "" + # extra_index_url: [https://nexus.example.invalid/repository/pypi-internal/simple] + # trusted_host: [nexus.example.invalid] # plain http, or a certificate you cannot fix + # cert: /etc/pki/tls/certs/corp-ca.pem # a RUNNER-side path, like identity_file + timeout: 60 + retries: 5 + +# Which JVM the workers run the tests under. `major` alone is enough when the VMs already +# carry a suitable JDK somewhere under search_paths; `archive` covers the ones that do not. +java: + major: 17 + # home: /opt/jdk-17.0.11 # use exactly this JDK, no search + search_paths: [/opt, /usr/lib/jvm, /usr/java] + # archive: ~/jdk/OpenJDK17U-jdk_x64_linux_hotspot.tar.gz # coordinator-side + # install_root: /opt # defaults to cluster.install_root + # name: jdk-17 # target directory; defaults to the archive's own name + ssh_environment: true # write ~/.ssh/environment + bashrc: true # and a marked block at the top of ~/.bashrc + deploy: dist_dir: ./dist diff --git a/modules/ducktests/tests/ducktests_remote/globals_builder.py b/modules/ducktests/tests/ducktests_remote/globals_builder.py index 7b70df8d397b7..9d081bd5c4501 100644 --- a/modules/ducktests/tests/ducktests_remote/globals_builder.py +++ b/modules/ducktests/tests/ducktests_remote/globals_builder.py @@ -25,12 +25,13 @@ import json import os -import re from pathlib import Path -from ducktests_remote.config import ConfigError, deep_merge, set_dotted +from ducktests_remote.config import ConfigError, deep_merge, interpolate, set_dotted -_PLACEHOLDER = re.compile(r"\$\{(env|file):([^}]+)\}") +# Re-exported: ``interpolate`` lives in `config` so that the non-globals sections can use +# it too, without `config` having to import this module back. +__all__ = ["Redactor", "build", "dumps", "interpolate", "load_raw_layer", "parse_kv_override"] class Redactor: @@ -84,43 +85,6 @@ def redact_structure(self, data): return data -def interpolate(value, redactor, environ=None, source=""): - """ - Resolve ``${env:NAME}`` and ``${file:PATH}`` placeholders throughout a structure. - - A missing variable or file is a hard error naming both the placeholder and the file - it came from. Substituting an empty string instead would produce a run that fails - three hours later with an authentication error nobody can trace back to here. - """ - environ = os.environ if environ is None else environ - - if isinstance(value, dict): - return {k: interpolate(v, redactor, environ, source) for k, v in value.items()} - if isinstance(value, list): - return [interpolate(v, redactor, environ, source) for v in value] - if not isinstance(value, str): - return value - - resolved = value - for match in _PLACEHOLDER.finditer(value): - kind, ref = match.group(1), match.group(2).strip() - if kind == "env": - if ref not in environ: - raise ConfigError( - "%s: environment variable %r referenced by ${env:%s} is not set" - % (source, ref, ref)) - replacement = environ[ref] - else: - path = Path(os.path.expanduser(ref)) - if not path.is_file(): - raise ConfigError( - "%s: file %s referenced by ${file:%s} does not exist" % (source, path, ref)) - replacement = path.read_text(encoding="utf-8").strip() - redactor.add(replacement) - resolved = resolved.replace(match.group(0), replacement) - return resolved - - def parse_kv_override(item): """ Parse a ``-g a.b.c=value`` argument. diff --git a/modules/ducktests/tests/ducktests_remote/java.py b/modules/ducktests/tests/ducktests_remote/java.py new file mode 100644 index 0000000000000..0aae54b09cb62 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/java.py @@ -0,0 +1,589 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Choosing the JVM the workers run the tests under. + +**Why PATH matters more than JAVA_HOME here.** ``ignitetest`` reaches a JVM four +different ways, and only one of them respects ``JAVA_HOME``: + +=================================================== ========================== +consumer mechanism +=================================================== ========================== +``ignite.sh``, via ``IgniteSpec.envs()`` honours ``JAVA_HOME`` +``jvm_utils.java_version()`` -> ``java -version`` bare ``java``, so PATH +``services/kafka/kafka.py`` -> ``nohup java ...`` bare ``java``, so PATH +``jmx_utils`` -> ``java -jar jmxterm.jar`` bare ``java``, so PATH +=================================================== ========================== + +All four run over *non-interactive* ssh, where ``~/.profile`` is never sourced. Setting +``JAVA_HOME`` alone therefore changes what ``ignite.sh`` uses and nothing else; what the +rest of the suite gets is decided by the non-interactive PATH. Both are written, and the +result is then verified over a fresh connection rather than assumed - see +:func:`env_script` and :func:`verify_script`. + +The resolution ladder, in :func:`discovery_script` (rungs 1-3, pure discovery, safe to run +from ``doctor``) and in ``commands/provision.py`` (rungs 4-5, which mutate): + +1. ``java.home`` is set - verify it, and fail naming the host when it is not there. + Explicit means explicit; falling back would defeat the point of saying it. +2. the JVM the non-interactive shell already provides matches ``java.major`` - use it. +3. a JDK of the right major exists under one of ``java.search_paths`` - use it. +4. ``java.archive`` is set - deliver it (``commands/provision.py``). +5. otherwise fail, listing every JDK that *was* found. +""" + +import posixpath +import shlex +import tarfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Optional + +from ducktests_remote.config import ConfigError, expand_path + +BLOCK_BEGIN = "# BEGIN ducktests-remote" +BLOCK_END = "# END ducktests-remote" + +# Sources, in the order the ladder tries them. +EXPLICIT = "explicit" +CURRENT = "current" +SEARCH = "search" +DELIVERED = "delivered" +NONE = "none" + +_TAR_SUFFIXES = (".tar.gz", ".tgz", ".tar", ".tar.bz2", ".tbz2", ".tar.xz", ".txz") + + +@dataclass +class JavaConfig: + """The ``java`` config section, with the cluster-level fallbacks already applied.""" + + major: Optional[int] = None + home: Optional[str] = None + search_paths: List[str] = field(default_factory=list) + archive: Optional[str] = None + install_root: str = "/opt" + name: Optional[str] = None + ssh_environment: bool = True + bashrc: bool = True + + +@dataclass +class Resolution: + """What one worker reported, and what should be done about it.""" + + host: str = "-" + home: Optional[str] = None # the JDK the ladder selected + version: str = "" + major: Optional[int] = None + source: str = NONE + path_java: Optional[str] = None # what bare `java` resolves to right now + path_real: Optional[str] = None # ... with symlinks resolved + path_version: str = "" + path_major: Optional[int] = None + env_home: Optional[str] = None # $JAVA_HOME as the non-interactive shell sees it + candidates: List[tuple] = field(default_factory=list) # [(home, major, version)] + + @property + def selected(self): + """:return: True when the ladder found a JDK of the requested major.""" + return bool(self.home) + + def path_matches(self, requested_major): + """ + :return: True when the JVM the tests will actually get has the requested major. + + This, not :attr:`selected`, is the question ``doctor`` has to answer: a perfect + JDK sitting in ``/opt`` that the non-interactive PATH does not point at is not the + JVM the suite will run under. + """ + if self.path_major is None: + return False + return requested_major is None or self.path_major == requested_major + + @property + def home_in_effect(self): + """ + :return: True when the JDK on PATH is the one the ladder selected. + + Compared on the resolved path, so ``/usr/bin/java`` symlinked into the selected + home counts as a match. A same-major JDK from somewhere else does not, which is + worth a warning but not a failure - the tests still get the version they asked + for. + """ + return _same_home(self.path_real or self.path_java, self.home) + + def summary(self): + """:return: a short human description of the selected JDK.""" + if not self.selected: + return "no JDK of the requested version" + return "%s (%s, found by %s)" % (self.home, self.version or "unknown version", + self.source) + + +def config_of(ctx): + """:return: the :class:`JavaConfig` for this context, cluster fallbacks applied.""" + section = ctx.config.get("java") or {} + major = section.get("major") + if major is not None: + try: + major = int(major) + except (TypeError, ValueError) as ex: + raise ConfigError("java.major must be a whole number, found %r" % (major,)) from ex + search = section.get("search_paths") or [] + if isinstance(search, str): + search = [search] + return JavaConfig( + major=major, + home=section.get("home") or None, + search_paths=[str(p) for p in search], + archive=section.get("archive") or None, + install_root=(section.get("install_root") + or ctx.cluster_cfg.get("install_root") or "/opt"), + name=section.get("name") or None, + ssh_environment=bool(section.get("ssh_environment", True)), + bashrc=bool(section.get("bashrc", True))) + + +def major_of(version): + """ + :return: the major version of a Java version string, or None. + + Mirrors ``ignitetest.services.utils.jvm_utils.java_major_version``: ``1.8.0_292`` is + 8, ``11.0.19`` is 11, ``17.0.11+9`` is 17. + """ + text = str(version or "").strip().strip('"') + if not text: + return None + parts = text.split(".") + chosen = parts[1] if parts[0] == "1" and len(parts) > 1 else parts[0] + digits = "" + for char in chosen: + if not char.isdigit(): + break + digits += char + return int(digits) if digits else None + + +def discovery_script(cfg: JavaConfig): + """ + :return: a POSIX shell script that reports the JDK situation on one host. + + It only ever *reads*, which is what makes it safe to run from ``doctor`` and from + ``provision --only ssh-env`` as well as from the ``jdk`` step, and it always exits 0: + the status travels in the fields, so one unusable host cannot abort a fan-out. + """ + return _DISCOVERY % { + "requested": shlex.quote(str(cfg.major) if cfg.major else ""), + "explicit": shlex.quote(cfg.home or ""), + "search": " ".join(shlex.quote(p) for p in cfg.search_paths), + } + + +_DISCOVERY = r""" +set -u +say() { printf '%%s=%%s\n' "$1" "$2"; } +requested=%(requested)s +explicit=%(explicit)s + +jv_version() { "$1" -version 2>&1 | head -n1 | sed -n 's/.*version "\([^"]*\)".*/\1/p'; } +jv_major() { + case "$1" in + "") printf '' ;; + 1.*) printf '%%s' "$1" | cut -d. -f2 | tr -cd '0-9' ;; + *) printf '%%s' "$1" | cut -d. -f1 | tr -cd '0-9' ;; + esac +} +home_of() { + p=$(readlink -f "$1" 2>/dev/null || printf '%%s' "$1") + dirname "$(dirname "$p")" +} + +cands="" +add_cand() { + [ -n "$1" ] || return 0 + [ -x "$1/bin/java" ] || return 0 + for seen in $cands; do [ "$seen" = "$1" ] && return 0; done + cands="$cands $1" +} + +say java_requested "$requested" + +# What the tests will actually get: bare `java`, over non-interactive ssh. +cur=$(command -v java 2>/dev/null || true) +say java "$(java -version 2>&1 | head -n1 | tr -d '\r' || echo missing)" +if [ -n "$cur" ]; then + cv=$(jv_version "$cur"); say java_path "$cur"; say java_path_version "$cv" + say java_path_major "$(jv_major "$cv")" + say java_path_real "$(readlink -f "$cur" 2>/dev/null || printf '%%s' "$cur")" + add_cand "$(home_of "$cur")" +fi +say java_env_home "${JAVA_HOME:-}" +add_cand "${JAVA_HOME:-}" +add_cand "$explicit" + +for root in %(search)s; do + add_cand "$root" + for entry in "$root"/*; do add_cand "$entry"; done +done + +report="" +for c in $cands; do + v=$(jv_version "$c/bin/java") + m=$(jv_major "$v") + [ -n "$m" ] || continue + report="$report,$c:$m:$v" +done +say java_candidates "${report#,}" +exit 0 +""" + + +def parse_probe(host, text, cfg: JavaConfig): + """:return: :func:`parse_facts` applied to one host's raw discovery output.""" + return parse_facts(host, _parse_kv(text), cfg) + + +def parse_facts(host, facts, cfg: JavaConfig): + """ + Turn one host's discovery facts into a :class:`Resolution`. + + The selection happens here rather than in the shell so that it is unit-testable and + identical for every caller. ``doctor`` already parses the worker probe into a dict, + which is why this takes facts and :func:`parse_probe` takes text. + """ + res = Resolution(host=host) + res.path_java = facts.get("java_path") or None + res.path_real = facts.get("java_path_real") or None + res.path_version = facts.get("java_path_version") or "" + res.path_major = _int_or_none(facts.get("java_path_major")) + res.env_home = facts.get("java_env_home") or None + res.candidates = _parse_candidates(facts.get("java_candidates")) + + versions = {home: version for home, _, version in res.candidates} + majors = {home: major for home, major, _ in res.candidates} + + if cfg.home: + if majors.get(cfg.home) is None: + return res # explicit home absent or unusable: the caller fails hard + res.home, res.major, res.version = cfg.home, majors[cfg.home], versions[cfg.home] + res.source = EXPLICIT + return res + + wanted = cfg.major + current_home = _home_of(res.path_real or res.path_java) if res.path_java else None + if wanted is None: + # No requested version: whatever the host already provides is the answer. + if current_home and res.path_major is not None: + res.home = current_home + res.major, res.version, res.source = res.path_major, res.path_version, CURRENT + return res + + if res.path_major == wanted and current_home: + res.home = current_home + res.major, res.version, res.source = wanted, res.path_version, CURRENT + return res + + matching = [(home, version) for home, major, version in res.candidates if major == wanted] + if matching: + # Highest patch level wins, compared numerically: sorting the strings would put + # 17.0.9 above 17.0.11. The path breaks ties, so the choice is deterministic. + res.home = max(matching, key=lambda item: (version_key(item[1]), item[0]))[0] + res.major, res.version, res.source = wanted, versions[res.home], SEARCH + return res + + +def version_key(version): + """:return: a numeric tuple for ordering Java version strings (17.0.9 < 17.0.11).""" + numbers = [] + current = "" + for char in str(version or ""): + if char.isdigit(): + current += char + elif current: + numbers.append(int(current)) + current = "" + if current: + numbers.append(int(current)) + return tuple(numbers) + + +def env_script(cfg: JavaConfig, java_home, path_extra=()): + """ + :return: a script writing ``JAVA_HOME``/``PATH`` where non-interactive ssh will see it. + + Two mechanisms, written from the same resolved value in the same step so they cannot + drift apart: + + ``~/.ssh/environment`` + what the Dockerfile does, and silently ignored unless sshd carries + ``PermitUserEnvironment yes``, which no site guarantees. + + ``~/.bashrc`` + a marked block at the very *top* of the file, above the + ``case $- in *i*) ;; *) return;; esac`` guard that the stock Ubuntu file opens + with - that guard exists precisely because bash does source ``~/.bashrc`` for + non-interactive ssh commands. It does nothing when the login shell is not bash. + + Neither is trusted afterwards: :func:`verify_script` re-asks over a fresh connection. + """ + if not (cfg.ssh_environment or cfg.bashrc): + raise ConfigError("java.ssh_environment and java.bashrc are both false, so there " + "is no way to put the JDK on the workers' non-interactive PATH") + fallback = ("echo \"NOTE the ~/.bashrc block below is what will carry JAVA_HOME.\"" + if cfg.bashrc else + "echo \"NOTE java.bashrc is off, so nothing else will carry it: ask for\"\n" + " echo \"NOTE PermitUserEnvironment, or name a JDK already on PATH.\"") + body = _ENV_HEAD % {"home": shlex.quote(str(java_home))} + if cfg.ssh_environment: + body += _ENV_SSH % {"extra": "".join(":%s" % p for p in (path_extra or [])), + "fallback": fallback} + if cfg.bashrc: + body += _ENV_BASHRC % {"begin": BLOCK_BEGIN, "end": BLOCK_END} + return body + _ENV_TAIL + + +_ENV_HEAD = r""" +set -u +jh=%(home)s +[ -x "$jh/bin/java" ] || { echo "no java under $jh" >&2; exit 1; } +changed=0 +""" + +_ENV_SSH = r""" +mkdir -p ~/.ssh +chmod 700 ~/.ssh +touch ~/.ssh/environment +chmod 600 ~/.ssh/environment +# Drop the entries we are about to prepend before reading $PATH back, or a host that +# honours ~/.ssh/environment would grow one more copy of $jh/bin on every run and this +# step would report CHANGED for ever. +base_path=$(printf '%%s' "$PATH" | awk -v RS=: -v strip="$jh/bin%(extra)s" ' + BEGIN { ORS=""; n = split(strip, s, ":") } + { + keep = ($0 != "") + for (i = 1; i <= n; i++) if ($0 == s[i]) keep = 0 + if (keep) { if (out++) printf ":"; printf "%%s", $0 } + }') +want_path="PATH=$jh/bin:$base_path%(extra)s" +want_home="JAVA_HOME=$jh" +for line in "$want_path" "$want_home" "LANG=C.UTF-8"; do + key=${line%%%%=*} + if grep -q "^$key=" ~/.ssh/environment 2>/dev/null; then + current=$(grep "^$key=" ~/.ssh/environment | head -n1) + [ "$current" = "$line" ] && continue + grep -v "^$key=" ~/.ssh/environment > ~/.ssh/environment.tmp || true + mv ~/.ssh/environment.tmp ~/.ssh/environment + fi + printf '%%s\n' "$line" >> ~/.ssh/environment + changed=1 +done +chmod 600 ~/.ssh/environment +[ "$changed" -eq 1 ] && echo "CHANGED wrote ~/.ssh/environment" +if ! grep -qi '^ *PermitUserEnvironment *yes' /etc/ssh/sshd_config 2>/dev/null; then + echo "NOTE sshd has no 'PermitUserEnvironment yes'; ~/.ssh/environment is ignored." + %(fallback)s +fi +""" + +_ENV_BASHRC = r""" +# The block goes at the TOP, above the `case $- in *i*)` early return that the stock +# ~/.bashrc opens with, or it would never run for a non-interactive ssh command. +tmp=$(mktemp) +{ + printf '%%s\n' "%(begin)s" + printf '%%s\n' "export JAVA_HOME=$jh" + printf '%%s\n' 'export PATH="$JAVA_HOME/bin:$PATH"' + printf '%%s\n' "%(end)s" +} > "$tmp" +if [ -f ~/.bashrc ]; then + awk 'BEGIN{skip=0} + /^# BEGIN ducktests-remote$/{skip=1; next} + /^# END ducktests-remote$/{skip=0; next} + skip==0{print}' ~/.bashrc >> "$tmp" +fi +if cmp -s "$tmp" ~/.bashrc 2>/dev/null; then + rm -f "$tmp" +else + mv "$tmp" ~/.bashrc + changed=1 + echo "CHANGED wrote the ducktests-remote block in ~/.bashrc" +fi +""" + +_ENV_TAIL = r""" +if [ "$changed" -eq 0 ]; then echo "up to date"; fi +exit 0 +""" + + +def verify_script(): + """ + :return: a script reporting what a *fresh* non-interactive session actually gets. + + This is the authority. ``~/.ssh/environment`` and ``~/.bashrc`` are both best effort, + and the failure they exist to prevent - ``java`` missing three hours into a run - is + only really excluded by asking the way ducktape will ask. + """ + return r""" +set -u +say() { printf '%s=%s\n' "$1" "$2"; } +jv_version() { "$1" -version 2>&1 | head -n1 | sed -n 's/.*version "\([^"]*\)".*/\1/p'; } +jv_major() { + case "$1" in + "") printf '' ;; + 1.*) printf '%s' "$1" | cut -d. -f2 | tr -cd '0-9' ;; + *) printf '%s' "$1" | cut -d. -f1 | tr -cd '0-9' ;; + esac +} +say java "$(java -version 2>&1 | head -n1 | tr -d '\r' || echo missing)" +say java_env_home "${JAVA_HOME:-}" +cur=$(command -v java 2>/dev/null || true) +if [ -n "$cur" ]; then + cv=$(jv_version "$cur") + say java_path "$cur" + say java_path_version "$cv" + say java_path_major "$(jv_major "$cv")" + say java_path_real "$(readlink -f "$cur" 2>/dev/null || printf '%s' "$cur")" +fi +exit 0 +""" + + +@dataclass +class ArchivePlan: + """A coordinator-side JDK archive or directory, inspected before anything is sent.""" + + path: Path + kind: str # "tar" or "dir" + top_level: Optional[str] # single top-level directory inside a tarball + strip: int # --strip-components for tar + bytes: int + name: str # default target directory name + + +def archive_plan(archive, name=None): + """ + Inspect ``java.archive`` on the coordinator. + + Reading the member list with :mod:`tarfile` rather than guessing in the shell is what + lets a bad archive fail *before* it is copied to every host: a Temurin tarball unpacks + into a single ``jdk-17.0.11+9/`` directory, and an archive with no ``bin/java`` under + it (a macOS build, with ``Contents/Home``) is worth catching here. + """ + path = Path(expand_path(archive)) + if not path.exists(): + raise ConfigError("java.archive %s does not exist on this machine" % path) + + if path.is_dir(): + if not (path / "bin" / "java").exists(): + raise ConfigError("java.archive %s has no bin/java; point it at a JDK home" + % path) + total = sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) + return ArchivePlan(path=path, kind="dir", top_level=None, strip=0, bytes=total, + name=name or path.name) + + lowered = path.name.lower() + if lowered.endswith(".zip"): + raise ConfigError( + "java.archive %s is a zip; ducktests-remote unpacks .tar.gz/.tgz/.tar only. " + "Linux JDKs ship as tarballs, and an untested zip branch on every run is " + "worse than this message." % path) + if not lowered.endswith(_TAR_SUFFIXES): + raise ConfigError("java.archive %s is neither a directory nor a tar archive" + % path) + + try: + with tarfile.open(path) as tar: + members = tar.getmembers() + except (tarfile.TarError, OSError) as ex: + raise ConfigError("java.archive %s could not be read: %s" % (path, ex)) from ex + + names = [m.name.lstrip("./") for m in members if m.name not in (".", "./")] + if not names: + raise ConfigError("java.archive %s is empty" % path) + total = sum(m.size for m in members if m.isfile()) + tops = {n.split("/", 1)[0] for n in names if n} + + if len(tops) == 1: + top = tops.pop() + _require_java(names, "%s/bin/java" % top, path) + return ArchivePlan(path=path, kind="tar", top_level=top, strip=1, bytes=total, + name=name or top) + _require_java(names, "bin/java", path) + return ArchivePlan(path=path, kind="tar", top_level=None, strip=0, bytes=total, + name=name or _strip_suffix(path.name)) + + +def _require_java(names, expected, path): + if expected not in names: + raise ConfigError( + "java.archive %s does not contain %s. A JDK for Linux unpacks with bin/java " + "directly under its top-level directory; a macOS build has Contents/Home in " + "between and cannot be used here." % (path, expected)) + + +def target_dir(cfg: JavaConfig, plan: ArchivePlan): + """:return: where ``plan`` is installed on a worker.""" + return posixpath.join(cfg.install_root, cfg.name or plan.name) + + +def _strip_suffix(filename): + for suffix in _TAR_SUFFIXES: + if filename.lower().endswith(suffix): + return filename[: -len(suffix)] + return filename + + +def _home_of(java_binary): + return posixpath.dirname(posixpath.dirname(str(java_binary))) + + +def _same_home(java_binary, home): + if not java_binary or not home: + return False + return _home_of(java_binary).rstrip("/") == str(home).rstrip("/") + + +def _parse_candidates(raw): + out = [] + for item in (raw or "").split(","): + if not item.strip(): + continue + parts = item.split(":") + if len(parts) < 2: + continue + major = _int_or_none(parts[1]) + if major is None: + continue + out.append((parts[0], major, parts[2] if len(parts) > 2 else "")) + return out + + +def _parse_kv(text): + fields = {} + for line in (text or "").split("\n"): + if "=" in line: + key, value = line.split("=", 1) + fields[key.strip()] = value.strip() + return fields + + +def _int_or_none(value): + try: + return int(str(value).strip()) + except (TypeError, ValueError): + return None diff --git a/modules/ducktests/tests/ducktests_remote/pipconf.py b/modules/ducktests/tests/ducktests_remote/pipconf.py new file mode 100644 index 0000000000000..8947fc271dcd9 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/pipconf.py @@ -0,0 +1,159 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Index configuration for the pip commands the CLI runs on the runner. + +A runner inside a corporate network usually cannot reach PyPI at all, only an internal +mirror, and frequently one behind a private CA. Everything needed for that is expressed +in the ``pip`` config section and turned into *command-line flags* here. + +Flags rather than a generated ``pip.conf`` or ``PIP_*`` environment variables: ``--dry-run`` +printing the literal command it would run is this CLI's contract, and configuration hidden +in a file the operator cannot see breaks it. pip carries its finder settings into PEP 517 +build isolation, so ``pip install -e`` picks the same index up for its build dependencies. + +The module is named ``pipconf`` rather than ``pip`` so that it can never shadow the real +``pip`` package for anything that ends up with this directory on ``sys.path``. +""" + +import os +import re +import shlex + +from ducktests_remote.config import ConfigError + +# Matches the credentials in https://user:token@host/simple, which is how an internal +# index is usually handed out. +_USERINFO = re.compile(r"(?<=://)[^/@\s]+(?=@)") + + +def pip_args(config): + """ + :return: the pip flags implied by the ``pip`` config section, as an argv list. + + Empty configuration yields an empty list, so a command line with nothing configured + stays byte for byte what it was before this section existed. + """ + section = (config or {}).get("pip") or {} + args = [] + + index_url = _text(section.get("index_url"), "pip.index_url") + if index_url: + args += ["--index-url", index_url] + for url in _as_list(section.get("extra_index_url"), "pip.extra_index_url"): + args += ["--extra-index-url", url] + for host in _as_list(section.get("trusted_host"), "pip.trusted_host"): + args += ["--trusted-host", host] + + timeout = _positive_int(section.get("timeout"), "pip.timeout") + if timeout is not None: + args += ["--timeout", str(timeout)] + retries = _positive_int(section.get("retries"), "pip.retries") + if retries is not None: + args += ["--retries", str(retries)] + + cert = _text(section.get("cert"), "pip.cert") + if cert: + # A runner-side path, like cluster.identity_file: it is opened by pip on the + # runner, so a file that exists on this machine proves nothing. + args += ["--cert", os.path.expanduser(cert) if cert.startswith("~") else cert] + + return args + + +def pip_args_str(config): + """:return: :func:`pip_args` shell-quoted for splicing into a generated script.""" + return " ".join(shlex.quote(arg) for arg in pip_args(config)) + + +def describe(config, redactor=None): + """ + :return: a one-line, credential-free summary of where pip will install from. + + The value-keyed :class:`~ducktests_remote.globals_builder.Redactor` only knows about + values it resolved itself, so an index URL typed straight into a config file would + otherwise reach the terminal with its token attached. Masking the userinfo here + covers that case regardless of where the URL came from. + """ + section = (config or {}).get("pip") or {} + parts = [] + index_url = _text(section.get("index_url"), "pip.index_url") + parts.append("index %s" % mask_credentials(index_url) if index_url + else "index default (PyPI)") + extras = _as_list(section.get("extra_index_url"), "pip.extra_index_url") + if extras: + parts.append("extra %s" % ", ".join(mask_credentials(url) for url in extras)) + hosts = _as_list(section.get("trusted_host"), "pip.trusted_host") + if hosts: + parts.append("trusted %s" % ", ".join(hosts)) + if section.get("cert"): + parts.append("cert %s" % section["cert"]) + timeout = _positive_int(section.get("timeout"), "pip.timeout") + if timeout is not None: + parts.append("timeout %ds" % timeout) + retries = _positive_int(section.get("retries"), "pip.retries") + if retries is not None: + parts.append("retries %d" % retries) + line = ", ".join(parts) + return redactor.redact(line) if redactor is not None else line + + +def mask_credentials(url): + """:return: ``url`` with any ``user:password@`` replaced by ``***@``.""" + return _USERINFO.sub("***", str(url or "")) + + +def _text(value, key): + if value is None: + return None + if not isinstance(value, str): + raise ConfigError("%s must be a string, found %s" % (key, type(value).__name__)) + return value.strip() or None + + +def _as_list(value, key): + """A single string is accepted wherever a list is: that is what operators type.""" + if value is None: + return [] + if isinstance(value, str): + values = [value] + elif isinstance(value, (list, tuple)): + values = list(value) + else: + raise ConfigError("%s must be a string or a list, found %s" + % (key, type(value).__name__)) + out = [] + for item in values: + if not isinstance(item, str): + raise ConfigError("%s: every entry must be a string, found %s" + % (key, type(item).__name__)) + if item.strip(): + out.append(item.strip()) + return out + + +def _positive_int(value, key): + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, str)): + raise ConfigError("%s must be a whole number of seconds, found %r" % (key, value)) + try: + number = int(value) + except ValueError as ex: + raise ConfigError("%s must be a whole number, found %r" % (key, value)) from ex + if number <= 0: + raise ConfigError("%s must be greater than zero, found %d" % (key, number)) + return number From cdb5e42ddfeeb1b64e40b110594da5eed4d79574 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 28 Jul 2026 16:30:24 +0300 Subject: [PATCH 3/9] Add a full manual for ducktests-remote under docs/ The README is a tour: install, describe a cluster, get a run going. It cannot carry the class of question that actually comes up once the tool is in use - "when exactly is the JDK copied to the workers", "which layer set this value", "what does this command do, in what order, and what crosses the network". Seven documents under ducktests_remote/docs/, with index.md routing a question to the one that answers it: concepts the three roles, what lives where, the invariants configuration every key, its default and who reads it; layering, ${env:}, redaction commands every command and flag, each with its order of operations and a table of what it transfers, to whom, and when java why PATH decides rather than JAVA_HOME, the resolution ladder, and the exact conditions under which an archive is delivered runs the run directory, run states, detach/follow/stop, exit codes troubleshooting symptom to cause to fix, plus the ssh failure classes internals module map, transport contract, fan-out, and the ignitetest facts this CLI is pinned to Documentation only; no behaviour change. --- .../tests/ducktests_remote/README.md | 7 + .../tests/ducktests_remote/docs/commands.md | 340 ++++++++++++++++++ .../tests/ducktests_remote/docs/concepts.md | 125 +++++++ .../ducktests_remote/docs/configuration.md | 291 +++++++++++++++ .../tests/ducktests_remote/docs/index.md | 82 +++++ .../tests/ducktests_remote/docs/internals.md | 168 +++++++++ .../tests/ducktests_remote/docs/java.md | 216 +++++++++++ .../tests/ducktests_remote/docs/runs.md | 143 ++++++++ .../ducktests_remote/docs/troubleshooting.md | 160 +++++++++ 9 files changed, 1532 insertions(+) create mode 100644 modules/ducktests/tests/ducktests_remote/docs/commands.md create mode 100644 modules/ducktests/tests/ducktests_remote/docs/concepts.md create mode 100644 modules/ducktests/tests/ducktests_remote/docs/configuration.md create mode 100644 modules/ducktests/tests/ducktests_remote/docs/index.md create mode 100644 modules/ducktests/tests/ducktests_remote/docs/internals.md create mode 100644 modules/ducktests/tests/ducktests_remote/docs/java.md create mode 100644 modules/ducktests/tests/ducktests_remote/docs/runs.md create mode 100644 modules/ducktests/tests/ducktests_remote/docs/troubleshooting.md diff --git a/modules/ducktests/tests/ducktests_remote/README.md b/modules/ducktests/tests/ducktests_remote/README.md index 98c94638b0ce7..43070df821733 100644 --- a/modules/ducktests/tests/ducktests_remote/README.md +++ b/modules/ducktests/tests/ducktests_remote/README.md @@ -23,6 +23,13 @@ Run Apache Ignite ducktests against a real VM cluster, from wherever you happen cluster of real machines, driven by a long `ducktape ...` command line that until now only existed inside a Jenkins job. +**This file is the tour. [`docs/`](docs/index.md) is the manual** — every command and +config key, and exactly what happens in what order. Start at +[docs/index.md](docs/index.md), which routes questions to the document that answers them: +[concepts](docs/concepts.md) · [configuration](docs/configuration.md) · +[commands](docs/commands.md) · [java](docs/java.md) · [runs](docs/runs.md) · +[troubleshooting](docs/troubleshooting.md) · [internals](docs/internals.md). + ## The model, in three sentences The **coordinator** is the machine where this CLI runs — your laptop, a VM inside the diff --git a/modules/ducktests/tests/ducktests_remote/docs/commands.md b/modules/ducktests/tests/ducktests_remote/docs/commands.md new file mode 100644 index 0000000000000..5dc941824ed20 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/commands.md @@ -0,0 +1,340 @@ + + +# Command reference + +Ten commands: `run`, `status`, `logs`, `fetch`, `stop`, `provision`, `deploy`, `clean`, +`doctor`, `keys`. + +## Flags every command accepts + +| Flag | Effect | +| --- | --- | +| `--config FILE` | extra config document; repeatable, applied in order | +| `--profile NAME` | named profile; repeatable, applied in order | +| `--runner HOST` | override `cluster.runner`; `local` runs ducktape here | +| `--jobs N` | fan-out parallelism (default 16) | +| `-v, --verbose` | print every command as `+ host$ ...`, and full per-host output | +| `-q, --quiet` | only results and errors | +| `--dry-run` | print what would happen; execute nothing | +| `--no-color` | disable ANSI colour | +| `--fail-fast` | stop scheduling further hosts after the first failure | + +`--version` prints the CLI version. Everything after a bare `--` is passed straight to +ducktape by `run`. + +## What each command transfers + +The question that matters most on a slow link, answered in one table. "Coordinator → X" +means bytes leave your machine. + +| Command | Transfers | To whom | When | +| --- | --- | --- | --- | +| `run` | source tree, `cluster.json`, `globals.json`, `parameters.json`, `run.sh`, `launch.sh`, `meta.json` | **runner only** | every run, unless `--no-sync` (which skips the source tree only) | +| `provision --only jdk` | the `java.archive` tarball | **workers**, and only those with no matching JDK | see [java.md](java.md) | +| `deploy` | one tarball per distribution | **workers**, skipping hosts whose manifest already matches | every invocation, unless skipped or `--dry-run` | +| `keys push` | private key + `.pub` to the runner, public key text to the workers | runner and workers | every invocation | +| `fetch` | *downloads* a results archive | runner → coordinator | every invocation | +| `doctor`, `status`, `logs`, `clean`, `stop` | nothing | — | never | + +Nothing is transferred implicitly by any other command, and `--dry-run` transfers nothing +anywhere. + +--- + +## `run` + +``` +run [TEST_PATH...] [-t PATH]... [-g KEY=VALUE]... [-p KEY=VALUE]... + [--globals-json JSON | --globals-file PATH] [--params-json JSON] + [--cluster-file PATH] [-n N] [--source-root PATH] [--no-sync] + [--exclude PATTERN]... [--work-dir PATH] [--install-sources] + [--pip-index-url URL] [--pip-extra-index-url URL]... [--pip-trusted-host HOST]... + [--pip-timeout SECONDS] [--pip-cert PATH] + [--repeat N] [--max-parallel N] [--test-runner-timeout MS] [--results-root PATH] + [--skip-preflight] [--follow | --detach] [-- EXTRA DUCKTAPE ARGS] +``` + +At least one test path is required, positionally or with `-t`. + +### Order of operations + +1. **Compose `globals`**: `--globals-json`/`--globals-file` (the raw layer), then config + `globals`, then `-g` overrides. `parameters` likewise from `--params-json`, config + `parameters`, `-p`. Placeholders resolve here; a missing one aborts now. +2. **Select nodes** (`-n` takes the first N) and build the cluster payload — generated from + the inventory, or `--cluster-file` read and validated but uploaded byte for byte. +3. **Topology warnings**: the runner also appearing in `cluster.nodes`, and an inventory + below three hosts (most suites declare `@cluster(num_nodes=...)` well above that and + would be skipped as un-runnable). Warnings only. +4. **Preflight**: the full `doctor` check set, unless `--skip-preflight` or `--dry-run`. + Any FAIL stops here with exit 2, before anything is created. +5. **Allocate the run id** — `max-20260727-141233-9f2a` — and derive the run directory, + work directory and results root. +6. **Render `run.sh`** and, with `--dry-run` or `-v`, print it along with `cluster.json` + and a redacted `globals.json`. **`--dry-run` returns here.** +7. **Create** the run and results directories on the runner. +8. **Sync the source tree** to `/src/` unless `--no-sync`. The payload + is measured first and refused above `run.max_payload_mb` (200 MB): that almost always + means a build directory leaked in. rsync when both ends have it, otherwise a tar stream + over scp. +9. **Ensure the venv**: create `/venv` when missing, and install + `docker/requirements.txt` into it when `import ducktape` fails. This is where `pip.*` + applies. Runs after the sync because the requirements file comes from the synced tree. +10. **`--install-sources`** (opt-in): `pip install -e /modules/ducktests/tests`, + with the same `pip.*` flags. +11. **Write the artifacts**: `cluster.json`, `globals.json` (`0600`), `parameters.json` + (`0600`, when non-empty), `run.sh` (`0755`), `launch.sh` (`0755`), `meta.json`, and + update the `runs/latest` symlink. +12. **Launch detached** — `setsid nohup bash launch.sh`, falling back to `nohup` plus + `disown` — and record the pid. +13. **Follow** the log, or print the reattach commands and exit with `--detach`. + +### Following, and Ctrl-C + +The run is detached from second zero, so following is just streaming a file: + +- **Ctrl-C detaches. It does not stop the run.** Losing a three-hour run to a reflex is + not recoverable. +- A **second Ctrl-C within 3 seconds** offers to stop it; in a non-interactive shell it + simply detaches. +- Reattach with `logs -f` from any coordinator. + +### Exit codes + +`0` success · `2` preflight failed · `4` ducktape exited non-zero · `1` configuration +error · `5` transport error · `130` interrupted. See [runs.md](runs.md#exit-codes) for why +4 and 5 are distinct. + +--- + +## `doctor` + +``` +doctor [--json] [-n N] +``` + +Probes the coordinator, the runner and every worker **in parallel**, never stopping at the +first failure, and **changes nothing anywhere**. Also runs implicitly before `run`. + +### What it checks, in order + +1. **Coordinator** — with a remote runner: `ssh` and `scp` on `PATH`. With + `--runner local`: ducktape importable here, and its version against the pin in + `docker/requirements.txt`. +2. **Runner** — `bash`, `python3`, `setsid` present (a missing `setsid` is a WARN: there is + a `nohup` fallback); ducktape importable through the venv python; `identity_file` exists + with sane permissions; free space at `state_root`; the effective pip index. +3. **Workers, reachability** — one connection each from the coordinator, every failure + classified (see [troubleshooting.md](troubleshooting.md#ssh-failure-classes)). +4. **Workers, substance** — one script per reachable host returning: clock skew, free space + at `install_root` and at `/mnt/service`, writability of both, stale JVMs matching + `clean.process_pattern`, passwordless sudo, the distributions present under + `install_root`, and the full JDK probe. +5. **Name resolution** — from the first reachable worker, `getent hosts` for every peer. + Workers that cannot resolve each other fail deep inside discovery with no message that + points here. +6. **Runner → workers** — the connection *ducktape itself* will make, from the runner, with + the runner-side identity. A coordinator that can reach a worker proves nothing about the + runner being able to. + +### The administrator block + +When hosts are unusable, the report ends with a copy-pasteable block naming the hosts, the +account, the key fingerprint being offered, and the exact line to append to +`authorized_keys` — plus, for missing sudo, the precise sudoers line and the two test +suites that need it. Forward it as is. + +### Verdicts + +FAIL stops a run at preflight; WARN never does. `--json` prints every check with its scope, +host, name, status and message, plus an `ok` boolean. Exit `0` when nothing failed, `2` +otherwise. + +--- + +## `provision` + +``` +provision [--only STEP]... [--skip STEP]... [--sudo] [--install-jdk] + [--java-home PATH] [--java-major N] [--java-archive PATH] [--force] + [--create-user NAME] [--authorize-key PATH] [--write-hosts] [-n N] [--json] +``` + +Brings unconfigured VMs up to the state `docker/Dockerfile` guarantees. Steps run in this +order, each idempotent and independently selectable: + +| Step | Does | Needs root | In the default set | +| --- | --- | --- | --- | +| `packages` | installs `provision.packages`; detects apt/dnf/yum, an unknown manager is a clear failure rather than a guess | yes | yes | +| `jdk` | resolves a JDK per host, delivers `java.archive` to hosts that need one — [java.md](java.md) | only with `--install-jdk`, or to write into an unwritable `install_root` | yes | +| `python` | reports whether `python3` exists. **Verifies only**: workers do not need Python, and nothing here will install it | no | yes | +| `user` | `--create-user NAME` plus `--authorize-key` | yes | only with `--create-user` or `--only user` | +| `ssh-env` | points the non-interactive `PATH`/`JAVA_HOME` at the resolved JDK, then verifies over a fresh connection | no | yes | +| `dirs` | creates and chowns `provision.dirs` plus `install_root` | yes | yes | +| `hosts` | rewrites only the block between `# BEGIN ducktests-remote` and `# END ducktests-remote` in `/etc/hosts` | yes | only with `--write-hosts` or `--only hosts` | + +Anything needing root goes through `sudo -n`. If `--sudo` was not given, the step is +reported as skipped and **the remaining steps still run** — a partial provision with an +honest report beats an all-or-nothing failure. Unless `--dry-run`, the command always +finishes by running the `doctor` checks, so it ends with evidence rather than an +assumption; a doctor FAIL makes it exit 2. + +`--dry-run` prints the exact script per host and probes nothing. + +--- + +## `deploy` + +``` +deploy [--dist-dir PATH] [--only NAME]... [--install-root PATH] [--via HOST] + [--sudo] [--owner USER] [--force] [--checksum] [-n N] [--json] +``` + +Each subdirectory of `--dist-dir` is copied verbatim to `/`. The name +is never interpreted or checked against version parsing, which is what makes fork layouts +work without special cases — you name the directories to match what the tests expect. + +Per distribution, per host: + +1. Build a manifest on the coordinator: sorted relative paths plus sizes and mtimes, or + content hashes with `--checksum`. +2. Read `.ducktests-deploy.json` from the target; if its hash matches, **skip this host** + (unless `--force`). +3. Refuse if `install_root` is not writable and `--sudo` was not passed. +4. Extract into a staging directory beside the target, then **swap it into place** and + delete the old tree. A half-copied distribution that looks present is worse than an + absent one. + +`--via HOST` uploads the payload once to an intermediate host and fans out from there. +`deploy` prints the total bytes before it starts — on a twelve-host cluster a 300 MB +distribution is 3.7 GB from a laptop — and suggests `--via` when that total is large. + +### Where the directory names come from + +ignitetest resolves a distribution home as `/`, where `product` is +`str(IgniteVersion(version))`, and `IgniteVersion.__str__` **normalises**: + +| `ignite_versions` entry | directory | +| --- | --- | +| `dev` | `ignite-dev` | +| `2.17.0` | `ignite-2.17.0` | +| `ise-0-32` | `ise-0-32` | +| `ise--6` | `ise-6` — note the collapsed dash | + +A fork can override `product`, so `doctor` reports a missing directory as a WARN listing +what it *did* find rather than failing on a guessed mapping. + +--- + +## `clean` + +``` +clean [-n N] [--keep-paths] +``` + +Kills processes matching `clean.process_pattern` (SIGTERM, five seconds, then SIGKILL for +survivors) and removes `clean.paths`. `--keep-paths` kills without deleting. + +Every path is validated **on the coordinator, before it is sent anywhere**: it must be +absolute, must sit under one of `clean.allowed_roots`, and must not *be* one of those roots. +A bug here would delete distributions across every machine at once, so the rule is +deliberately blunt. + +`--dry-run` prints the exact process list (pid plus command line) and the exact paths with +their sizes, and kills and deletes nothing. Use it first, always. + +--- + +## `keys push` + +``` +keys push [--identity PATH] [--generate] [-n N] +``` + +Installs the private key on the **runner** (mode `0600`, `.pub` alongside at `0644`) and +appends the public key to `authorized_keys` on every worker and extra host. + +Why it exists: ducktape connects from the runner to the workers with the identity named in +the cluster file, and a detached run outlives your SSH session. Agent forwarding dies with +that session, producing a run that authenticates for the first few minutes and then fails +on every subsequent connection. A real key file on the runner is the only arrangement that +survives. + +`--generate` creates an RSA keypair when the identity does not exist. Already-authorised +hosts report `ok` rather than appending a duplicate. + +--- + +## `status` + +``` +status [RUN_ID] [--all] [--json] [-n LINES] +``` + +With no run id, the most recent run on the runner. Prints state, pid, start time, elapsed, +exit code, test paths, cluster and node count, run directory, results path, and the last +15 log lines (`-n`). `--all` prints a table of every run, newest first. `--json` is the +Jenkins-friendly form. + +Because all state lives on the runner, this works from any coordinator, including one that +did not start the run. + +--- + +## `logs` + +``` +logs [RUN_ID] [-f] [-n LINES] +``` + +Prints the last 200 lines (`-n`) of `ducktape.log`, then streams with `-f` until the run +ends. Ctrl-C stops following and never touches the run. Output passes through the redactor, +so a secret that reached the log is masked on the way to your terminal. + +--- + +## `fetch` + +``` +fetch [RUN_ID] [--dest DIR] [--full] +``` + +Archives the results **on the runner**, downloads one file, and extracts it into +`/` (default `./ducktests-results/`), then downloads `ducktape.log` +alongside. By default only ducktape's reports — `report.html`, `report.txt`, `report.json`, +`test_log.info`, `session.log`; `--full` takes the whole results tree. + +`globals.json` is **always** excluded, at both ends. Extraction refuses any member that +would escape the destination directory. + +--- + +## `stop` + +``` +stop [RUN_ID] [--kill] [--timeout SECONDS] [--no-clean] [-n N] +``` + +1. Touches `stopped` in the run directory, so the final state is reported as `stopped` + rather than `failed`. +2. SIGTERMs the run's **process group**, waits up to `--timeout` (default 60s), and with + `--kill` SIGKILLs whatever survives. Without `--kill` it says so and leaves it. +3. Writes exit code `143` if the process never wrote one itself. +4. Runs `clean` across the workers, unless `--no-clean`. + +Stopping a run that already ended is safe: it says so, and still cleans. diff --git a/modules/ducktests/tests/ducktests_remote/docs/concepts.md b/modules/ducktests/tests/ducktests_remote/docs/concepts.md new file mode 100644 index 0000000000000..2e3b72bfde9cb --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/concepts.md @@ -0,0 +1,125 @@ + + +# Concepts + +## The three roles + +| Role | What it is | What runs there | +| --- | --- | --- | +| **coordinator** | wherever you typed `ducktests-remote` — laptop, cluster VM, Jenkins agent | the CLI itself, and only the CLI | +| **runner** | the host the `ducktape` process lives on: `cluster.runner`, or `local` | ducktape, the Python venv, all run state | +| **workers** | `cluster.nodes` — the machines ducktape drives over SSH | Ignite JVMs, the Java applications, Kafka/ZooKeeper when a test needs them | + +The coordinator can be any of the three at once. `cluster.runner: local` makes the +coordinator the runner; a worker listed in `extra_hosts` can be the runner too. The roles +are about *function*, not about hardware, and every command is written so that the local +and remote cases take the same code path — see [internals.md](internals.md#transport). + +A worker never needs Python and never runs any of this code. ducktape drives it over plain +SSH, which is why `provision --only python` verifies rather than installs, and why the JDK +is the only runtime the workers must actually have. + +## What lives where + +**On the coordinator:** your config files (`~/.ducktests-remote/config.yaml`, profiles), +your source tree, and whatever `deploy --dist-dir` / `java.archive` point at. Nothing else. +Closing the laptop loses nothing. + +**On the runner, under `cluster.state_root`** (default `~/.ducktests-remote`): + +``` +venv/ the Python environment ducktape runs in +src// the synced source tree for one run +runs// one run directory; see runs.md +runs/latest symlink to the newest run +``` + +**On the workers:** `cluster.install_root` (default `/opt`) holds the Ignite +distributions and, when delivered, the JDK. `/mnt/service` (ignitetest's +`persistent_root`) holds everything a test writes. `~/.ssh/environment` and `~/.bashrc` +carry `JAVA_HOME`/`PATH`. + +## Why all run state lives on the runner + +Because a run outlives the thing that started it. A three-hour suite launched from a +laptop must survive the laptop being closed, and a Jenkins job that starts a run must be +followable from a workstation afterwards. + +So: `run` detaches the ducktape process from second zero, writes everything about the run +into a directory on the runner, and every other command (`status`, `logs`, `fetch`, `stop`) +reads that directory. Any coordinator with SSH access to the runner can inspect, follow or +stop a run that a different coordinator started. There is no local state to get out of sync +and nothing to clean up on the coordinator. + +## Invariants + +These hold everywhere in the codebase. Each has a unit check, and breaking one is a bug +even when the result looks correct. + +**The CLI never imports ducktape.** It renders artifacts and drives the ducktape process on +the runner. That keeps it installable on a coordinator with no ducktape at all, and lets +the two versions move independently. `checks/check_remote_transport.py` runs a subprocess +to prove the import boundary holds. + +**Everything remote goes through a `Transport`.** No module above `transport.py` shells out +to `ssh` or `scp`. `--runner local` and `--runner build-vm-01` therefore exercise identical +code paths, and the unit checks can substitute a recording fake. + +**`--dry-run` executes nothing.** Not even a read-only probe. Commands that need facts from +a host to decide report `not probed (--dry-run)` rather than guessing, and commands that +would transfer print the size instead. + +**`doctor` never mutates.** It is safe on a cluster someone else is using. The JDK +discovery script it runs is checked to contain no `mkdir`, `rm`, `mv`, `chmod` or +redirection (`checks/check_remote_java.py`). + +**A fan-out never stops at the first failure** unless `--fail-fast` says so. When the +outcome of a command is a request to whoever administers the machines, a partial list of +broken hosts is worse than useless. + +**Secrets are redacted by value, not by key name.** Anything resolved from `${env:}` or +`${file:}` is registered with a `Redactor` and replaced with `***` in everything the CLI +prints — including `--dry-run` output, streamed logs and error messages. A password that +leaks into an unrelated field is still caught. See +[configuration.md § Secrets](configuration.md#secrets-and-redaction). + +## The two flows this replaces + +**`docker/run_tests.sh`** — the local Docker flow. Every node is a container, `ducker01` is +the runner, and the image guarantees the node state. That flow is unchanged and remains the +right one for developing tests. + +**A Jenkins one-liner** — a very long `ducktape --globals '' ...` +command that only existed inside a job definition. `ducktests-remote` takes that apart: +the inventory becomes `cluster.nodes`, the JSON blob becomes a profile, and the secrets +become `${env:}` placeholders resolved at launch and written to a `0600` file on the +runner instead of appearing on a command line in build logs. `--globals-json` and +`--cluster-file` exist so the blob can be moved over verbatim first and split up +afterwards; see [commands.md § run](commands.md#run). + +## What this tool deliberately does not do + +- **It is not configuration management.** `provision` covers the specific package / + directory / ssh-environment set the Dockerfile installs, and must not grow into Ansible. +- **It does not manage OS repositories.** `pip.*` configures pip; which apt/dnf mirrors a + VM talks to is set when the image is built. +- **It does not download a JDK from the internet.** It moves one you already have. +- **It takes no cluster lease.** One runner, one operator; queueing belongs to whatever + schedules the runs. Exit code 3 is reserved should that ever change. +- **It never removes a host key.** A changed host key is reported with the exact + `ssh-keygen -R` line for you to run after you have verified it. diff --git a/modules/ducktests/tests/ducktests_remote/docs/configuration.md b/modules/ducktests/tests/ducktests_remote/docs/configuration.md new file mode 100644 index 0000000000000..903b2d6d9c97b --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/configuration.md @@ -0,0 +1,291 @@ + + +# Configuration reference + +Every default in this document comes from `DEFAULTS` in `config.py`. When the two +disagree, `config.py` is right and this file is a bug. + +## Layering + +Later layers win. Dicts deep-merge; **every other type, lists included, replaces +outright** — a later layer must be able to *shrink* a list, which is exactly what an +override is for. + +1. built-in defaults (`config.py`) +2. `~/.ducktests-remote/config.yaml` +3. `--config FILE`, repeatable, applied in order +4. `--profile NAME`, repeatable, applied in order +5. `DTR_*` environment variables +6. explicit command-line flags +7. `-g KEY=VALUE` / `-p KEY=VALUE` — `globals` and `parameters` only, applied by + `globals_builder` + +After merging, `${env:}` / `${file:}` placeholders are resolved everywhere except +`globals` and `parameters`, which `globals_builder` resolves per layer so its errors can +name the profile a placeholder came from. + +**Unknown keys are a hard error**, with a "did you mean" suggestion from `difflib`: + +``` +ERROR profile.yaml: unknown config key 'cluster.instal_root'; did you mean 'install_root'? +``` + +A typo silently ignored in a config that drives a three-hour run is expensive. The two +free-form sections, `globals` and `parameters`, are passed through unvalidated by design — +they belong to ducktape and to the tests, not to this CLI. + +Files are parsed as **YAML or JSON by content, not by extension**: a document starting with +`{` or `[` is parsed as JSON, anything else as YAML. + +### Profiles + +`--profile NAME` searches, in order: + +1. `/NAME.yaml`, `.yml`, `.json`, or `NAME` exactly +2. the same four under the current directory +3. `examples/profile-NAME.*` and `examples/NAME.*` shipped in the package + +A profile is a normal config document; it is not restricted to `globals`. + +### Environment variables + +- `DTR_CLUSTER__RUNNER=build-vm-01` → `cluster.runner`. A **double** underscore separates + path segments, so single underscores stay usable inside key names + (`DTR_RUN__MAX_PAYLOAD_MB` → `run.max_payload_mb`). +- Short aliases: `DTR_RUNNER` → `cluster.runner`, `DTR_JOBS` → `jobs`, `DTR_CLUSTER` → + `cluster.name`, `DTR_STATE_ROOT` → `cluster.state_root`, `DTR_INSTALL_ROOT` → + `cluster.install_root`, `DTR_USER` → `cluster.user`. +- `DTR_CONFIG` and `DTR_PROFILE` are ignored here (they select files, not values). +- **Any other single-segment `DTR_*` variable is ignored.** Profiles interpolate secrets + with `${env:DTR_SOMETHING}`, and treating every such variable as a config path would turn + a password into a config error. The `__` form is still validated, so a typo there is + caught. +- Values are parsed as JSON when they parse: `DTR_JOBS=8` is an integer, `DTR_RUNNER=vm` is + a string. + +## Sections + +### `cluster` + +| Key | Default | Meaning | +| --- | --- | --- | +| `name` | `default` | label only; appears in `meta.json` and `status` | +| `user` | your `$USER` | SSH account on the workers. Never defaults to `ducker` — that account exists only in the Docker image | +| `identity_file` | `~/.ssh/id_rsa` | private key **as the runner sees it**; this is the path written into `cluster.json` and opened by ducktape | +| `port` | `22` | default SSH port, per-host overridable | +| `install_root` | `/opt` | where distributions live on the workers; `/` is what ignitetest resolves a version to | +| `runner` | `local` | host running ducktape, or `local` | +| `state_root` | `~/.ducktests-remote` | runner-side root for the venv, sources and run directories | +| `nodes` | `[]` | the inventory; see below | +| `extra_hosts` | `[]` | hosts that `provision`/`deploy`/`clean`/`doctor` should also target but that ducktape must **not** schedule Ignite nodes onto — typically the runner itself | + +`identity_file` is the single most misread key. It is a **runner-side** path. If the runner +is not the coordinator, a file that exists on your laptop proves nothing; `doctor` checks +it on the runner and `keys push` installs it there. + +#### The inventory + +An entry is either a bare string or a mapping with `host` and optional `ip`, `user`, +`port`, `identity_file`: + +```yaml +nodes: + - host: worker[01-12].dc.local # expands to twelve hosts + - host: worker13.dc.local + ip: 10.0.0.13 # -> externally_routable_ip in cluster.json + - host: worker14.dc.local + user: someone-else # per-host override, for mixed clusters + - 10.0.0.15 # bare string is shorthand for {host: ...} +``` + +- **Ranges**: `[01-12]` zero-pads to the width of the lower bound, so `worker01`…`worker12`; + `[1-12]` gives `worker1`…`worker12`. Several ranges in one pattern expand as a cartesian + product, left to right. `ip` cannot be combined with a range. +- Duplicate hosts are an error. An unknown key inside an entry is an error. +- `-n/--num-nodes N` takes the **first N** entries, the analogue of + `IGNITE_NUM_CONTAINERS` in `docker/run_tests.sh`. Asking for more than the inventory + holds is an error naming both numbers. + +The generated `cluster.json` is exactly what `ducktape.cluster.json.JsonCluster` reads: +a `nodes` list of `{externally_routable_ip, ssh_config}`, where `ssh_config` is passed +into `RemoteAccountSSHConfig(host, hostname, user, port, password, identityfile)`. +`--cluster-file` bypasses generation entirely and uploads your file byte for byte. + +### `runner` + +| Key | Default | Meaning | +| --- | --- | --- | +| `venv` | `null` | explicit venv path on the runner. Unset means `/venv` | +| `python` | `python3` | interpreter used to create the venv and to probe for ducktape | +| `create_venv` | `true` | when false and `venv` is unset, no venv is used and ducktape is expected on `PATH` | +| `requirements` | `null` | requirements file; unset means `/modules/ducktests/tests/docker/requirements.txt`, so the ducktape pin has one source of truth | + +### `pip` + +Read only on the runner, by `run`: the venv install and `--install-sources`. Workers never +run pip. Full treatment in [commands.md § run](commands.md#run). + +| Key | Default | pip flag | +| --- | --- | --- | +| `index_url` | `null` | `--index-url` | +| `extra_index_url` | `[]` | `--extra-index-url`, repeated; a bare string is accepted | +| `trusted_host` | `[]` | `--trusted-host`, repeated; a bare string is accepted | +| `timeout` | `null` | `--timeout`, seconds | +| `retries` | `null` | `--retries` | +| `cert` | `null` | `--cert`; a **runner-side** path to a CA bundle | + +With nothing set, no flags are added at all and the rendered commands are byte for byte +what they were before this section existed. + +### `java` + +Read by `provision` (`jdk`, `ssh-env`) and by `doctor`. Full treatment in +[java.md](java.md). + +| Key | Default | Meaning | +| --- | --- | --- | +| `major` | `17` | Java major version the tests need. Derived from the Dockerfile's `ARG jdk_version="eclipse-temurin:17"`; the Dockerfile wins if they ever disagree | +| `home` | `null` | an explicit JDK home on the workers. Set means *exactly this*: no search, and a host without it fails | +| `search_paths` | `[/opt, /usr/lib/jvm, /usr/java]` | where to look for an existing JDK. Each entry may be a directory *of* JDK homes or a JDK home itself | +| `archive` | `null` | coordinator-side `.tar.gz`/`.tgz`/`.tar` or an unpacked directory, delivered to hosts that have no matching JDK | +| `install_root` | `null` | where a delivered JDK is unpacked; unset means `cluster.install_root` | +| `name` | `null` | target directory name; unset means the archive's own top-level directory name | +| `ssh_environment` | `true` | write `~/.ssh/environment` | +| `bashrc` | `true` | write a marked block at the top of `~/.bashrc` | + +Setting both `ssh_environment` and `bashrc` to false is an error: there would be no way to +put the JDK on the workers' non-interactive `PATH`. + +### `run` + +| Key | Default | Meaning | +| --- | --- | --- | +| `source_root` | `null` | directory synced to the runner; unset means the current directory | +| `work_dir` | `null` | runner-side working directory for ducktape; unset means the synced source directory | +| `exclude` | `[]` | extra sync exclusions, appended to the built-in list | +| `max_payload_mb` | `200` | refuse to sync more than this. A build directory leaking into the payload is the usual cause | +| `install_sources` | `false` | `pip install -e` the synced sources. Not needed for discovery: ducktape's loader walks up from each test file while `__init__.py` exists and puts the resulting top-level directory on `sys.path` | + +Built-in sync exclusions: `.git`, `target`, `results`, `__pycache__`, `*.pyc`, `.idea`, +`venv`, `.venv`, `*.egg-info`, `.tox`, `.pytest_cache`. A `.ducktestsignore` file at the +source root replaces the built-in list; `--exclude` replaces both. + +### `deploy` + +| Key | Default | Meaning | +| --- | --- | --- | +| `dist_dir` | `./dist` | one subdirectory per distribution | +| `install_root` | `null` | target root; unset means `cluster.install_root` | +| `sudo` | `false` | use `sudo -n` for the remote side | +| `owner` | `null` | `chown -R` the extracted tree | +| `staging_dir` | `/tmp/ducktests-remote-staging` | where `--via` parks the payload | +| `checksum` | `false` | hash file contents for the manifest instead of size+mtime | + +### `provision` + +| Key | Default | Meaning | +| --- | --- | --- | +| `packages` | see below | system utilities, derived from `docker/Dockerfile` | +| `install_jdk` | `false` | allow the distribution's JDK package as the last rung | +| `ssh_env_path_extra` | `[]` | extra `PATH` entries appended in `~/.ssh/environment` | +| `dirs` | `[/mnt/service]` | directories to create and chown; `install_root` is always added | + +Default packages: `sudo`, `netcat-traditional`, `iptables`, `rsync`, `unzip`, `wget`, +`curl`, `jq`, `coreutils`, `net-tools`. The Dockerfile is the source of truth; image-only +entries (`openssh-server`, `vim`, `mc`, build toolchain) are deliberately not replicated — +a real VM already has an sshd, and no compiler is needed to *run* tests. + +### `clean` + +| Key | Default | Meaning | +| --- | --- | --- | +| `process_pattern` | `org.apache.ignite` | `pgrep -f` pattern | +| `paths` | `[/mnt/service]` | directories to remove | +| `allowed_roots` | `[/mnt, /tmp, /var/tmp]` | every path in `paths` must sit under one of these | + +The default pattern covers all four main classes ignitetest launches: +`CommandLineStartup`, `CdcCommandLineStartup`, `IgniteAwareApplicationService`, +`KafkaToIgniteCommandLineStartup`. + +### `ssh`, `jobs`, `profiles_dir` + +| Key | Default | Meaning | +| --- | --- | --- | +| `ssh.connect_timeout` | `15` | seconds, passed as `ConnectTimeout` to every connection | +| `jobs` | `16` | fan-out parallelism | +| `profiles_dir` | `~/.ducktests-remote/profiles` | where `--profile` looks first | + +### `globals` and `parameters` + +Free-form. `globals` becomes ducktape's `--globals` payload; `parameters` becomes +`--parameters`. Neither is validated by this CLI, both are deep-merged like everything +else, and both are written to `0600` files on the runner rather than onto a command line. + +`-g` / `-p` take dotted overrides whose values are parsed as JSON when they parse, so +`-g ssl.enabled=true` is a boolean and `-g project=ise` is a string. This mirrors +`_extend_json` in `docker/run_tests.sh`, with nesting added. + +## Placeholders + +`${env:NAME}` and `${file:PATH}` are resolved **on the coordinator, at launch**, in every +section: + +```yaml +cluster: + user: ${env:DTR_SSH_USER} +pip: + index_url: ${env:NEXUS_URL} +globals: + authentication: + password: ${file:~/.secrets/ise-password} +``` + +A missing variable or an unreadable file is a hard error naming both the placeholder and +where it came from — never an empty string, never a run that fails on authentication three +hours later. `${file:}` strips surrounding whitespace, which is what you want for a file +holding one token. + +## Secrets and redaction + +- Every value resolved from `${env:}` or `${file:}` is registered with a `Redactor` and + replaced by `***` in everything the CLI prints: normal output, `--dry-run`, streamed + logs, warnings, error messages, and the config summary inside `meta.json`. +- Redaction is keyed on the **value**, so the same secret is caught in an unrelated field + or in a rendered command line. Key-name matching (`password`, `passwd`, `secret`, + `token`, `keystore_pass`, `truststore_pass`) is only a fallback for values the CLI never + resolved itself. +- Values shorter than three characters are not registered — masking `a` everywhere would + destroy the output. +- `pip` index URLs get a second, independent treatment: any `user:password@` in a URL is + masked wherever the CLI prints it, even when the URL was typed straight into a config + file and the redactor has never seen it. +- The composed `globals.json` is written `0600` on the runner, and `fetch` always excludes + it. +- The example profiles in `examples/` contain no real hostnames, addresses, accounts or + passwords, and must stay that way: this directory is in a public Apache repository. + +## Seeing the result + +```bash +ducktests-remote run --dry-run -t ./ignitetest/tests/smoke_test.py +``` + +prints the composed `cluster.json`, the redacted `globals.json`, and the rendered `run.sh` +without creating, uploading or starting anything. `doctor` additionally prints the +effective pip index and the resolved identity path. When two configurations disagree, diff +their `--dry-run` output rather than reasoning about the layering. diff --git a/modules/ducktests/tests/ducktests_remote/docs/index.md b/modules/ducktests/tests/ducktests_remote/docs/index.md new file mode 100644 index 0000000000000..4012608f1c607 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/index.md @@ -0,0 +1,82 @@ + + +# ducktests-remote manual + +`../README.md` is the tour: install it, describe a cluster, get a run going. This is the +reference: every command, every configuration key, and — the part a README cannot carry — +**exactly what happens in what order, and what crosses the network when**. + +The organising rule of these documents: when the answer is "it depends", the condition is +written down. If you find yourself guessing, that is a documentation bug worth fixing. + +## The documents + +| Document | What it answers | +| --- | --- | +| [concepts.md](concepts.md) | Which machine is which, what lives where, and the invariants the whole tool rests on | +| [configuration.md](configuration.md) | Every config key, its default, who reads it; layering, `${env:}`, secrets | +| [commands.md](commands.md) | Every command and flag, and the exact order of operations inside each | +| [java.md](java.md) | How the workers' JVM is chosen, and precisely when a JDK is copied to them | +| [runs.md](runs.md) | The run directory, run states, detach/follow/stop, exit codes | +| [troubleshooting.md](troubleshooting.md) | Symptom → cause → fix, for the failures this cluster actually produces | +| [internals.md](internals.md) | Module map, the transport contract, fan-out, redaction; where to change things | + +## Find an answer fast + +**"When does X get sent to the workers?"** +Nothing is ever sent implicitly. Three commands transfer files, and nothing else does: +`deploy` (distributions), `provision --only jdk` (a JDK, and only to hosts that lack one), +`keys push` (the public key). `run` uploads only to the *runner*. See +[commands.md § What each command transfers](commands.md#what-each-command-transfers). + +**"Why did my test get the wrong Java?"** → [java.md](java.md), and in particular +[§ Why PATH decides, not JAVA_HOME](java.md#why-path-decides-not-java_home). + +**"Where does this setting come from?"** → [configuration.md § Layering](configuration.md#layering) +then `--dry-run`, which prints the composed result of every layer. + +**"What does this command actually do?"** → [commands.md](commands.md). Each entry lists +its steps in execution order, what it reads, what it writes, and what `--dry-run` skips. + +**"The run ended — where is everything?"** → [runs.md § The run directory](runs.md#the-run-directory). + +**"Is it safe to run twice?"** Yes, for every command; the per-command entry says how +idempotence is achieved. `clean` and `stop` are the two that destroy things, and both have +a `--dry-run` that prints the exact kill list and path list. + +**"What does this exit code mean?"** → [runs.md § Exit codes](runs.md#exit-codes). + +## Two habits worth having + +**`--dry-run` first.** It is genuinely side-effect free on every command: no probes, no +uploads, no processes. It prints the commands it would run and the files it would generate, +including the rendered `run.sh`, `cluster.json` and a redacted `globals.json`. Most +questions about "what would this do" are answered faster by running it than by reading. + +**`-v` when a message is not enough.** Verbose prints every command as it is issued +(`+ host$ ...`) and the full per-host output rather than only the failures. + +## Conventions in these documents + +- **coordinator**, **runner**, **worker** are used precisely; see + [concepts.md](concepts.md#the-three-roles). A sentence that does not name one of them is + about all three. +- Paths like `services/utils/path.py` are relative to `modules/ducktests/tests/ignitetest/`; + paths like `commands/run.py` are relative to `modules/ducktests/tests/ducktests_remote/`. +- Code references name a file and a function rather than a line number, so they survive + edits: "`_deliver_jdk` in `commands/provision.py`". diff --git a/modules/ducktests/tests/ducktests_remote/docs/internals.md b/modules/ducktests/tests/ducktests_remote/docs/internals.md new file mode 100644 index 0000000000000..5e30e95050bd5 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/internals.md @@ -0,0 +1,168 @@ + + +# Internals + +For changing the CLI rather than using it. + +## Module map + +| Module | Responsibility | +| --- | --- | +| `cli.py` | argument parsing, the shared `Context`, `Console`, exit-code mapping | +| `config.py` | defaults, layering, validation, `${env:}`/`${file:}` interpolation | +| `globals_builder.py` | the `--globals` payload and the `Redactor` | +| `cluster.py` | inventory → `Node` list → ducktape's `cluster.json` | +| `transport.py` | the only place that shells out to `ssh`/`scp` | +| `fanout.py` | bounded parallel per-host execution and the result table | +| `runs.py` | run ids, run directory layout, state derivation, script rendering | +| `sshdiag.py` | SSH failure classification and the administrator block | +| `java.py` | JDK discovery, environment files, archive inspection | +| `pipconf.py` | pip index configuration → command-line flags | +| `commands/*.py` | one module per subcommand, each with `register()` and `execute(ctx)` | +| `templates/run.sh.tmpl` | the generated ducktape command | +| `checks/` | unit checks; no network, no Docker, no ducktape | + +`pipconf` is not called `pip` so that it can never shadow the real `pip` package for +anything that ends up with this directory on `sys.path`. + +## Context + +`Context` (in `cli.py`) is what every command receives. It holds the composed config, the +parsed args and the console, and lazily builds transports: + +- `ctx.nodes` — inventory, truncated by `-n` +- `ctx.all_nodes` — inventory plus `extra_hosts`; used by `provision`, `deploy`, `clean`, + `doctor`, `keys` +- `ctx.runner` — a transport to the runner, created once +- `ctx.worker(node)` — a transport per worker, cached by host +- `ctx.state_root_resolved()` — the state root with `~` expanded against the *runner's* home + +A subtlety worth knowing: `cluster.identity_file` is a runner-side path, so the coordinator +uses it only when a file of that name happens to exist locally; otherwise the system ssh +client falls back to `~/.ssh/config` and the agent. + +## Transport + +Three implementations behind one interface: + +| Class | Used for | +| --- | --- | +| `LocalTransport` | `--runner local`, and anything else on this machine | +| `SshTransport` | a remote runner or a worker, through the **system** `ssh`/`scp` | +| `ProxiedTransport` | a second hop: `deploy --via`, and probing a worker over the connection ducktape itself will make | + +The interface is `run(argv)`, `run_script(script)`, `upload`, `download`, `upload_dir`, +`exists`, `mkdirs`, `write_file`, `read_file`, `home`, `expand`. + +Design decisions that are load-bearing: + +- **`argv` is always a list, never a shell string.** Quoting is done once, in the transport. +- **`run_script` feeds a script to `bash -s` over stdin.** Long quoted one-liners are the + single largest source of remote-execution bugs; anything longer than a couple of words + belongs in a script. +- **The system ssh client, not paramiko.** It brings `~/.ssh/config`, `ProxyJump`, agent and + Kerberos support for free, and it is the same client an engineer uses by hand when + reproducing a failure. +- **Connection multiplexing** (`ControlMaster`) is enabled everywhere except Windows, where + OpenSSH has no support for it and the options fail hard. `doctor` and `deploy` open many + connections per host; multiplexing makes every extra one a free channel. +- **`write_file` writes bytes with explicit LF endings.** A coordinator on Windows would + otherwise translate every newline to CRLF, and a shell script with carriage returns fails + on the runner with a message naming the wrong line. +- **`expand`** resolves a leading `~` against the *remote* home, once per connection, because + paths are shell-quoted before they reach the remote side and a literal tilde would never + be expanded there. + +## Fan-out + +`fanout(hosts, operation, jobs=…, fail_fast=…)` runs `operation(host)` in a bounded thread +pool and returns results **in inventory order**. An exception inside one operation becomes +that host's `FAILED` result rather than killing the batch — per-host isolation is the whole +point. Statuses: `ok`, `changed`, `skipped`, `warn`, `failed`. + +`render_table` prints failures' detail by default and everything's detail with `-v`; +`summarise` prints `9 ok, 2 failed`, ordered worst-first. + +## Redaction + +`Redactor` keys on resolved **values**, not on key names. Anything coming out of `${env:}` +or `${file:}` is registered at config load, and `Console` runs every line through it, so +redaction cannot be bypassed by printing from an unusual place. `redact_structure` also +masks a small set of sensitive key names as a fallback for values the CLI never resolved +itself. Independently, `pipconf.mask_credentials` masks `user:password@` in URLs. + +If you add a new output path, print through `Console` — not `print`. + +## Adding a command + +1. Create `commands/.py` with `register(subparsers, common)` and `execute(ctx)`. +2. Add it to the import and the loop in `cli.build_parser`. The order there is the order in + `--help`. +3. Return one of the `EXIT_*` constants from `cli.py`. +4. Support `--dry-run` honestly: print what would happen, execute nothing, probe nothing. +5. Route every remote action through `ctx.runner` / `ctx.worker(node)`, and every host loop + through `fanout`. +6. Add checks under `checks/check_remote_.py` using `FakeTransport`. + +## Conventions + +- ASF licence header on every file, Python and Markdown alike. +- flake8, max line length **120**; config in `tox.ini`. +- Docstrings in the `:param:` / `:return:` style used throughout. +- Comments explain *why*, not *what*. The reason a decision was made is the part that + cannot be recovered from the code later. +- No new runtime dependency. The package uses the standard library plus `PyYAML`, and + **never imports ducktape**. + +## Tests + +```bash +cd modules/ducktests/tests +pytest ducktests_remote/checks # unit only: no network, no Docker, no ducktape +flake8 ducktests_remote +``` + +`[pytest]` in `tox.ini` collects `check_*.py` files, `Check` classes and `check_*` +functions, which is why the files are named that way. `checks/fake_transport.py` provides a +recording transport that simulates a small filesystem and returns canned output for +commands matching a needle. The only subprocess in the suite is the deliberate one that +proves the ducktape import boundary still holds. + +Checks are named as sentences — `check_lists_replace_and_do_not_concatenate` — because the +name is the specification and shows up in the failure output. + +## Where the ignitetest facts come from + +Several behaviours are pinned to things in `ignitetest` rather than invented here. When +those move, these move: + +| Fact | Source | +| --- | --- | +| distribution home is `/` | `services/utils/path.py`, `services/utils/ignite_aware.py` | +| version strings normalise (`ise--6` → `ise-6`) | `utils/version.py` | +| `persistent_root` defaults to `/mnt/service` | `services/utils/path.py` | +| `sudo iptables` is used by exactly two suites | `IgniteAwareService.drop_network` | +| the four Ignite main classes `clean` must match | `services/ignite.py`, `services/ignite_app.py`, `services/utils/cdc/*` | +| Java major parsing | `services/utils/jvm_utils.py` | +| which Java consumers use `PATH` vs `JAVA_HOME` | `services/utils/ignite_spec.py`, `jvm_utils.py`, `kafka/kafka.py`, `jmx_utils.py` | +| the package list and the JDK version | `docker/Dockerfile` | +| the ducktape pin | `docker/requirements.txt` | + +`docker/Dockerfile` is the source of truth for what a prepared node looks like. Where it +and this CLI disagree, the Dockerfile wins — and the derived lists in `config.py` carry a +comment saying so, to keep the drift visible at the next review. diff --git a/modules/ducktests/tests/ducktests_remote/docs/java.md b/modules/ducktests/tests/ducktests_remote/docs/java.md new file mode 100644 index 0000000000000..3bb6dacd4ba8e --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/java.md @@ -0,0 +1,216 @@ + + +# Choosing the workers' JVM + +## Why PATH decides, not JAVA_HOME + +`ignitetest` reaches a JVM four different ways, and only one of them respects `JAVA_HOME`: + +| Consumer | Mechanism | +| --- | --- | +| `ignite.sh`, via `IgniteSpec.envs()` → `export …;` | honours `JAVA_HOME` | +| `jvm_utils.java_version()` → `java -version` over ssh | bare `java`, so **PATH** | +| `services/kafka/kafka.py` → `nohup java …` | bare `java`, so **PATH** | +| `jmx_utils` → `java -jar jmxterm.jar` | bare `java`, so **PATH** | + +All four run over **non-interactive** SSH, where `~/.profile` is never sourced. So setting +`JAVA_HOME` alone changes what `ignite.sh` uses and nothing else, and a `java` that works +perfectly when you log in by hand can be absent or wrong during a test run. Both are set, +and — because neither mechanism is guaranteed to work at a given site — what a fresh +non-interactive session actually gets is then *measured*. + +This is also why `doctor` judges the JVM on `PATH` rather than the JDK it can find: a +perfect Java 17 sitting in `/opt` that the non-interactive `PATH` does not point at is not +the JVM the suite will run under. + +The runner needs no JVM at all. ducktape is pure Python. Java is purely a worker concern. + +## The resolution ladder + +Run by `provision --only jdk`, per host. Rungs 1–3 are pure discovery — one round trip, +no writes — which is why `doctor` and `provision --only ssh-env` can run the identical +script. + +**1. `java.home` is set.** Verify `$home/bin/java`. If it is missing or unusable on a host, +that host **fails, by name**. There is no fallback: an explicit `java.home` that silently +resolves to a different JVM would defeat the point of saying it. This is checked before +anything else, so an explicit home is never overridden by a lucky match elsewhere. + +**2. The JVM already on the non-interactive PATH matches `java.major`.** Use it as is, +report `ok`. A correctly prepared VM never gets touched. + +**3. A JDK under `java.search_paths` matches.** Each search path is probed both as a +directory *of* JDK homes (`/usr/lib/jvm/*/bin/java`) and as a JDK home itself +(`/opt/bin/java`), so both layouts work. Every candidate's `java -version` is parsed the +same way `jvm_utils.java_major_version` does: `1.8.0_292` → 8, `11.0.19` → 11, +`17.0.11+9` → 17. Among candidates of the requested major, the **highest patch level** +wins, compared numerically — string sorting would put `17.0.9` above `17.0.11` — and the +path breaks ties, so the choice is deterministic. + +**4. `java.archive` is delivered.** Only now, and only to the hosts that reached this rung. +Details below. + +**5. Nothing worked.** The host fails with the list of every JDK that *was* found and the +config keys that would fix it. With `--install-jdk` **and** `--sudo`, the distribution's +own `openjdk-N-jdk` / `java-N-openjdk-devel` package is attempted first as a last rung. + +## When the archive is actually sent + +The single question this section exists to answer. + +**Only `provision`, step `jdk`, ever sends a JDK.** Not `doctor` (it never mutates +anything), not `ssh-env` (it only points `PATH` at a JDK that is already there), not `run`, +not `deploy`. + +```bash +ducktests-remote provision --only jdk # just this step +ducktests-remote provision --sudo # all steps; jdk runs before ssh-env +``` + +Within that step: + +| Phase | Where | What happens | +| --- | --- | --- | +| 1 | coordinator, once | `java.archive` is opened with `tarfile`, checked for `bin/java` under its single top-level directory, and its size printed. **A bad archive fails here**, before a byte moves | +| 2 | each host, in parallel | the discovery script runs rungs 1–3 | +| 3 | each host that found nothing | delivery, below | + +A host reaches delivery only when **all** of these hold: + +- rungs 1–3 found no JDK of `java.major` on it, **and** +- `java.home` is unset — if it is set and missing, that is a failure, never a delivery, **and** +- `java.archive` is set — otherwise the host fails with the "set `java.archive` or + `java.home`" message. + +So on a twelve-host cluster where eleven already carry Java 17, exactly one upload happens. + +### What delivery does on the host + +1. Read `.ducktests-java.json` in the target directory. If its hash matches the archive, + **skip** — nothing is uploaded. `--force` overrides. +2. Check `install_root` is writable; if not and `--sudo` was not passed, fail naming the + directory and the account. +3. Create a staging directory beside the target, upload the archive into it, and unpack — + `--strip-components=1` when the tarball has a single top-level directory, which is what + a stock Temurin tarball has. +4. Verify `bin/java` exists in the staging tree; if not, remove the staging tree and fail. +5. Write the manifest, then **swap** the staging tree into place and delete the old one. + +The staging-and-swap is `deploy`'s, reused rather than reimplemented: a half-extracted JDK +that looks present is exactly as bad as a half-extracted distribution. + +### Archive formats + +| Given | Result | +| --- | --- | +| `.tar.gz` / `.tgz` / `.tar` with one top-level dir | stripped; target defaults to that dir's name, e.g. `/opt/jdk-17.0.11+9` | +| the same, flat (`bin/java` at the root) | not stripped; target defaults to the file name without its suffix | +| an unpacked directory containing `bin/java` | tarred on the fly and sent | +| no `bin/java` where expected | **refused on the coordinator** — a macOS build with `Contents/Home` is the realistic case | +| `.zip` | **refused**, naming `.tar.gz`. Linux JDKs ship as tarballs, and a zip branch would be untested code on every real run | + +`java.name` overrides the target directory name; `java.install_root` overrides where it +goes (defaulting to `cluster.install_root`). + +### Cost + +The archive is uploaded from the coordinator to each host that needs it. `provision` prints +the archive size up front and, above 200 MB with more than three hosts, warns with the +worst-case total. `provision` has no `--via`; if that total is painful, a good pattern is +to deliver to one machine, or to place the JDK in your `deploy` dist directory once and +point `java.home` at the result. + +## Making the choice stick + +The `ssh-env` step writes **both** files, from one resolved value, in one step, so they +cannot drift apart: + +**`~/.ssh/environment`** — what the Dockerfile does. Contains `JAVA_HOME`, a `PATH` with +`$JAVA_HOME/bin` **first** (plus `provision.ssh_env_path_extra`), and `LANG=C.UTF-8`. +Silently ignored unless sshd carries `PermitUserEnvironment yes`; the step says so when it +is absent. The existing `PATH` is filtered of the entries about to be prepended, so running +the step repeatedly neither grows the variable nor reports a change for ever. + +**`~/.bashrc`** — a block between `# BEGIN ducktests-remote` and `# END ducktests-remote` +at the **top** of the file, above the `case $- in *i*) ;; *) return;; esac` guard the stock +Ubuntu file opens with. That guard exists precisely because bash *does* source `~/.bashrc` +for non-interactive ssh commands, so appending would be writing to `/dev/null`. Only the +block between the markers is ever rewritten. This does nothing when the account's login +shell is not bash. + +Either can be switched off with `java.ssh_environment: false` / `java.bashrc: false`; +switching off both is an error. + +**Then it verifies.** A fresh non-interactive connection runs `java -version` and reads +`$JAVA_HOME`. That result is the outcome of the step: if the session still gets the wrong +JVM, the step fails there — with the advice that sshd may be ignoring `~/.ssh/environment` +*and* the login shell may not be bash — rather than reporting success and letting a test +fail three hours later. + +## What `doctor` says + +| Situation | Verdict | Message | +| --- | --- | --- | +| no `java` on the non-interactive PATH | **FAIL** | run `provision --only jdk --only ssh-env` | +| PATH java's major ≠ `java.major`, but a matching JDK is on the host | **FAIL** | names the JDK it found and the command that would point PATH at it | +| PATH java's major ≠ `java.major`, nothing matching installed | **FAIL** | lists what was found; names `java.archive` and `java.home` | +| major matches, but an explicit `java.home` is not the JDK in effect | **WARN** | the tests will run on the right version; the pin is simply not active | +| major matches | **OK** | version and resolved path | +| hosts disagree on version | **WARN** on a `java-consistency` row | majority version and the outliers | + +A FAIL blocks `run` at preflight with exit 2. `--skip-preflight` remains the escape hatch, +and `java.major: null` disables the version requirement entirely (whatever is on the host +is then accepted). + +## Worked scenarios + +**VMs already have the right JDK.** `java.major: 17`, nothing else. Rung 2 matches +everywhere, `ssh-env` makes sure non-interactive sessions see it, no transfers at all. + +**A JDK sits in `/opt/jdk-17.0.11` but `java` is 11.** Rung 3 finds it. +`provision --only jdk --only ssh-env` selects it and points `PATH` at it. No transfers. + +**Fresh VMs, JDK pre-downloaded on your machine.** + +```yaml +java: + major: 17 + archive: ~/jdk/OpenJDK17U-jdk_x64_linux_hotspot.tar.gz +``` + +```bash +ducktests-remote provision --dry-run --only jdk # validates the archive, prints the size +ducktests-remote provision --only jdk --only ssh-env +``` + +Rung 4 delivers to every host, unpacks to `/opt/jdk-17.0.11+9`, and `ssh-env` points at it. +Re-running transfers nothing: the manifest matches. + +**A vendor JDK you must use exactly.** `java.home: /opt/corp-jdk-17`. Rungs 2–4 are skipped +entirely; a host without that directory fails by name. + +**Mixed cluster, some hosts short.** Set both `java.major` and `java.archive`. Hosts that +already have a match are untouched; the rest are delivered to. One command, one honest +per-host table. + +## Configuration recap + +See [configuration.md § java](configuration.md#java) for the full table. The short version: +`major` is what the tests need, `home` is an exact pin, `search_paths` is where to look, +`archive` is what to fall back on, and `ssh_environment`/`bashrc` control how the answer is +made to stick. diff --git a/modules/ducktests/tests/ducktests_remote/docs/runs.md b/modules/ducktests/tests/ducktests_remote/docs/runs.md new file mode 100644 index 0000000000000..13611e5bab583 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/runs.md @@ -0,0 +1,143 @@ + + +# Runs + +## The run directory + +Everything about a run lives on the runner, under +`/runs//`: + +| File | Written by | Contents | +| --- | --- | --- | +| `meta.json` | `run`, at launch | run id, CLI version, coordinator user/host/platform, runner, cluster name, node list, test paths, work dir, results root, start time, and a redacted config summary | +| `cluster.json` | `run` | exactly what ducktape was given | +| `globals.json` | `run`, mode `0600` | the composed globals, secrets resolved | +| `parameters.json` | `run`, mode `0600` | present only when `parameters` is non-empty | +| `run.sh` | `run`, mode `0755` | the exact ducktape command. SSH in and execute it to reproduce the run by hand | +| `launch.sh` | `run`, mode `0755` | wrapper that waits on `run.sh` and records the exit code | +| `pid`, `pgid` | `launch.sh` / the detach script | what `stop` signals | +| `exit_code` | `launch.sh`, last | written after ducktape exits | +| `stopped` | `stop` | marker that distinguishes "stopped" from "failed" | +| `ducktape.log` | `launch.sh` | combined stdout and stderr | +| `results/` | ducktape | `--results-root`, with ducktape's own `latest` symlink inside | + +Alongside: `/runs/latest` points at the newest run, and +`/src//` holds that run's synced sources. + +Run ids look like `max-20260727-141233-9f2a`: the account and timestamp make a directory +listing readable, and four hex characters keep two runs started in the same second apart. + +## Reproducing a run by hand + +```bash +ssh build-vm-01 +cd ~/.ducktests-remote/runs/max-20260727-141233-9f2a +cat run.sh # every path is shell-quoted; nothing is hidden +bash run.sh +``` + +`run.sh` activates the venv and execs ducktape with `--results-root`, `--cluster-file`, +`--globals` and the test paths. `--globals` takes a *file path*: ducktape 0.13 checks +`os.path.isfile` before parsing the argument as JSON, so the composed blob never crosses a +command line and never lands in shell history or a process listing. + +## Run states + +`status` derives the state from three observable facts — is the pid alive, is there an +`exit_code`, is there a `stopped` marker: + +| State | Meaning | +| --- | --- | +| `running` | pid alive, no exit code yet | +| `finished` | exit code 0 | +| `failed` | non-zero exit code, no `stopped` marker | +| `stopped` | exit code present *and* a `stopped` marker, or the marker with no live pid | +| `unknown` | no pid, no exit code, no marker — usually a run that died before `launch.sh` got going | + +`exit_code` wins over liveness because the file is written last: a process that has already +exited is never reported as running just because its pid got reused. + +## Detach, follow, reattach + +`run` detaches from second zero — `setsid nohup bash launch.sh`, with `nohup` plus `disown` +where `setsid` is missing — and then attaches to the log. `setsid` matters: it puts +ducktape in its own session so a dropped SSH connection cannot SIGHUP it and leave Ignite +JVMs alive on every worker. + +Consequences: + +- **Ctrl-C detaches; it does not stop the run.** A second Ctrl-C within 3 seconds offers to + stop it (and simply detaches in a non-interactive shell). +- `--detach` skips following entirely and prints the reattach commands. +- `logs -f` reattaches from **any** coordinator, including one that did not start + the run. +- Following is just streaming a file by byte offset, so it costs the runner nothing and can + be interrupted freely. + +## Stopping + +`stop` touches `stopped`, SIGTERMs the process **group**, waits `--timeout` seconds +(default 60), and with `--kill` SIGKILLs survivors. If the process never wrote an exit code, +`143` is recorded. Then it cleans the workers, unless `--no-clean`. + +The follow-up clean is the important half: killing ducktape does not kill the Ignite JVMs +it started on twelve machines, and those are what break the *next* run. See +[commands.md § clean](commands.md#clean). + +## Exit codes + +| Code | Meaning | +| --- | --- | +| `0` | success | +| `1` | usage or configuration error — a bad key, a missing file, an unset `${env:}` | +| `2` | preflight failed; `run` stopped before creating anything | +| `3` | reserved: this deployment has a single runner and takes no cluster lease | +| `4` | ducktape ran and reported test failures | +| `5` | transport or infrastructure error | +| `130` | interrupted by the operator | + +`4` and `5` are deliberately distinct: Jenkins needs "tests failed" separate from "the +cluster is broken". Note that ducktape itself exits `1` both for test failures and for its +own startup errors, so a `4` strictly means "ducktape ran and exited non-zero" — the log +distinguishes the two. + +## Getting results back + +```bash +ducktests-remote fetch # newest run, reports only +ducktests-remote fetch --full # the whole results tree +``` + +Results land in `./ducktests-results//`, with `ducktape.log` beside them. +`globals.json` is always excluded. + +The results also stay on the runner indefinitely — nothing is pruned automatically. On a +long-lived runner, `/runs` and `/src` grow one directory per run; +`status --all` lists them, and removing old ones is an ordinary `rm -rf` on the runner. + +## Jenkins + +```bash +ducktests-remote --profile ise-perf run -t "$TC_PATHS" --detach +ducktests-remote status --json > status.json +``` + +`--detach` returns immediately with the run id; `status --json` is machine-readable; the +exit code separates test failures from infrastructure failures. Because run state lives on +the runner, a job that times out and is retried can pick the same run back up rather than +starting a second one on top of it. diff --git a/modules/ducktests/tests/ducktests_remote/docs/troubleshooting.md b/modules/ducktests/tests/ducktests_remote/docs/troubleshooting.md new file mode 100644 index 0000000000000..fb96b0d9f3bb4 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/troubleshooting.md @@ -0,0 +1,160 @@ + + +# Troubleshooting + +Start with `ducktests-remote doctor`. It probes everything in parallel, never stops at the +first failure, and changes nothing, so it is always safe to run — including on a cluster +someone else is using. + +## The classic failures + +### Stale Ignite JVMs + +The single most common source of baffling failures on a shared cluster: a previous run was +killed and its JVMs still hold ports and `/mnt/service`. Tests fail in ways that have +nothing to do with the change you are testing. + +`doctor` reports it as a FAIL with the host list. + +```bash +ducktests-remote clean --dry-run # prints every pid and command line it would kill +ducktests-remote clean +``` + +### `java: command not found`, or the wrong JDK, deep inside a test + +Non-interactive SSH does not source `~/.profile`, so the `java` you get when you log in by +hand is not necessarily the one the tests get. + +```bash +ducktests-remote provision --only jdk --only ssh-env +``` + +If it still reports the wrong JVM afterwards, sshd is ignoring `~/.ssh/environment` **and** +the login shell is not bash. Set `java.home` to a JDK the site already puts on the default +`PATH`. Full detail in [java.md](java.md). + +### `identity_file` does not exist on the runner + +`cluster.identity_file` is the path *ducktape* will open, **on the runner**. A file that +exists on your laptop proves nothing. `doctor` checks it there and reports its mode; +`keys push` installs it. + +### Agent forwarding does not survive a detached run + +`ssh -A` gives you an agent for the lifetime of your session; a run that lasts hours +outlives it, and every subsequent worker connection then fails — typically after the first +few minutes, which makes it look like an intermittent cluster problem. Use a real key file +on the runner (`keys push`). + +### Discovery failures with no useful message + +Workers that cannot resolve each other's hostnames fail inside Ignite discovery, far from +the cause. `doctor` runs an N-way resolution probe from one worker. + +```bash +ducktests-remote provision --sudo --write-hosts # escape hatch when cluster DNS cannot be fixed +``` + +### "source payload is N MB, above the limit" + +A build directory leaked into the sync. Distributions go through `deploy`, never through +the source sync. Adjust `--exclude`, add a `.ducktestsignore` at the source root, or raise +`run.max_payload_mb` if the payload really is that big. + +### The venv cannot be prepared + +The message names the index it tried. On a network without PyPI access, set `pip.index_url` +(see [configuration.md § pip](configuration.md#pip)) or point `runner.venv` at an +environment that already has ducktape. `doctor` prints the effective index on the runner +row, with credentials masked. + +### Tests are skipped as un-runnable + +Most `ignitetest` suites declare `@cluster(num_nodes=...)` above three. An inventory smaller +than the largest declaration means those tests are never scheduled. `run` warns when the +inventory is below three hosts; ducktape's own report lists what it skipped and why. + +### No distribution for a version + +`doctor` reports a WARN listing what it *did* find under `install_root`. Remember that +ignitetest normalises version strings: `ise--6` maps to `/opt/ise-6`. See +[commands.md § deploy](commands.md#where-the-directory-names-come-from). + +## SSH failure classes + +Every failed connection is classified and mapped to a concrete next action, because the +first-time experience is usually not a subtle bug but "nobody has added me to these +machines yet". + +| Class | Meaning | Next action | +| --- | --- | --- | +| `unresolved` | the hostname does not resolve from here | check VPN/DNS, or put the address in `cluster.nodes[].ip` | +| `no-sshd` | the host answers but nothing listens on the port | sshd is down or on another port | +| `unreachable` | no network path | firewall, routing, or the host is down | +| `no-access` | your key is not authorised for that account | `keys push`, or the administrator block | +| `no-user` | the account does not exist there | ask for it, or use a per-host `user` override | +| `hostkey` | the host key changed | **verify first**, then run the printed `ssh-keygen -R` yourself; this tool will never remove a host key for you | +| `no-sudo` | passwordless sudo is missing | only the two network-segmentation suites need it | +| `unknown` | unrecognised | rerun with `-v` for the full stderr | + +When any host is unusable, the report ends with a **"what to ask your administrator"** +block: the hosts, the account, the fingerprint of the key being offered, the exact line to +append to `authorized_keys`, and — for `no-sudo` — the precise sudoers line plus the two +suites that need it. It is meant to be forwarded verbatim. + +The classification patterns come from OpenSSH's own message strings and have not been +replayed against every distribution's build. Adding one is a one-line change in +`sshdiag.py`; `checks/check_remote_sshdiag.py` is table-driven over recorded samples. + +## Privileges the tests actually need + +Grepped from the `ignitetest` sources, not assumed. Useful when an administrator offers you +a privileged account you do not need: + +- **An ordinary unprivileged account** for everything except the two suites below. It needs + write access to `persistent_root` (default `/mnt/service`) and read access to + `install_root` (default `/opt`). +- **Passwordless `sudo` for `iptables` only**, and only for + `ignitetest/tests/discovery_test.py` and `ignitetest/tests/cellular_affinity_test.py`, + which reach `sudo iptables`, `iptables-save` and `iptables-restore` through + `IgniteAwareService.drop_network`. +- **Write access to `install_root`** only if you use `deploy` without `--sudo`. + +Nothing else in `ignitetest` needs root. + +## When the message is not enough + +```bash +ducktests-remote -v doctor # every command as it is issued, full per-host output +ducktests-remote --dry-run # the exact scripts and files, executed nowhere +ducktests-remote status --json # machine-readable run state +``` + +On the runner, the run directory holds the whole truth: `run.sh` is the exact command, +`ducktape.log` is the combined output, and `meta.json` records the configuration the run +was launched with. See [runs.md](runs.md). + +If a failure looks like a bug in this CLI rather than in the cluster, the unit checks run +in under a second and touch no network: + +```bash +cd modules/ducktests/tests +pytest ducktests_remote/checks +flake8 ducktests_remote +``` From 3aeba7fe6cd7213ab1567b875db56ca085ac3d46 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 28 Jul 2026 18:03:54 +0300 Subject: [PATCH 4/9] Run ducktests-remote from the tests directory, and locate the JDK home Test paths now mean what they mean in the Docker flow. ducktape is started in /modules/ducktests/tests, the working directory docker/run_tests.sh uses, so a run reads ducktests-remote run ./ignitetest/tests/smoke_test.py::Cls.test instead of repeating modules/ducktests/tests in every path. Paths are accepted in whatever form resolves on the coordinator - relative to the current directory, to the tests directory, or to the checkout root - and are rewritten from there, so the previous form keeps working. A path inside the checkout but outside the tests directory becomes a runner-side absolute path, which is the only form that cannot be misread once the working directory has moved. The checkout itself is found by walking up from the current directory, and a source root that is not one is refused before anything is uploaded. Standing in the wrong place used to sync that directory to the runner and surface minutes later as MISSING_REQUIREMENTS, blaming the pip index for a file that was never there; that exit is now reported separately from a pip failure. java.archive no longer assumes how deeply the JDK is wrapped. bin/java is located in the member list and everything above it stripped, so a hand-repacked tarball (openjdk-17/jdk-17.0.11+9/bin/java) and one with stray entries beside the JDK are both accepted, where before only depth 0 and depth 1 were. The shallowest bin/java wins, so a bundled JRE cannot drag the strip depth with it. A macOS build is still refused, now by checking the resolved prefix rather than by guessing in the error text. Two defects found while reading that code: * .tar.xz and .tar.bz2 passed the coordinator-side check and then failed on every worker, because the extract was a hardcoded `tar -xzf`. The flag now follows the suffix. * An unpacked java.archive directory was packed inside the per-host operation, so twelve workers meant twelve concurrent gzips of the same JDK on the coordinator. JdkPayload packs it once, caches the manifest with it, and cleans up after the fan-out. --- .../tests/ducktests_remote/README.md | 19 +- .../checks/check_remote_java.py | 67 +++++-- .../checks/check_remote_runs.py | 8 +- .../checks/check_remote_sources.py | 176 ++++++++++++++++++ .../ducktests_remote/commands/provision.py | 98 +++++++--- .../tests/ducktests_remote/commands/run.py | 152 +++++++++++++-- .../tests/ducktests_remote/docs/commands.md | 25 ++- .../ducktests_remote/docs/configuration.md | 8 +- .../tests/ducktests_remote/docs/java.md | 24 ++- .../examples/profile-smoke.yaml | 2 +- .../ducktests/tests/ducktests_remote/java.py | 87 +++++++-- .../ducktests/tests/ducktests_remote/runs.py | 8 +- .../ducktests_remote/templates/run.sh.tmpl | 2 +- 13 files changed, 579 insertions(+), 97 deletions(-) create mode 100644 modules/ducktests/tests/ducktests_remote/checks/check_remote_sources.py diff --git a/modules/ducktests/tests/ducktests_remote/README.md b/modules/ducktests/tests/ducktests_remote/README.md index 43070df821733..784d8cf26e8da 100644 --- a/modules/ducktests/tests/ducktests_remote/README.md +++ b/modules/ducktests/tests/ducktests_remote/README.md @@ -127,9 +127,13 @@ line to append to `authorized_keys`. ### 3. Run ```bash -ducktests-remote run ./modules/ducktests/tests/ignitetest/tests/smoke_test.py +ducktests-remote run ./ignitetest/tests/smoke_test.py ``` +Test paths are the ones you already use with `docker/run_tests.sh`: relative to +`modules/ducktests/tests`. Run the command from anywhere inside the checkout - the +repository root is found by walking up, and paths relative to it work too. + ## The three coordinators **Laptop.** The runner is remote, so the key ducktape uses has to live on the runner. @@ -139,7 +143,7 @@ dies with it. ```bash ducktests-remote keys push # installs the identity on the runner, authorises it on the workers ducktests-remote doctor -ducktests-remote run -t ./modules/ducktests/tests/ignitetest/tests/smoke_test.py --detach +ducktests-remote run -t ./ignitetest/tests/smoke_test.py --detach ducktests-remote logs -f # reattach later, from anywhere ``` @@ -148,7 +152,7 @@ ducktests-remote logs -f # reattach later, from anywhere ```bash ducktests-remote --runner local doctor -ducktests-remote --runner local run ./modules/ducktests/tests/ignitetest/ +ducktests-remote --runner local run ./ignitetest/ ``` **Jenkins agent.** Use `--detach` plus `status --json`, and read the exit code. @@ -333,10 +337,11 @@ java: 3. **a JDK under `search_paths`** — `/opt/jdk-17.0.11`, `/usr/lib/jvm/java-17-openjdk`. Highest patch level wins, compared numerically, so `17.0.11` beats `17.0.9`. 4. **`java.archive`**, delivered from the coordinator to the hosts that got this far — - and only to those. A `.tar.gz`, `.tgz`, `.tar` or an unpacked directory; a single - top-level directory is stripped, so a stock Temurin tarball lands as - `/opt/jdk-17.0.11+9`. Bad archives (no `bin/java`, a macOS build with `Contents/Home`, - a zip) fail on the coordinator, before anything is copied to twelve machines. + and only to those. A JDK tarball (`.tar.gz`, `.tgz`, `.tar`, `.tar.bz2`, `.tar.xz`) or + an unpacked directory. Whatever wraps the JDK home inside the archive is stripped, so + both a stock Temurin tarball and a hand-repacked one land as `/opt/jdk-17.0.11+9`. Bad + archives (no `bin/java` anywhere, a macOS build with `Contents/Home`, a zip) fail on + the coordinator, before anything is copied to twelve machines. 5. otherwise a failure listing every JDK that *was* found. `--install-jdk` (with `--sudo`) adds the distribution's own package as a last rung. diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py index b1421edda6841..17fe2358ad6a3 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py @@ -209,18 +209,63 @@ def check_a_single_top_level_directory_is_stripped(self, tmp_path): assert plan.name == "jdk-17.0.11+9", "the target directory takes the JDK's own name" assert plan.bytes == 8 - def check_an_archive_without_bin_java_is_refused(self, tmp_path): - # A macOS build has Contents/Home in between and is worth catching here rather - # than on twelve hosts at once. + def check_a_macos_build_is_refused_by_name(self, tmp_path): + # It unpacks perfectly well and then fails on every worker, so it has to be + # caught here rather than on twelve hosts at once. with pytest.raises(ConfigError) as ex: java.archive_plan(_tarball(tmp_path, ["Contents/Home/bin/java"])) - assert "bin/java" in str(ex.value) + assert "macOS" in str(ex.value) and "Linux x64" in str(ex.value) + + def check_an_archive_without_any_java_is_refused(self, tmp_path): + with pytest.raises(ConfigError) as ex: + java.archive_plan(_tarball(tmp_path, ["lib/modules", "release"])) + assert "does not contain bin/java" in str(ex.value) def check_a_flat_archive_is_not_stripped(self, tmp_path): plan = java.archive_plan(_tarball(tmp_path, ["bin/java", "lib/modules"], top="")) assert plan.strip == 0 and plan.top_level is None assert plan.name == "jdk" + def check_a_deeper_layout_is_stripped_to_the_java_home(self, tmp_path): + # `tar czf` of a directory that itself holds the unpacked JDK. The wrapper is + # not a reason to refuse the archive; it is a reason to strip one level more. + plan = java.archive_plan( + _tarball(tmp_path, ["jdk-17.0.11+9/bin/java", "jdk-17.0.11+9/lib/modules"], + top="openjdk-17")) + assert plan.strip == 2 and plan.top_level == "openjdk-17/jdk-17.0.11+9" + assert plan.name == "jdk-17.0.11+9", "the target takes the JDK's own name" + + def check_extra_entries_beside_the_jdk_do_not_confuse_the_plan(self, tmp_path): + # AppleDouble junk, a stray LICENSE, a second top-level file: none of it changes + # where bin/java is. + path = tmp_path / "jdk.tar.gz" + with tarfile.open(path, "w:gz") as tar: + for name in ("._jdk-17.0.11", "LICENSE", "jdk-17.0.11/bin/java"): + info = tarfile.TarInfo(name) + info.size = 4 + tar.addfile(info, io.BytesIO(b"data")) + plan = java.archive_plan(str(path)) + assert plan.strip == 1 and plan.top_level == "jdk-17.0.11" + + def check_the_shallowest_java_wins(self, tmp_path): + # A bundled JRE deeper in the tree must not drag the strip depth with it. + plan = java.archive_plan( + _tarball(tmp_path, ["bin/java", "legal/jre/bin/java"], top="jdk-17")) + assert plan.strip == 1 and plan.top_level == "jdk-17" + + def check_the_tar_flag_follows_the_suffix(self, tmp_path): + # The worker-side `tar` is told how to decompress; guessing gzip for an .xz + # archive passes every coordinator-side check and fails on every host. + gz = java.archive_plan(_tarball(tmp_path, ["bin/java"])) + assert gz.tar_flag == "z" + + xz_path = tmp_path / "jdk.tar.xz" + with tarfile.open(xz_path, "w:xz") as tar: + info = tarfile.TarInfo("jdk-17/bin/java") + info.size = 4 + tar.addfile(info, io.BytesIO(b"data")) + assert java.archive_plan(str(xz_path)).tar_flag == "J" + def check_a_directory_source_is_accepted(self, tmp_path): home = tmp_path / "jdk-17" (home / "bin").mkdir(parents=True) @@ -297,8 +342,8 @@ def check_a_host_without_a_match_gets_the_archive(self, tmp_path): ctx, fake = _context(PROBE, major=21, archive=str(archive), install_root="/opt") cfg = java.config_of(ctx) plan = java.archive_plan(cfg.archive) - result = provision._jdk_on_host(ctx, NODE, cfg, plan, # noqa: SLF001 - java.discovery_script(cfg)) + result = provision._jdk_on_host( # noqa: SLF001 + ctx, NODE, cfg, provision.JdkPayload(plan), java.discovery_script(cfg)) assert result.status == provision.CHANGED assert fake.uploads, "the archive has to reach the host" @@ -317,8 +362,8 @@ def check_a_host_that_already_has_that_archive_is_skipped(self, tmp_path): digest = provision._tar_manifest(plan)["hash"] # noqa: SLF001 fake.when("cat", json.dumps({"hash": digest})) - result = provision._jdk_on_host(ctx, NODE, cfg, plan, # noqa: SLF001 - java.discovery_script(cfg)) + result = provision._jdk_on_host( # noqa: SLF001 + ctx, NODE, cfg, provision.JdkPayload(plan), java.discovery_script(cfg)) assert result.status == provision.OK and not fake.uploads def check_no_archive_and_no_match_fails_with_the_config_keys(self): @@ -334,9 +379,9 @@ def check_a_missing_explicit_home_is_never_papered_over(self, tmp_path): archive = _tarball(tmp_path, ["bin/java"]) ctx, fake = _context(PROBE, home="/opt/vendor-jdk", archive=str(archive)) cfg = java.config_of(ctx) - result = provision._jdk_on_host(ctx, NODE, cfg, # noqa: SLF001 - java.archive_plan(cfg.archive), - java.discovery_script(cfg)) + result = provision._jdk_on_host( # noqa: SLF001 + ctx, NODE, cfg, provision.JdkPayload(java.archive_plan(cfg.archive)), + java.discovery_script(cfg)) assert result.status == provision.FAILED and "/opt/vendor-jdk" in result.message assert not fake.uploads diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_runs.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_runs.py index 5499a12a9089d..07cea8c26dce4 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_runs.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_runs.py @@ -23,7 +23,7 @@ GOLDEN_RUN_SH = """set -euo pipefail -cd '/opt/my sources/ignite' +cd '/opt/my sources/ignite/modules/ducktests/tests' # activate the runner venv; `set +u` because older activate # scripts read unset variables set +u @@ -124,7 +124,7 @@ class CheckRunScript: def _render(self): return runs.render_run_script( version="0.1.0", timestamp="2026-07-27T00:00:00+00:00", author="max@laptop", - work_dir="/opt/my sources/ignite", + cwd="/opt/my sources/ignite/modules/ducktests/tests", results_root="/state/runs/r/results", cluster_file="/state/runs/r/cluster.json", globals_file="/state/runs/r/globals.json", @@ -154,7 +154,7 @@ def check_globals_is_passed_as_a_file_path(self): def check_optional_flags_are_omitted_when_unset(self): rendered = runs.render_run_script( - version="0.1.0", timestamp="t", author="a", work_dir="/w", + version="0.1.0", timestamp="t", author="a", cwd="/w", results_root="/r", cluster_file="/c", globals_file="/g", test_paths=["./t.py"], venv=None) assert "--parameters" not in rendered @@ -164,7 +164,7 @@ def check_optional_flags_are_omitted_when_unset(self): def check_a_json_like_test_path_is_quoted(self): rendered = runs.render_run_script( - version="0.1.0", timestamp="t", author="a", work_dir="/w", + version="0.1.0", timestamp="t", author="a", cwd="/w", results_root="/r", cluster_file="/c", globals_file="/g", test_paths=['./t.py::C.m@{"x": 1}'], venv=None) assert """'./t.py::C.m@{"x": 1}'""" in rendered diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_sources.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_sources.py new file mode 100644 index 0000000000000..53db575e46fa4 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_sources.py @@ -0,0 +1,176 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for finding the Ignite checkout and for the test paths handed to ducktape.""" + +import os + +import pytest + +from fake_transport import FakeTransport + +from ducktests_remote.cli import Console, Context +from ducktests_remote.commands import run +from ducktests_remote.config import ConfigError, load_config + +TESTS = ("modules", "ducktests", "tests") + + +def _checkout(tmp_path, name="ignite"): + """Build the smallest tree that counts as an Ignite checkout.""" + root = tmp_path / name + tests = root.joinpath(*TESTS) + (tests / "docker").mkdir(parents=True, exist_ok=True) + (tests / "docker" / "requirements.txt").write_text("ducktape==0.13.0\n", encoding="utf-8") + suite = tests / "ignitetest" / "tests" + suite.mkdir(parents=True, exist_ok=True) + (suite / "smoke_test.py").write_text("# tests\n", encoding="utf-8") + return root.resolve() + + +def _context(**args): + class Args: # pylint: disable=too-few-public-methods + """Minimal stand-in for the parsed command line.""" + + parsed = Args() + parsed.dry_run = False + parsed.source_root = None + parsed.no_sync = False + parsed.work_dir = None + for key, value in args.items(): + setattr(parsed, key, value) + ctx = Context(load_config(user_config=None), parsed, Console(color=False)) + ctx._runner = FakeTransport() # noqa: SLF001 - the point of the fake + return ctx + + +@pytest.fixture(name="in_dir") +def _in_dir(): + """Run a check from a chosen directory and restore the old one afterwards.""" + previous = os.getcwd() + yield os.chdir + os.chdir(previous) + + +class CheckSourceRoot: + """Where the sources are taken from when nothing says so explicitly.""" + + def check_the_checkout_root_is_found_from_the_tests_directory(self, tmp_path, in_dir): + # The Docker flow is run from here, so this is where operators stand. + root = _checkout(tmp_path) + in_dir(str(root.joinpath(*TESTS))) + assert run._source_root(_context()) == root.resolve() # noqa: SLF001 + + def check_the_checkout_root_is_found_from_deeper_still(self, tmp_path, in_dir): + root = _checkout(tmp_path) + in_dir(str(root.joinpath(*TESTS) / "ignitetest" / "tests")) + assert run._source_root(_context()) == root.resolve() # noqa: SLF001 + + def check_the_root_itself_is_recognised(self, tmp_path, in_dir): + root = _checkout(tmp_path) + in_dir(str(root)) + assert run._source_root(_context()) == root.resolve() # noqa: SLF001 + + def check_an_explicit_root_is_taken_as_given(self, tmp_path, in_dir): + root = _checkout(tmp_path) + in_dir(str(tmp_path)) + ctx = _context(source_root=str(root)) + assert run._source_root(ctx) == root.resolve() # noqa: SLF001 + + def check_a_directory_outside_any_checkout_falls_back_to_itself(self, tmp_path, in_dir): + # Kept as the old behaviour so the failure comes from the validation below, with + # a message, rather than from an unrelated parent directory being picked up. + elsewhere = tmp_path / "state" + elsewhere.mkdir() + in_dir(str(elsewhere)) + assert run._source_root(_context()) == elsewhere.resolve() # noqa: SLF001 + + +class CheckSourceRootValidation: + """The mistake this exists for: syncing something that is not the checkout.""" + + def check_a_non_checkout_is_refused_before_anything_is_uploaded(self, tmp_path): + state = tmp_path / ".ducktests-remote" + state.mkdir() + with pytest.raises(ConfigError) as ex: + run._check_source_root(_context(), state) # noqa: SLF001 + message = str(ex.value) + assert str(state) in message, "the message names the directory that was wrong" + assert "modules/ducktests/tests/docker/requirements.txt" in message + assert "--source-root" in message, "and how to fix it" + + def check_a_real_checkout_passes(self, tmp_path): + run._check_source_root(_context(), _checkout(tmp_path)) # noqa: SLF001 + + def check_no_sync_against_an_unseen_tree_is_allowed(self, tmp_path): + # With --no-sync the path describes the runner, which the coordinator cannot check. + run._check_source_root(_context(no_sync=True), # noqa: SLF001 + tmp_path / "on-the-runner-only") + + def check_no_sync_still_refuses_a_local_directory_that_is_wrong(self, tmp_path): + state = tmp_path / ".ducktests-remote" + state.mkdir() + with pytest.raises(ConfigError): + run._check_source_root(_context(no_sync=True), state) # noqa: SLF001 + + +class CheckTestPaths: + """ducktape runs from the tests directory, so the paths it gets are relative to it.""" + + @staticmethod + def _paths(root, given, cwd=None): + os.chdir(str(cwd or root.joinpath(*TESTS))) + return run._test_paths(_context(), given, root, # noqa: SLF001 + "/state/runs/r/src") + + def check_a_docker_style_path_survives_unchanged(self, tmp_path, in_dir): + in_dir(str(tmp_path)) + assert self._paths(_checkout(tmp_path), ["./ignitetest/tests/smoke_test.py"]) == \ + ["./ignitetest/tests/smoke_test.py"] + + def check_the_class_and_method_suffix_is_preserved(self, tmp_path, in_dir): + in_dir(str(tmp_path)) + given = "./ignitetest/tests/smoke_test.py::SmokeServicesTest.test_ignite_start_stop" + assert self._paths(_checkout(tmp_path), [given]) == [given] + + def check_a_repository_relative_path_is_shortened(self, tmp_path, in_dir): + # What README used to require, typed from the checkout root. + in_dir(str(tmp_path)) + root = _checkout(tmp_path) + given = "./modules/ducktests/tests/ignitetest/tests/smoke_test.py" + assert self._paths(root, [given], cwd=root) == ["./ignitetest/tests/smoke_test.py"] + + def check_an_absolute_path_is_shortened_too(self, tmp_path, in_dir): + in_dir(str(tmp_path)) + root = _checkout(tmp_path) + absolute = str(root.joinpath(*TESTS) / "ignitetest" / "tests" / "smoke_test.py") + assert self._paths(root, [absolute]) == ["./ignitetest/tests/smoke_test.py"] + + def check_a_path_inside_the_checkout_but_outside_tests_becomes_absolute( + self, tmp_path, in_dir): + # Relative would be resolved against the tests directory and miss; the runner-side + # absolute path is the only form that cannot be misread. + in_dir(str(tmp_path)) + root = _checkout(tmp_path) + (root / "extra").mkdir(exist_ok=True) + (root / "extra" / "other_test.py").write_text("# t\n", encoding="utf-8") + assert self._paths(root, ["./extra/other_test.py"], cwd=root) == \ + ["/state/runs/r/src/extra/other_test.py"] + + def check_an_unresolvable_path_is_passed_through(self, tmp_path, in_dir): + # --no-sync against a checkout this machine does not have. + in_dir(str(tmp_path)) + assert self._paths(_checkout(tmp_path), ["./ignitetest/tests/absent.py"]) == \ + ["./ignitetest/tests/absent.py"] diff --git a/modules/ducktests/tests/ducktests_remote/commands/provision.py b/modules/ducktests/tests/ducktests_remote/commands/provision.py index 204df63d5974c..704dd14db831f 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/provision.py +++ b/modules/ducktests/tests/ducktests_remote/commands/provision.py @@ -28,6 +28,7 @@ import posixpath import shlex import tempfile +import threading import uuid from pathlib import Path @@ -249,6 +250,7 @@ def _run_jdk_step(ctx, nodes): % deploy.human(plan.bytes * len(nodes))) script = java.discovery_script(cfg) + payload = JdkPayload(plan) if plan else None def operation(node): if ctx.dry_run: @@ -256,13 +258,70 @@ def operation(node): % (node.host, cfg.major or "any")) ctx.console.detail(script) return HostResult(node.host, SKIPPED, "dry-run") - return _jdk_on_host(ctx, node, cfg, plan, script) + return _jdk_on_host(ctx, node, cfg, payload, script) - return fanout(nodes, operation, jobs=ctx.jobs, - fail_fast=getattr(ctx.args, "fail_fast", False)) + try: + return fanout(nodes, operation, jobs=ctx.jobs, + fail_fast=getattr(ctx.args, "fail_fast", False)) + finally: + if payload: + payload.close() -def _jdk_on_host(ctx, node, cfg, plan, script): +class JdkPayload: + """ + The bytes to upload for one :class:`~ducktests_remote.java.ArchivePlan`, prepared once. + + A tarball is uploaded as it is, so there is nothing to prepare. A directory has to be + packed first, and packing it inside the per-host operation meant gzipping the same JDK + once per worker, in parallel, on the coordinator. Both the tarball and the manifest + are therefore built on first use and shared by every host after that. + """ + + def __init__(self, plan): + self.plan = plan + self._lock = threading.Lock() + self._tmp = None + self._archive = None + self._manifest = None + + def archive(self): + """:return: the coordinator-side path of the tarball to upload.""" + if self.plan.kind == "tar": + return self.plan.path + with self._lock: + if self._archive is None: + self._tmp = tempfile.TemporaryDirectory() + packed = Path(self._tmp.name) / "jdk.tar.gz" + make_tarball(self.plan.path, packed) + self._archive = packed + return self._archive + + def manifest(self): + """:return: the manifest that decides whether a host already has this JDK.""" + with self._lock: + if self._manifest is None: + self._manifest = (deploy.build_manifest(self.plan.path) + if self.plan.kind == "dir" else _tar_manifest(self.plan)) + return self._manifest + + def strip(self): + """:return: ``--strip-components`` depth; a packed directory is already flat.""" + return 0 if self.plan.kind == "dir" else self.plan.strip + + def tar_flag(self): + """:return: the ``tar`` decompression flag; a packed directory is always gzip.""" + return "z" if self.plan.kind == "dir" else self.plan.tar_flag + + def close(self): + """Remove the packed tarball, if one was made.""" + if self._tmp is not None: + self._tmp.cleanup() + self._tmp = None + self._archive = None + + +def _jdk_on_host(ctx, node, cfg, payload, script): transport = ctx.worker(node) probe = transport.run_script(script, check=False) if not probe.ok: @@ -280,8 +339,8 @@ def _jdk_on_host(ctx, node, cfg, plan, script): "java.home %s has no usable bin/java on this host" % cfg.home, detail=_found(res)) - if plan is not None: - return _deliver_jdk(ctx, node, cfg, plan) + if payload is not None: + return _deliver_jdk(ctx, node, cfg, payload) if ctx.args.install_jdk: return _install_jdk(ctx, node, cfg) @@ -299,16 +358,17 @@ def _found(res): for home, major, _ in res.candidates) -def _deliver_jdk(ctx, node, cfg, plan): +def _deliver_jdk(ctx, node, cfg, payload): """ Copy the JDK to one worker, reusing ``deploy``'s staging and atomic swap. A half-extracted JDK that looks present is exactly as bad as a half-extracted distribution, which is why this does not extract in place. """ + plan = payload.plan transport = ctx.worker(node) target = java.target_dir(cfg, plan) - manifest = deploy.build_manifest(plan.path) if plan.kind == "dir" else _tar_manifest(plan) + manifest = payload.manifest() if not ctx.args.force: existing = transport.read_file(posixpath.join(target, JAVA_MANIFEST_NAME)) @@ -330,21 +390,13 @@ def _deliver_jdk(ctx, node, cfg, plan): uuid.uuid4().hex[:8]) transport.run_script(deploy.prepare_script(staging, ctx.args.sudo)).check() - with tempfile.TemporaryDirectory() as tmp: - if plan.kind == "dir": - archive = Path(tmp) / "jdk.tar.gz" - make_tarball(plan.path, archive) - strip = 0 - else: - archive = plan.path - strip = plan.strip - remote = "%s/.payload.tar.gz" % staging - transport.upload(archive, remote) - transport.run_script( - "set -eu\ntar -xzf %s -C %s%s\nrm -f -- %s\n" - % (shlex.quote(remote), shlex.quote(staging), - " --strip-components=%d" % strip if strip else "", - shlex.quote(remote))).check() + remote = "%s/.payload.tar" % staging + transport.upload(payload.archive(), remote) + transport.run_script( + "set -eu\ntar -x%sf %s -C %s%s\nrm -f -- %s\n" + % (payload.tar_flag(), shlex.quote(remote), shlex.quote(staging), + " --strip-components=%d" % payload.strip() if payload.strip() else "", + shlex.quote(remote))).check() check = transport.run(["test", "-x", "%s/bin/java" % staging], check=False) if not check.ok: diff --git a/modules/ducktests/tests/ducktests_remote/commands/run.py b/modules/ducktests/tests/ducktests_remote/commands/run.py index 8a0f6647ba5a0..5a684369e22f8 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/run.py +++ b/modules/ducktests/tests/ducktests_remote/commands/run.py @@ -36,9 +36,16 @@ IGNITE_IGNORE_FILE = ".ducktestsignore" +# Where the ducktests live inside an Ignite checkout, and the file that proves it is one. +TESTS_SUBDIR = ("modules", "ducktests", "tests") +CHECKOUT_MARKER = ("docker", "requirements.txt") + FOLLOW_POLL_SEC = 1.5 DOUBLE_INTERRUPT_SEC = 3.0 +# run.sh exits with this when the requirements file it would install from is absent. +MISSING_REQUIREMENTS_EXIT = 3 + def register(subparsers, common): """Wire up the ``run`` subcommand.""" @@ -47,7 +54,9 @@ def register(subparsers, common): description="Compose the configuration, generate cluster.json / globals.json / " "run.sh on the runner, and launch ducktape detached.") parser.add_argument("test_paths", nargs="*", metavar="TEST_PATH", - help="test paths, as ducktape understands them") + help="test paths, relative to modules/ducktests/tests as in the " + "Docker flow (./ignitetest/tests/smoke_test.py::Cls.method); " + "paths relative to the checkout root also work") parser.add_argument("-t", "--tc-path", action="append", default=[], metavar="PATH", help="test path; repeatable, equivalent to a positional argument") parser.add_argument("-g", "--global", action="append", default=[], dest="globals_kv", @@ -65,13 +74,15 @@ def register(subparsers, common): parser.add_argument("-n", "--num-nodes", type=int, default=None, metavar="N", help="use the first N inventory hosts (default: all)") parser.add_argument("--source-root", metavar="PATH", - help="directory synced to the runner (default: the current directory)") + help="Ignite checkout synced to the runner (default: the checkout " + "containing the current directory)") parser.add_argument("--no-sync", action="store_true", help="assume the sources are already on the runner") parser.add_argument("--exclude", action="append", default=[], metavar="PATTERN", help="extra sync exclusion; repeatable") parser.add_argument("--work-dir", metavar="PATH", - help="runner-side working directory for ducktape " + help="runner-side Ignite checkout to run from; ducktape itself " + "runs in its modules/ducktests/tests directory " "(default: the synced source root)") parser.add_argument("--install-sources", action="store_true", help="pip install the synced sources into the runner venv. Not needed " @@ -105,15 +116,18 @@ def execute(ctx): # pylint: disable=too-many-return-statements args = ctx.args console = ctx.console - test_paths = list(args.test_paths) + list(args.tc_path) - if not test_paths: + raw_test_paths = list(args.test_paths) + list(args.tc_path) + if not raw_test_paths: raise ConfigError("no tests given; pass a path positionally or with -t/--tc-path") + source_root = _source_root(ctx) + _check_source_root(ctx, source_root) + composed, params = _compose_payloads(ctx) nodes = ctx.nodes cluster_payload, cluster_source, cluster_text = _cluster_payload(ctx, nodes) - _warn_about_topology(ctx, nodes, test_paths) + _warn_about_topology(ctx, nodes, raw_test_paths) if not args.skip_preflight and not ctx.dry_run: console.heading("PREFLIGHT") @@ -127,14 +141,14 @@ def execute(ctx): # pylint: disable=too-many-return-statements state_root = ctx.runner.expand(ctx.state_root) paths = runs.RunPaths(state_root, run_id) - source_root = _source_root(ctx) work_dir = _work_dir(ctx, paths, source_root) results_root = args.results_root or paths.results_dir + test_paths = _test_paths(ctx, raw_test_paths, source_root, work_dir) run_sh = runs.render_run_script( version=__version__, timestamp=runs.utc_now_iso(), author="%s@%s" % (_coordinator_user(), socket.gethostname()), - work_dir=work_dir, results_root=results_root, + cwd=_tests_dir(work_dir), results_root=results_root, cluster_file=paths.cluster_file, globals_file=paths.globals_file, test_paths=test_paths, venv=_venv_path(ctx), parameters_file=paths.parameters_file if params else None, @@ -259,8 +273,106 @@ def _versions(composed): def _source_root(ctx): + """ + :return: the Ignite checkout to sync, as a coordinator-side path. + + An explicit ``--source-root`` / ``run.source_root`` is taken as given. Otherwise the + current directory is walked upwards until a checkout is found, so the command works + from ``modules/ducktests/tests`` - where ``./docker/run_tests.sh`` is run from - as + well as from the repository root. + """ configured = ctx.args.source_root or ctx.config["run"].get("source_root") - return Path(expand_path(configured) or os.getcwd()).resolve() + if configured: + return Path(expand_path(configured)).resolve() + cwd = Path(os.getcwd()).resolve() + for candidate in (cwd,) + tuple(cwd.parents): + if is_ignite_checkout(candidate): + return candidate + return cwd + + +def is_ignite_checkout(path): + """:return: True when ``path`` is the root of an Ignite source tree.""" + return Path(path).joinpath(*TESTS_SUBDIR).joinpath(*CHECKOUT_MARKER).is_file() + + +def _check_source_root(ctx, source_root): + """ + Reject a source root that is not an Ignite checkout, before anything is uploaded. + + Getting this wrong is easy - the state root and the tests directory are both plausible + places to stand - and every symptom of it appears much later, as a missing + requirements file or as ducktape finding no tests. + """ + if is_ignite_checkout(source_root): + return + if ctx.args.no_sync and not source_root.exists(): + return # a runner-side path that this machine cannot see; nothing to check + raise ConfigError( + "%s is not an Ignite checkout: it has no %s.\nThe whole source tree is synced to " + "the runner and ducktape is run from its %s directory, so this must be the " + "repository root (the directory that contains `modules/`). Run the command from " + "anywhere inside the checkout, or pass --source-root / set run.source_root." + % (source_root, posixpath.join(*TESTS_SUBDIR, *CHECKOUT_MARKER), + posixpath.join(*TESTS_SUBDIR))) + + +def _tests_dir(work_dir): + """:return: the runner-side directory ducktape runs from.""" + return posixpath.join(work_dir, *TESTS_SUBDIR) + + +def _test_paths(ctx, raw_paths, source_root, work_dir): + """ + Rewrite the given test paths into the form ducktape sees on the runner. + + ducktape's working directory is the tests directory, the same one + ``./docker/run_tests.sh`` uses, so ``./ignitetest/tests/smoke_test.py`` means here + what it means locally. A path is accepted in whatever form resolves on the + coordinator - relative to the current directory, to the tests directory, or to the + checkout root - and is rewritten from there. + """ + tests_local = source_root.joinpath(*TESTS_SUBDIR) + return [_one_test_path(ctx, raw, source_root, tests_local, work_dir) + for raw in raw_paths] + + +def _one_test_path(ctx, raw, source_root, tests_local, work_dir): + body, sep, suffix = raw.partition("::") + resolved = _resolve_test_file(body, source_root, tests_local) + + if resolved is None: + # Legitimate under --no-sync against a checkout this machine does not have, and + # ducktape gives a better message than a guess would. + ctx.console.warn("test path %s does not exist on this machine; passing it to " + "ducktape unchanged" % body) + return raw + + try: + return "./%s%s%s" % (resolved.relative_to(tests_local).as_posix(), sep, suffix) + except ValueError: + pass + + try: + rel = resolved.relative_to(source_root) + except ValueError: + ctx.console.warn("test path %s is outside the source root %s, so it is not synced; " + "it will only run if that exact path exists on the runner" + % (resolved, source_root)) + return raw + return "%s%s%s" % (posixpath.join(work_dir, rel.as_posix()), sep, suffix) + + +def _resolve_test_file(body, source_root, tests_local): + """:return: the coordinator-side file ``body`` names, or None if it names none.""" + expanded = Path(expand_path(body)) + if expanded.is_absolute(): + return expanded.resolve() if expanded.exists() else None + for base in (Path(os.getcwd()), tests_local, source_root): + candidate = base / expanded + if candidate.exists(): + return candidate.resolve() + return None def _work_dir(ctx, paths, source_root): @@ -337,7 +449,7 @@ def _ensure_venv(ctx, work_dir): return python = ctx.config["runner"].get("python", "python3") requirements = ctx.config["runner"].get("requirements") or posixpath.join( - work_dir, "modules", "ducktests", "tests", "docker", "requirements.txt") + _tests_dir(work_dir), *CHECKOUT_MARKER) index_arg = pipconf.pip_args_str(ctx.config) script = """set -eu @@ -362,20 +474,28 @@ def _ensure_venv(ctx, work_dir): "python": shlex.quote(python), "index": index_arg} result = ctx.runner.run_script(script, check=False) + if result.returncode == MISSING_REQUIREMENTS_EXIT: + # Nothing was installed and nothing was reached: the file simply is not there. + raise ConfigError( + "the runner venv at %s has no ducktape, and the requirements file to install " + "it from does not exist on the runner:\n %s\nThat is /%s, so it " + "normally means the synced tree is not an Ignite checkout. Check --source-root, " + "or set runner.requirements to a file the runner does have." + % (venv, requirements, posixpath.join(*TESTS_SUBDIR, *CHECKOUT_MARKER))) if not result.ok: raise ConfigError( - "could not prepare the runner venv at %s:\n%s\nInstalling from: %s.\nEither " - "point runner.venv at an existing environment, make %s reachable on the " - "runner, or set pip.index_url to an index the runner can reach." - % (venv, (result.stderr or result.stdout).strip(), - pipconf.describe(ctx.config, ctx.console.redactor), requirements)) + "could not prepare the runner venv at %s:\n%s\nInstalling %s from: %s.\nEither " + "point runner.venv at an existing environment, or set pip.index_url to an " + "index the runner can reach." + % (venv, (result.stderr or result.stdout).strip(), requirements, + pipconf.describe(ctx.config, ctx.console.redactor))) ctx.console.info(result.out.splitlines()[-1] if result.out else "venv ready") def _install_sources(ctx, work_dir): venv = _venv_path(ctx) pip = posixpath.join(venv, "bin", "pip3") if venv else "pip3" - tests_dir = posixpath.join(work_dir, "modules", "ducktests", "tests") + tests_dir = _tests_dir(work_dir) ctx.console.info("installing sources from %s" % tests_dir) ctx.runner.run([pip, "install", "--disable-pip-version-check"] + pipconf.pip_args(ctx.config) + ["-e", tests_dir]).check() diff --git a/modules/ducktests/tests/ducktests_remote/docs/commands.md b/modules/ducktests/tests/ducktests_remote/docs/commands.md index 5dc941824ed20..7f708af8b58c8 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/commands.md +++ b/modules/ducktests/tests/ducktests_remote/docs/commands.md @@ -71,6 +71,28 @@ run [TEST_PATH...] [-t PATH]... [-g KEY=VALUE]... [-p KEY=VALUE]... At least one test path is required, positionally or with `-t`. +### Where you run it from, and what a test path means + +The whole Ignite checkout is synced to the runner, and ducktape is started in its +`modules/ducktests/tests` directory — the same working directory `docker/run_tests.sh` +uses. So a test path is written exactly as it is locally: + +``` +ducktests-remote run ./ignitetest/tests/smoke_test.py::SmokeServicesTest.test_ignite_start_stop +``` + +The checkout is found by walking up from the current directory, so the command works from +`modules/ducktests/tests`, from the repository root, or from anywhere in between. +`--source-root` / `run.source_root` overrides the search. A directory that is not a +checkout — no `modules/ducktests/tests/docker/requirements.txt` under it — is refused +before anything is uploaded. + +Test paths are accepted in any form that resolves on the coordinator (relative to the +current directory, to the tests directory, or to the checkout root) and are rewritten +into the tests-relative form ducktape sees. A path inside the checkout but outside the +tests directory becomes a runner-side absolute path; one that resolves nowhere is passed +to ducktape unchanged, with a warning. + ### Order of operations 1. **Compose `globals`**: `--globals-json`/`--globals-file` (the raw layer), then config @@ -84,7 +106,7 @@ At least one test path is required, positionally or with `-t`. 4. **Preflight**: the full `doctor` check set, unless `--skip-preflight` or `--dry-run`. Any FAIL stops here with exit 2, before anything is created. 5. **Allocate the run id** — `max-20260727-141233-9f2a` — and derive the run directory, - work directory and results root. + work directory and results root, and normalise the test paths against the checkout. 6. **Render `run.sh`** and, with `--dry-run` or `-v`, print it along with `cluster.json` and a redacted `globals.json`. **`--dry-run` returns here.** 7. **Create** the run and results directories on the runner. @@ -95,6 +117,7 @@ At least one test path is required, positionally or with `-t`. 9. **Ensure the venv**: create `/venv` when missing, and install `docker/requirements.txt` into it when `import ducktape` fails. This is where `pip.*` applies. Runs after the sync because the requirements file comes from the synced tree. + A missing requirements file is reported as such, separately from an unreachable index. 10. **`--install-sources`** (opt-in): `pip install -e /modules/ducktests/tests`, with the same `pip.*` flags. 11. **Write the artifacts**: `cluster.json`, `globals.json` (`0600`), `parameters.json` diff --git a/modules/ducktests/tests/ducktests_remote/docs/configuration.md b/modules/ducktests/tests/ducktests_remote/docs/configuration.md index 903b2d6d9c97b..23c490354f3f7 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/configuration.md +++ b/modules/ducktests/tests/ducktests_remote/docs/configuration.md @@ -162,9 +162,9 @@ Read by `provision` (`jdk`, `ssh-env`) and by `doctor`. Full treatment in | `major` | `17` | Java major version the tests need. Derived from the Dockerfile's `ARG jdk_version="eclipse-temurin:17"`; the Dockerfile wins if they ever disagree | | `home` | `null` | an explicit JDK home on the workers. Set means *exactly this*: no search, and a host without it fails | | `search_paths` | `[/opt, /usr/lib/jvm, /usr/java]` | where to look for an existing JDK. Each entry may be a directory *of* JDK homes or a JDK home itself | -| `archive` | `null` | coordinator-side `.tar.gz`/`.tgz`/`.tar` or an unpacked directory, delivered to hosts that have no matching JDK | +| `archive` | `null` | coordinator-side JDK tarball (`.tar.gz`/`.tgz`/`.tar`/`.tar.bz2`/`.tar.xz`) or an unpacked directory, delivered to hosts that have no matching JDK | | `install_root` | `null` | where a delivered JDK is unpacked; unset means `cluster.install_root` | -| `name` | `null` | target directory name; unset means the archive's own top-level directory name | +| `name` | `null` | target directory name; unset means the name of the JDK home found inside the archive | | `ssh_environment` | `true` | write `~/.ssh/environment` | | `bashrc` | `true` | write a marked block at the top of `~/.bashrc` | @@ -175,8 +175,8 @@ put the JDK on the workers' non-interactive `PATH`. | Key | Default | Meaning | | --- | --- | --- | -| `source_root` | `null` | directory synced to the runner; unset means the current directory | -| `work_dir` | `null` | runner-side working directory for ducktape; unset means the synced source directory | +| `source_root` | `null` | Ignite checkout synced to the runner; unset means the checkout containing the current directory, found by walking up | +| `work_dir` | `null` | runner-side Ignite checkout to run from; ducktape itself runs in its `modules/ducktests/tests`; unset means the synced source directory | | `exclude` | `[]` | extra sync exclusions, appended to the built-in list | | `max_payload_mb` | `200` | refuse to sync more than this. A build directory leaking into the payload is the usual cause | | `install_sources` | `false` | `pip install -e` the synced sources. Not needed for discovery: ducktape's loader walks up from each test file while `__init__.py` exists and puts the resulting top-level directory on `sys.path` | diff --git a/modules/ducktests/tests/ducktests_remote/docs/java.md b/modules/ducktests/tests/ducktests_remote/docs/java.md index 3bb6dacd4ba8e..1f57ed3616194 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/java.md +++ b/modules/ducktests/tests/ducktests_remote/docs/java.md @@ -86,7 +86,7 @@ Within that step: | Phase | Where | What happens | | --- | --- | --- | -| 1 | coordinator, once | `java.archive` is opened with `tarfile`, checked for `bin/java` under its single top-level directory, and its size printed. **A bad archive fails here**, before a byte moves | +| 1 | coordinator, once | `java.archive` is opened with `tarfile`, searched for `bin/java`, and its size printed. **A bad archive fails here**, before a byte moves. A directory is packed here too, once for the whole run | | 2 | each host, in parallel | the discovery script runs rungs 1–3 | | 3 | each host that found nothing | delivery, below | @@ -106,8 +106,8 @@ So on a twelve-host cluster where eleven already carry Java 17, exactly one uplo 2. Check `install_root` is writable; if not and `--sudo` was not passed, fail naming the directory and the account. 3. Create a staging directory beside the target, upload the archive into it, and unpack — - `--strip-components=1` when the tarball has a single top-level directory, which is what - a stock Temurin tarball has. + with `--strip-components` set to whatever wraps the JDK home (1 for a stock Temurin + tarball) and the decompression flag taken from the file's suffix. 4. Verify `bin/java` exists in the staging tree; if not, remove the staging tree and fail. 5. Write the manifest, then **swap** the staging tree into place and delete the old one. @@ -116,12 +116,22 @@ that looks present is exactly as bad as a half-extracted distribution. ### Archive formats +`.tar.gz`, `.tgz`, `.tar`, `.tar.bz2`/`.tbz2` and `.tar.xz`/`.txz` are all accepted; the +`tar` flag on the worker follows the suffix. + +The JDK home is *located* rather than assumed: the shallowest `bin/java` in the archive +wins, and everything above it is stripped. A second, deeper `bin/java` (a bundled JRE) +cannot pull the depth with it. + | Given | Result | | --- | --- | -| `.tar.gz` / `.tgz` / `.tar` with one top-level dir | stripped; target defaults to that dir's name, e.g. `/opt/jdk-17.0.11+9` | -| the same, flat (`bin/java` at the root) | not stripped; target defaults to the file name without its suffix | -| an unpacked directory containing `bin/java` | tarred on the fly and sent | -| no `bin/java` where expected | **refused on the coordinator** — a macOS build with `Contents/Home` is the realistic case | +| one top-level dir, `jdk-17.0.11+9/bin/java` | stripped by 1; target defaults to that dir's name, e.g. `/opt/jdk-17.0.11+9` | +| flat, `bin/java` at the root | not stripped; target defaults to the file name without its suffix | +| wrapped deeper, `openjdk-17/jdk-17.0.11+9/bin/java` | stripped by 2; target defaults to `jdk-17.0.11+9` | +| extra entries beside the JDK (a stray `LICENSE`, AppleDouble `._*` files) | ignored; they do not change where `bin/java` is | +| an unpacked directory containing `bin/java` | packed once on the coordinator and sent to every host that needs it | +| no `bin/java` anywhere | **refused on the coordinator**, naming the archive | +| a macOS build (`Contents/Home` in the way) | **refused on the coordinator** — it unpacks fine and then fails on every worker | | `.zip` | **refused**, naming `.tar.gz`. Linux JDKs ship as tarballs, and a zip branch would be untested code on every real run | `java.name` overrides the target directory name; `java.install_root` overrides where it diff --git a/modules/ducktests/tests/ducktests_remote/examples/profile-smoke.yaml b/modules/ducktests/tests/ducktests_remote/examples/profile-smoke.yaml index 20aa05a7b0a62..2daf8ac482f05 100644 --- a/modules/ducktests/tests/ducktests_remote/examples/profile-smoke.yaml +++ b/modules/ducktests/tests/ducktests_remote/examples/profile-smoke.yaml @@ -15,7 +15,7 @@ # Smallest useful profile: the smoke suite against the locally built master. # -# ducktests-remote --profile smoke run ./modules/ducktests/tests/ignitetest/tests/smoke_test.py +# ducktests-remote --profile smoke run ./ignitetest/tests/smoke_test.py # globals: ignite_versions: ["dev"] diff --git a/modules/ducktests/tests/ducktests_remote/java.py b/modules/ducktests/tests/ducktests_remote/java.py index 0aae54b09cb62..ae16f92bfb826 100644 --- a/modules/ducktests/tests/ducktests_remote/java.py +++ b/modules/ducktests/tests/ducktests_remote/java.py @@ -66,6 +66,11 @@ _TAR_SUFFIXES = (".tar.gz", ".tgz", ".tar", ".tar.bz2", ".tbz2", ".tar.xz", ".txz") +# `tar` decompression flag per suffix. Explicit rather than relying on GNU tar's +# auto-detection, so the worker-side command does not depend on which tar is installed. +_TAR_FLAGS = ((".tar.gz", "z"), (".tgz", "z"), (".tar.bz2", "j"), (".tbz2", "j"), + (".tar.xz", "J"), (".txz", "J"), (".tar", "")) + @dataclass class JavaConfig: @@ -469,10 +474,11 @@ class ArchivePlan: path: Path kind: str # "tar" or "dir" - top_level: Optional[str] # single top-level directory inside a tarball + top_level: Optional[str] # directory prefix holding the JDK home, if any strip: int # --strip-components for tar bytes: int name: str # default target directory name + tar_flag: str = "z" # tar decompression flag matching the suffix def archive_plan(archive, name=None): @@ -480,9 +486,11 @@ def archive_plan(archive, name=None): Inspect ``java.archive`` on the coordinator. Reading the member list with :mod:`tarfile` rather than guessing in the shell is what - lets a bad archive fail *before* it is copied to every host: a Temurin tarball unpacks - into a single ``jdk-17.0.11+9/`` directory, and an archive with no ``bin/java`` under - it (a macOS build, with ``Contents/Home``) is worth catching here. + lets a bad archive fail *before* it is copied to every host. The JDK home is located + by finding ``bin/java`` and stripping whatever wraps it, so a Temurin tarball + (``jdk-17.0.11+9/bin/java``), a flat one (``bin/java``) and a doubly wrapped one all + work. A macOS build is still refused: ``Contents/Home`` unpacks fine and then fails + on every worker, which is a far worse place to find out. """ path = Path(expand_path(archive)) if not path.exists(): @@ -494,7 +502,7 @@ def archive_plan(archive, name=None): % path) total = sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) return ArchivePlan(path=path, kind="dir", top_level=None, strip=0, bytes=total, - name=name or path.name) + name=name or path.name, tar_flag="z") lowered = path.name.lower() if lowered.endswith(".zip"): @@ -512,28 +520,58 @@ def archive_plan(archive, name=None): except (tarfile.TarError, OSError) as ex: raise ConfigError("java.archive %s could not be read: %s" % (path, ex)) from ex - names = [m.name.lstrip("./") for m in members if m.name not in (".", "./")] + names = [_member_name(m) for m in members if m.name not in (".", "./")] + names = [n for n in names if n] if not names: raise ConfigError("java.archive %s is empty" % path) total = sum(m.size for m in members if m.isfile()) - tops = {n.split("/", 1)[0] for n in names if n} - if len(tops) == 1: - top = tops.pop() - _require_java(names, "%s/bin/java" % top, path) - return ArchivePlan(path=path, kind="tar", top_level=top, strip=1, bytes=total, - name=name or top) - _require_java(names, "bin/java", path) - return ArchivePlan(path=path, kind="tar", top_level=None, strip=0, bytes=total, - name=name or _strip_suffix(path.name)) + prefix = _java_home_prefix(names, path) + strip = prefix.count("/") + 1 if prefix else 0 + return ArchivePlan(path=path, kind="tar", top_level=prefix or None, strip=strip, + bytes=total, name=name or (posixpath.basename(prefix) if prefix + else _strip_suffix(path.name)), + tar_flag=_tar_flag(path.name)) + + +def _member_name(member): + """:return: ``member``'s path with a leading ``./`` removed, and nothing else.""" + name = member.name + while name.startswith("./"): + name = name[2:] + return name.rstrip("/") -def _require_java(names, expected, path): - if expected not in names: +def _java_home_prefix(names, path): + """ + :return: the directory prefix inside the tarball that *is* the JDK home ("" if flat). + + The shallowest ``bin/java`` wins, so a JDK that happens to ship a second one deeper + down (a bundled JRE, a jmod staging directory) cannot pull the strip depth with it. + """ + best = None + for name in names: + if name == "bin/java": + return "" + if name.endswith("/bin/java"): + candidate = name[: -len("/bin/java")] + if best is None or candidate.count("/") < best.count("/"): + best = candidate + if best is None: raise ConfigError( - "java.archive %s does not contain %s. A JDK for Linux unpacks with bin/java " - "directly under its top-level directory; a macOS build has Contents/Home in " - "between and cannot be used here." % (path, expected)) + "java.archive %s does not contain bin/java anywhere. Point it at a JDK " + "tarball or at an unpacked JDK home." % path) + if _is_macos_layout(best): + raise ConfigError( + "java.archive %s is a macOS JDK: bin/java sits under %s/bin/java, with " + "Contents/Home in between. It cannot run on the Linux workers - download the " + "Linux x64 build instead." % (path, best)) + return best + + +def _is_macos_layout(prefix): + parts = prefix.split("/") + return "Contents" in parts and "Home" in parts def target_dir(cfg: JavaConfig, plan: ArchivePlan): @@ -548,6 +586,15 @@ def _strip_suffix(filename): return filename +def _tar_flag(filename): + """:return: the ``tar`` decompression flag for ``filename``'s suffix.""" + lowered = filename.lower() + for suffix, flag in _TAR_FLAGS: + if lowered.endswith(suffix): + return flag + return "z" + + def _home_of(java_binary): return posixpath.dirname(posixpath.dirname(str(java_binary))) diff --git a/modules/ducktests/tests/ducktests_remote/runs.py b/modules/ducktests/tests/ducktests_remote/runs.py index 8d73a6ffdd1f9..584f474b1422e 100644 --- a/modules/ducktests/tests/ducktests_remote/runs.py +++ b/modules/ducktests/tests/ducktests_remote/runs.py @@ -269,13 +269,17 @@ def render_template(name, mapping): return text -def render_run_script(*, version, timestamp, author, work_dir, results_root, cluster_file, +def render_run_script(*, version, timestamp, author, cwd, results_root, cluster_file, globals_file, test_paths, venv=None, parameters_file=None, repeat=None, max_parallel=None, test_runner_timeout=None, extra_args=()): """ Render the ``run.sh`` that the runner executes. + ``cwd`` is the directory ducktape is started in - the checkout's + ``modules/ducktests/tests``, matching ``docker/run_tests.sh`` - so the test paths in + the generated script read the same as they do in a local Docker run. + ducktape 0.13 accepts a *file path* for ``--globals`` (``command_line/main.py`` checks ``os.path.isfile`` before parsing the argument as JSON), so the composed blob is referenced by path and never crosses a shell command line. @@ -309,7 +313,7 @@ def render_run_script(*, version, timestamp, author, work_dir, results_root, clu "version": version, "timestamp": timestamp, "author": author, - "work_dir": shlex.quote(str(work_dir)), + "cwd": shlex.quote(str(cwd)), "venv_activate": venv_activate, "results_root": shlex.quote(str(results_root)), "cluster_file": shlex.quote(str(cluster_file)), diff --git a/modules/ducktests/tests/ducktests_remote/templates/run.sh.tmpl b/modules/ducktests/tests/ducktests_remote/templates/run.sh.tmpl index a59fd9580bc39..e8c0e102b3ff9 100644 --- a/modules/ducktests/tests/ducktests_remote/templates/run.sh.tmpl +++ b/modules/ducktests/tests/ducktests_remote/templates/run.sh.tmpl @@ -19,7 +19,7 @@ # host and execute this exact script to reproduce the run by hand. set -euo pipefail -cd {{work_dir}} +cd {{cwd}} {{venv_activate}} exec ducktape \ --results-root {{results_root}} \ From c9c6e7298661cf647440c2f95f1c2a3287b1c4aa Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 28 Jul 2026 20:19:39 +0300 Subject: [PATCH 5/9] Let ducktests-remote deploy leave files out of a distribution ignite-dev is normally a link to a source checkout, and a built checkout is well over a gigabyte of which the workers open almost nothing: they need modules/*/target/*.jar, modules/*/target/libs/*.jar, bin/ and the ducktests certs. deploy had no way to say that, so the whole tree went over the wire to every host. Add --exclude, a .ducktests-deploy.ignore file at the root of a distribution, and deploy.exclude in the configuration, in that order of precedence. The patterns are the rsync-style ones the source sync already uses. Excludes default to nothing, so a release directory is still shipped byte for byte. The manifest is built from the same filtered list as the tarball; otherwise a host would be reported up to date while holding a different set of files. The ignore file is deliberately not called .ducktestsignore: when ignite-dev links to a checkout the distribution root and the source root are the same directory, and the two lists are opposites - the source sync drops target, and deploy keeps little else. --- .../tests/ducktests_remote/README.md | 41 ++++++++ .../checks/check_remote_deploy.py | 99 +++++++++++++++++++ .../tests/ducktests_remote/commands/deploy.py | 71 ++++++++++--- .../tests/ducktests_remote/config.py | 4 + .../tests/ducktests_remote/docs/commands.md | 80 ++++++++++++++- .../ducktests_remote/docs/configuration.md | 7 ++ .../ducktests_remote/docs/troubleshooting.md | 13 +++ .../ducktests_remote/examples/cluster.yaml | 10 ++ 8 files changed, 310 insertions(+), 15 deletions(-) diff --git a/modules/ducktests/tests/ducktests_remote/README.md b/modules/ducktests/tests/ducktests_remote/README.md index 784d8cf26e8da..20a78e2f68f26 100644 --- a/modules/ducktests/tests/ducktests_remote/README.md +++ b/modules/ducktests/tests/ducktests_remote/README.md @@ -246,6 +246,47 @@ place, because a half-copied distribution that looks present is worse than an ab On a twelve-host cluster a 300 MB distribution is 3.7 GB over the wire from a laptop. `deploy` prints that total before it starts, and suggests `--via`. +### `ignite-dev` from your own checkout + +`ignite-dev` must look like a *built source tree* on the worker, not like a release: +`IgniteSpec` classpaths `modules//target` and `modules//target/libs`, +`bin/ignite.sh` is run from the same home, and SSL tests read +`modules/ducktests/tests/certs`. Everything else in a checkout is ballast — a bare copy +of a built repository is well over a gigabyte, most of it `.git` and sources the workers +never open. + +Link the distribution to your checkout, and trim it with excludes: + +```bash +mkdir -p ~/dist +ln -sfn ~/dev/ignite ~/dist/ignite-dev # relink any time; deploy follows it +``` + +```yaml +deploy: + dist_dir: ~/dist + exclude: [.git, .idea, src, docs, assembly, classes, test-classes, + generated-sources, generated-test-sources, maven-status, maven-archiver, + surefire-reports, javadoc, "*.tar.gz", "*.zip", __pycache__, "*.pyc"] +``` + +```bash +mvn package -pl :ignite-ducktests -am -DskipTests # you build, however you like +ducktests-remote deploy --only ignite-dev --dry-run # payload size + files dropped +ducktests-remote deploy --only ignite-dev +``` + +Excludes are the same rsync-style patterns the source sync uses, and default to nothing. +Three sources, most specific first: `--exclude PATTERN`, a `.ducktests-deploy.ignore` file +at the root of one distribution, then `deploy.exclude`. That file is deliberately *not* +named `.ducktestsignore`: when `ignite-dev` links to a checkout, the distribution root and +the source root are the same directory, and the two lists are opposites — the source sync +drops `target`, `deploy` keeps little else. The manifest is built from the filtered list, +so an `already at ` skip always means the host holds what was actually sent. + +Symlinks *inside* a distribution are not followed: they are shipped as links and arrive +dangling. Only the top-level link is resolved. + ### Where the directory names come from `ignitetest` resolves a distribution home as `/`, where `product` diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py index f890d9e4c849a..0a16c6344556f 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py @@ -17,6 +17,7 @@ import json import os +import tarfile import time import pytest @@ -26,6 +27,7 @@ from ducktests_remote.commands import deploy, provision from ducktests_remote.config import DEFAULTS, ConfigError from ducktests_remote.fanout import CHANGED, FAILED, OK, HostResult, fanout, summarise +from ducktests_remote.transport import make_tarball def _dist(tmp_path, name="ignite-dev", body="binary"): @@ -230,3 +232,100 @@ def check_two_and_forty_nine_hosts_produce_the_same_shape(self): def check_empty_inventory_is_not_an_error(self): assert fanout([], lambda h: None) == [] + + +class CheckExcludes: + """``ignite-dev`` is normally a link to a checkout; only the built jars are wanted.""" + + @staticmethod + def _checkout(tmp_path): + """A tree shaped like an Ignite source root after a build.""" + root = tmp_path / "ignite-dev" + for rel in ("bin/ignite.sh", + "modules/core/src/main/java/Ignite.java", + "modules/core/target/classes/Ignite.class", + "modules/core/target/ignite-core.jar", + "modules/core/target/libs/dep.jar", + "modules/ducktests/tests/certs/truststore.jks"): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rel, encoding="utf-8") + return root + + @staticmethod + def _ctx(tmp_path, exclude=None, config_exclude=None): + class _Args: # pylint: disable=too-few-public-methods + pass + + class _Ctx: # pylint: disable=too-few-public-methods + pass + + ctx = _Ctx() + ctx.args = _Args() + ctx.args.exclude = list(exclude or []) + ctx.config = {"deploy": {**DEFAULTS["deploy"], "exclude": list(config_exclude or [])}} + ctx.dist_dir = tmp_path + return ctx + + def check_manifest_leaves_out_excluded_files(self, tmp_path): + root = self._checkout(tmp_path) + whole = deploy.build_manifest(root) + filtered = deploy.build_manifest(root, excludes=["src", "classes"]) + assert filtered["files"] == whole["files"] - 2 + assert filtered["excluded"] == 2 + assert filtered["bytes"] < whole["bytes"] + assert filtered["hash"] != whole["hash"] + + def check_the_tarball_holds_exactly_what_the_manifest_counted(self, tmp_path): + root = self._checkout(tmp_path) + excludes = ["src", "classes"] + archive = tmp_path / "payload.tar.gz" + make_tarball(root, archive, excludes=excludes) + with tarfile.open(archive) as tar: + names = sorted(m.name for m in tar.getmembers()) + assert names == ["bin/ignite.sh", + "modules/core/target/ignite-core.jar", + "modules/core/target/libs/dep.jar", + "modules/ducktests/tests/certs/truststore.jks"] + assert len(names) == deploy.build_manifest(root, excludes=excludes)["files"], \ + "a host called up to date must hold the same files the tarball carries" + + def check_no_excludes_ships_the_tree_whole(self, tmp_path): + root = self._checkout(tmp_path) + manifest = deploy.build_manifest(root) + assert manifest["excluded"] == 0 and manifest["files"] == 6 + + def check_command_line_beats_the_ignore_file_and_the_config(self, tmp_path): + root = self._checkout(tmp_path) + (root / deploy.IGNORE_NAME).write_text("classes\n", encoding="utf-8") + ctx = self._ctx(tmp_path, exclude=["src"], config_exclude=["target"]) + assert deploy.resolve_excludes(ctx, root) == ["src", deploy.IGNORE_NAME] + + def check_the_ignore_file_beats_the_config(self, tmp_path): + root = self._checkout(tmp_path) + (root / deploy.IGNORE_NAME).write_text( + "# only the jars are wanted on the workers\n" + "src\n" + "\n" + "classes\n", encoding="utf-8") + ctx = self._ctx(tmp_path, config_exclude=["target"]) + assert deploy.resolve_excludes(ctx, root) == ["src", "classes", deploy.IGNORE_NAME] + + def check_the_ignore_file_is_never_shipped(self, tmp_path): + root = self._checkout(tmp_path) + (root / deploy.IGNORE_NAME).write_text("src\n", encoding="utf-8") + ctx = self._ctx(tmp_path) + excludes = deploy.resolve_excludes(ctx, root) + archive = tmp_path / "payload.tar.gz" + make_tarball(root, archive, excludes=excludes) + with tarfile.open(archive) as tar: + assert deploy.IGNORE_NAME not in [m.name for m in tar.getmembers()] + + def check_the_config_applies_when_nothing_more_specific_exists(self, tmp_path): + root = self._checkout(tmp_path) + ctx = self._ctx(tmp_path, config_exclude=["src", "classes"]) + assert deploy.resolve_excludes(ctx, root) == ["src", "classes"] + + def check_the_default_is_no_filtering(self, tmp_path): + assert deploy.resolve_excludes(self._ctx(tmp_path), self._checkout(tmp_path)) == [] + assert DEFAULTS["deploy"]["exclude"] == [] diff --git a/modules/ducktests/tests/ducktests_remote/commands/deploy.py b/modules/ducktests/tests/ducktests_remote/commands/deploy.py index 91bf5dcfc0b81..4ef6053cb384d 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/deploy.py +++ b/modules/ducktests/tests/ducktests_remote/commands/deploy.py @@ -21,6 +21,11 @@ version-parsing logic. The operator names the directories to match what the tests expect, which is also what makes fork layouts work without special cases. +The one filter is ``--exclude``, which exists because ``ignite-dev`` is normally a link +to a source checkout: the workers need the built jars under ``modules/*/target``, and +nothing else in the tree. Excludes are opt-in and default to nothing, so a distribution +without them is still shipped byte for byte. + :func:`build_manifest`, :func:`prepare_script`, :func:`swap_script` and :func:`human` are public because ``provision``'s ``jdk`` step delivers a JDK the same way and must not grow a second copy of the staging-and-swap logic. @@ -39,10 +44,16 @@ from ducktests_remote.config import ConfigError, expand_path from ducktests_remote.fanout import (CHANGED, FAILED, HostResult, SKIPPED, any_failed, fanout, render_table, summarise) -from ducktests_remote.transport import ProxiedTransport, make_tarball +from ducktests_remote.transport import ProxiedTransport, is_excluded, make_tarball MANIFEST_NAME = ".ducktests-deploy.json" +# Per-distribution exclude list, read from the root of the distribution itself. It is +# deliberately NOT called .ducktestsignore: when ignite-dev links to a checkout, the +# distribution root and the source root are the same directory, and the two lists are +# opposites - the source sync drops `target`, deploy keeps only `target`. +IGNORE_NAME = ".ducktests-deploy.ignore" + def register(subparsers, common): """Wire up the ``deploy`` subcommand.""" @@ -54,6 +65,9 @@ def register(subparsers, common): help="directory holding one subdirectory per distribution") parser.add_argument("--only", action="append", default=[], metavar="NAME", help="restrict to this distribution; repeatable") + parser.add_argument("--exclude", action="append", default=[], metavar="PATTERN", + help="rsync-style pattern to leave out of every distribution; " + "repeatable. Overrides %s and deploy.exclude" % IGNORE_NAME) parser.add_argument("--install-root", metavar="PATH", help="target root on the workers") parser.add_argument("--via", metavar="HOST", help="upload once to HOST, then fan out from there") @@ -93,15 +107,20 @@ def execute(ctx): # pylint: disable=too-many-locals use_checksum = args.checksum or ctx.config["deploy"].get("checksum", False) plans = [] for name in dists: - manifest = build_manifest(dist_dir / name, checksum=use_checksum) - plans.append((name, manifest)) + excludes = resolve_excludes(ctx, dist_dir / name) + manifest = build_manifest(dist_dir / name, checksum=use_checksum, excludes=excludes) + if excludes and manifest["excluded"]: + console.info("%s: %d file(s) left out by %d pattern(s)" + % (name, manifest["excluded"], len(excludes))) + console.detail("excludes: %s" % ", ".join(excludes)) + plans.append((name, manifest, excludes)) _print_cost(ctx, plans, nodes) overall = [] - for name, manifest in plans: + for name, manifest, excludes in plans: console.heading("%s -> %s/%s" % (name, install_root, name)) - results = _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes) + results = _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes, excludes) overall.extend(results) console.out(render_table(results, verbose=console.verbose)) console.out(summarise(results)) @@ -125,28 +144,54 @@ def _distributions(dist_dir, only): return names -def build_manifest(path, *, checksum=False): +def resolve_excludes(ctx, dist_root): + """ + :return: the exclude patterns for one distribution, most specific source winning. + + ``--exclude`` beats a :data:`IGNORE_NAME` file at the root of the distribution, which + beats ``deploy.exclude`` in the configuration. The list is never merged across + sources: a pattern list is read as a whole, the way ``run``'s source-sync list is. + """ + ignore_file = Path(dist_root) / IGNORE_NAME + # The list file is bookkeeping, not part of the distribution; never ship it. + tail = [IGNORE_NAME] if ignore_file.is_file() else [] + if ctx.args.exclude: + return list(ctx.args.exclude) + tail + if ignore_file.is_file(): + return [line.strip() for line in ignore_file.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.strip().startswith("#")] + tail + return list(ctx.config["deploy"].get("exclude") or []) + + +def build_manifest(path, *, checksum=False, excludes=()): """ :return: a manifest describing ``path``, used to skip hosts that already match. Sorted relative paths plus sizes and mtimes by default; ``--checksum`` adds a content sha256 per file, which is exact but reads gigabytes off disk every time. + + ``excludes`` must be the same list the tarball is built with, or a host would be + called up to date while holding a different set of files. """ entries = [] total = 0 + excluded = 0 root = Path(path) for entry in sorted(root.rglob("*")): if entry.is_dir() or entry.is_symlink(): continue + rel = entry.relative_to(root) + if is_excluded(rel, excludes): + excluded += 1 + continue stat = entry.stat() - rel = entry.relative_to(root).as_posix() total += stat.st_size if checksum: - entries.append("%s\0%d\0%s" % (rel, stat.st_size, _sha256(entry))) + entries.append("%s\0%d\0%s" % (rel.as_posix(), stat.st_size, _sha256(entry))) else: - entries.append("%s\0%d\0%d" % (rel, stat.st_size, int(stat.st_mtime))) + entries.append("%s\0%d\0%d" % (rel.as_posix(), stat.st_size, int(stat.st_mtime))) digest = hashlib.sha256("\n".join(entries).encode("utf-8")).hexdigest() - return {"hash": digest, "files": len(entries), "bytes": total, + return {"hash": digest, "files": len(entries), "bytes": total, "excluded": excluded, "mode": "checksum" if checksum else "size+mtime"} @@ -159,7 +204,7 @@ def _sha256(path): def _print_cost(ctx, plans, nodes): - total = sum(m["bytes"] for _, m in plans) + total = sum(m["bytes"] for _, m, _ in plans) per_host = human(total) console = ctx.console console.info("%d distribution(s), %s each, %d host(s) = %s total" @@ -179,7 +224,7 @@ def human(size): return "%.1f TB" % value -def _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes): +def _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes, excludes=()): args = ctx.args target = posixpath.join(install_root, name) manifest_body = json.dumps({ @@ -197,7 +242,7 @@ def _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes): with tempfile.TemporaryDirectory() as tmp: archive = Path(tmp) / ("%s.tar.gz" % name) - make_tarball(Path(dist_dir) / name, archive) + make_tarball(Path(dist_dir) / name, archive, excludes=excludes) via_transport = None staged_on_via = None diff --git a/modules/ducktests/tests/ducktests_remote/config.py b/modules/ducktests/tests/ducktests_remote/config.py index 40058a6bf7c87..8770b569468bf 100644 --- a/modules/ducktests/tests/ducktests_remote/config.py +++ b/modules/ducktests/tests/ducktests_remote/config.py @@ -128,6 +128,10 @@ "owner": None, "staging_dir": "/tmp/ducktests-remote-staging", "checksum": False, + # rsync-style patterns left out of every distribution. Empty by default: a + # release directory is shipped whole. The list matters when ignite-dev is a link + # to a source checkout, where only the built jars are wanted on the workers. + "exclude": [], }, "clean": { # pgrep -f patterns; see IGNITE_MAIN_CLASSES above. diff --git a/modules/ducktests/tests/ducktests_remote/docs/commands.md b/modules/ducktests/tests/ducktests_remote/docs/commands.md index 7f708af8b58c8..1c27fcec9dc38 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/commands.md +++ b/modules/ducktests/tests/ducktests_remote/docs/commands.md @@ -224,8 +224,8 @@ assumption; a doctor FAIL makes it exit 2. ## `deploy` ``` -deploy [--dist-dir PATH] [--only NAME]... [--install-root PATH] [--via HOST] - [--sudo] [--owner USER] [--force] [--checksum] [-n N] [--json] +deploy [--dist-dir PATH] [--only NAME]... [--exclude PATTERN]... [--install-root PATH] + [--via HOST] [--sudo] [--owner USER] [--force] [--checksum] [-n N] [--json] ``` Each subdirectory of `--dist-dir` is copied verbatim to `/`. The name @@ -247,6 +247,82 @@ Per distribution, per host: `deploy` prints the total bytes before it starts — on a twelve-host cluster a 300 MB distribution is 3.7 GB from a laptop — and suggests `--via` when that total is large. +### Leaving files out + +Excludes are rsync-style patterns, matched against paths relative to the distribution +root: a pattern matches the whole relative path, a path prefix, or any single path +component. So `src` drops every `modules/*/src`, `*.jar` drops jars anywhere, and +`modules/indexing` drops that one subtree — but `target/libs` matches only a `target/libs` +directly at the root, not `modules/core/target/libs`. They default to nothing: a +distribution without them is shipped byte for byte. Three sources, most specific winning: + +| Source | Scope | +| --- | --- | +| `--exclude PATTERN` (repeatable) | every distribution in this invocation | +| `.ducktests-deploy.ignore` at the root of a distribution | that distribution | +| `deploy.exclude` in the configuration | every distribution | + +The list is read from one source as a whole; sources are never merged. The ignore file is +one pattern per line, `#` comments allowed, and is itself never shipped. + +It is **not** called `.ducktestsignore`: when `ignite-dev` links to a checkout, the +distribution root and the source root are the same directory, and the two lists are +opposites — the source sync drops `target`, `deploy` keeps almost nothing else. + +The manifest is built from the same filtered file list, so a host reported as up to date +holds exactly the files the tarball carried. Change the excludes and every host is +redeployed, as it should be. + +### `ignite-dev` from your own checkout + +`ignite-dev` is the distribution the tests resolve `DEV_BRANCH` to, and on a worker it +must have the layout of a *built source tree*, not of a release: `IgniteSpec` puts +`modules//target` and `modules//target/libs` on the classpath for every +module a test asks for (`ignitetest/services/utils/ignite_spec.py`), `path.py` runs +`bin/ignite.sh` from the same home and reads certificates from +`modules/ducktests/tests/certs`. Everything else in a checkout is ballast for a worker. + +So link the distribution to your checkout and let the excludes do the trimming: + +```bash +mkdir -p ~/dist +ln -sfn ~/Development/vanilla/ignite ~/dist/ignite-dev # relink any time +``` + +`deploy` follows that link: `is_dir()` accepts it as a distribution, the tree is walked +through it, and the workers receive ordinary files. Symlinks *inside* a distribution are +a different matter — they are stored as links and arrive dangling, so keep real files +below the top level. + +Then, in your configuration: + +```yaml +deploy: + dist_dir: ~/dist + exclude: [.git, .idea, src, docs, assembly, classes, test-classes, + generated-sources, generated-test-sources, maven-status, maven-archiver, + surefire-reports, javadoc, "*.tar.gz", "*.zip", __pycache__, "*.pyc"] +``` + +`src` as a pattern drops every `modules/*/src`, `classes` drops the exploded output Java +never reads off a classpath directory, and `target/*.jar` plus `target/libs/*.jar` +survive. The daily loop is then two commands: + +```bash +mvn package -pl :ignite-ducktests -am -DskipTests # in the checkout, however you build +ducktests-remote deploy --only ignite-dev +``` + +Check the damage before the first real transfer — `--dry-run` prints the payload size and +how many files the patterns dropped, and transfers nothing: + +```bash +ducktests-remote deploy --only ignite-dev --dry-run +``` + +Note that a distribution is all-or-nothing: rebuild one module and the whole distribution +is re-tarred and re-uploaded, because the manifest hash covers the tree. + ### Where the directory names come from ignitetest resolves a distribution home as `/`, where `product` is diff --git a/modules/ducktests/tests/ducktests_remote/docs/configuration.md b/modules/ducktests/tests/ducktests_remote/docs/configuration.md index 23c490354f3f7..c8cc5263ba4de 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/configuration.md +++ b/modules/ducktests/tests/ducktests_remote/docs/configuration.md @@ -195,6 +195,13 @@ source root replaces the built-in list; `--exclude` replaces both. | `owner` | `null` | `chown -R` the extracted tree | | `staging_dir` | `/tmp/ducktests-remote-staging` | where `--via` parks the payload | | `checksum` | `false` | hash file contents for the manifest instead of size+mtime | +| `exclude` | `[]` | rsync-style patterns left out of every distribution | + +`exclude` is empty by default, so a release directory is shipped whole. It earns its keep +when `ignite-dev` is a link to a source checkout, where only the built jars are wanted on +the workers; `--exclude` and a `.ducktests-deploy.ignore` file at the root of one +distribution override it, in that order. See +[commands.md](commands.md#ignite-dev-from-your-own-checkout) for the full recipe. ### `provision` diff --git a/modules/ducktests/tests/ducktests_remote/docs/troubleshooting.md b/modules/ducktests/tests/ducktests_remote/docs/troubleshooting.md index fb96b0d9f3bb4..a427c39580796 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/troubleshooting.md +++ b/modules/ducktests/tests/ducktests_remote/docs/troubleshooting.md @@ -71,6 +71,19 @@ the cause. `doctor` runs an N-way resolution probe from one worker. ducktests-remote provision --sudo --write-hosts # escape hatch when cluster DNS cannot be fixed ``` +### `deploy` wants to send a gigabyte of `ignite-dev` + +The distribution is a link to a checkout and nothing is being filtered. The workers need +`modules/*/target/*.jar`, `modules/*/target/libs/*.jar`, `bin/` and +`modules/ducktests/tests/certs`; `.git` and every `src` tree are ballast. Set +`deploy.exclude` (or `--exclude`, or a `.ducktests-deploy.ignore` at the root of the +distribution) and confirm with `--dry-run`, which prints the payload size and how many +files the patterns dropped. The recipe is in +[commands.md](commands.md#ignite-dev-from-your-own-checkout). + +Do not reach for `.ducktestsignore` here — that file is the *source sync* list and its +patterns are the opposite ones. + ### "source payload is N MB, above the limit" A build directory leaked into the sync. Distributions go through `deploy`, never through diff --git a/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml b/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml index 7b9e26ba57ec9..58c901881d872 100644 --- a/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml +++ b/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml @@ -86,6 +86,16 @@ java: bashrc: true # and a marked block at the top of ~/.bashrc deploy: + # One subdirectory per distribution. `ignite-dev` is usually a symlink to a checkout: + # ln -sfn ~/dev/ignite ./dist/ignite-dev dist_dir: ./dist + # Patterns left out of every distribution, rsync-style. Empty by default, so a release + # directory is shipped whole. The list below is for the symlinked-checkout case: the + # workers need modules/*/target/*.jar, modules/*/target/libs/*.jar, bin/ and the + # ducktests certs, and nothing else in the tree. + # exclude: [.git, .idea, src, docs, assembly, classes, test-classes, + # generated-sources, generated-test-sources, maven-status, maven-archiver, + # surefire-reports, javadoc, "*.tar.gz", "*.zip", __pycache__, "*.pyc"] + jobs: 16 From abc6ed1a70edf5127c1c2ea49ec05217e0cb483b Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 28 Jul 2026 20:42:53 +0300 Subject: [PATCH 6/9] Send only what changed when ducktests-remote deploy redeploys A distribution is all-or-nothing by design: rebuild one module and the manifest hash for the whole tree changes, so every host is redeployed. Until now that meant re-tarring and re-uploading everything, which for a linked checkout is a gigabyte per host to deliver one jar. Fill the staging directory with rsync instead, hardlinked with --link-dest against the deployment already on the host: unchanged files are never sent and cost no disk, and the tree is still built from scratch and still swapped in atomically, so an interrupted transfer cannot leave a live distribution half updated. Deleting the old tree afterwards only drops link counts. rsync is fed the exact file list on stdin rather than its own --exclude patterns, because its matching rules differ from is_excluded's and a distribution that differs from the manifest describing it is the one bug the manifest exists to prevent. The tarball path stays for --no-rsync, deploy.rsync: false, --via, a worker or coordinator without rsync, and a Windows coordinator, where rsync would read C:/dist/ignite-dev as a host named C. rsync is probed per host, so a mixed cluster falls back only where it must, and the tarball is now built lazily - when every host takes the fast path, nothing is compressed at all. --- .../tests/ducktests_remote/README.md | 19 +- .../checks/check_remote_deploy.py | 224 +++++++++++++++++- .../tests/ducktests_remote/commands/deploy.py | 209 ++++++++++++++-- .../tests/ducktests_remote/config.py | 3 + .../tests/ducktests_remote/docs/commands.md | 49 +++- .../ducktests_remote/docs/configuration.md | 6 + .../tests/ducktests_remote/docs/internals.md | 5 + .../ducktests_remote/examples/cluster.yaml | 4 + 8 files changed, 493 insertions(+), 26 deletions(-) diff --git a/modules/ducktests/tests/ducktests_remote/README.md b/modules/ducktests/tests/ducktests_remote/README.md index 20a78e2f68f26..db6fb755b05ed 100644 --- a/modules/ducktests/tests/ducktests_remote/README.md +++ b/modules/ducktests/tests/ducktests_remote/README.md @@ -240,8 +240,23 @@ ducktests-remote deploy --sudo --owner max # when /opt is root-owned Each host gets a `.ducktests-deploy.json` manifest (sorted paths + sizes + mtimes; `--checksum` hashes contents instead). Hosts whose manifest already matches are skipped -unless `--force`. Extraction goes to a temporary directory and is then swapped into -place, because a half-copied distribution that looks present is worse than an absent one. +unless `--force`. The staging directory is filled and then swapped into place, because a +half-copied distribution that looks present is worse than an absent one. + +By default the staging directory is filled by **rsync**, hardlinked with `--link-dest` +against the deployment already on the host, so a redeploy after rebuilding one module +sends one jar rather than the whole tree: + +``` +w01 changed 4.1s rsync: 12 of 8431 file(s) changed, 4.0 MB sent +``` + +rsync is fed the exact file list on stdin — never its own `--exclude` patterns, whose +matching differs from the manifest's. `deploy` falls back to a tarball of the whole +distribution when either end has no rsync (probed per host), with `--via`, with +`--no-rsync` or `deploy.rsync: false`, or on a Windows coordinator, where rsync would read +`C:/dist/ignite-dev` as a host named `C`. Nothing is tarred at all when every host takes +the rsync path. On a twelve-host cluster a 300 MB distribution is 3.7 GB over the wire from a laptop. `deploy` prints that total before it starts, and suggests `--via`. diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py index 0a16c6344556f..b070e384acd8b 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py @@ -23,11 +23,12 @@ import pytest from fake_transport import FakeTransport +from ducktests_remote.cluster import Node from ducktests_remote.commands import clean as clean_cmd from ducktests_remote.commands import deploy, provision from ducktests_remote.config import DEFAULTS, ConfigError from ducktests_remote.fanout import CHANGED, FAILED, OK, HostResult, fanout, summarise -from ducktests_remote.transport import make_tarball +from ducktests_remote.transport import Result, make_tarball def _dist(tmp_path, name="ignite-dev", body="binary"): @@ -329,3 +330,224 @@ def check_the_config_applies_when_nothing_more_specific_exists(self, tmp_path): def check_the_default_is_no_filtering(self, tmp_path): assert deploy.resolve_excludes(self._ctx(tmp_path), self._checkout(tmp_path)) == [] assert DEFAULTS["deploy"]["exclude"] == [] + + +class _RsyncTransport(FakeTransport): + """A worker whose rsync is present, with the ssh details rsync needs.""" + + rsync = True + + @property + def target(self): + return "tester@w1" + + def ssh_options(self, *, for_scp=False): # pylint: disable=unused-argument + return ["-o", "BatchMode=yes", "-i", "/home/tester/.ssh/id_ed25519"] + + def has_rsync(self): + """:return: whether this worker is on the incremental path.""" + return self.rsync + + +class CheckRsyncFastPath: + """Rebuild one module, send one jar.""" + + STATS = ("Number of files: 1,234 (reg: 1,200, dir: 34)\n" + "Number of regular files transferred: 12\n" + "Total file size: 524,288,000 bytes\n" + "Total transferred file size: 4,194,304 bytes\n") + + @staticmethod + def _args(**kw): + class _Args: # pylint: disable=too-few-public-methods + force = False + sudo = False + owner = None + checksum = False + no_rsync = False + via = None + + args = _Args() + for key, value in kw.items(): + setattr(args, key, value) + return args + + @classmethod + def _ctx(cls, transport, **kw): + class _Console: # pylint: disable=too-few-public-methods + verbose = False + + @staticmethod + def detail(message): + """Swallow the traced command line.""" + + class _Ctx: # pylint: disable=too-few-public-methods + pass + + ctx = _Ctx() + ctx.args = cls._args(**kw) + ctx.config = {"deploy": dict(DEFAULTS["deploy"])} + ctx.console = _Console() + ctx.dry_run = False + ctx.worker = lambda node: transport + return ctx + + @staticmethod + def _dist_and_payload(tmp_path): + root = _dist(tmp_path / "d") + return root, deploy._Payload(root, tmp_path / "p.tar.gz", ()) # noqa: SLF001 + + # -- the command line -------------------------------------------------------- + + def check_the_file_list_comes_from_stdin_not_from_rsync_patterns(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1") + assert "--files-from=-" in argv + assert not [a for a in argv if a.startswith("--exclude")], \ + "rsync pattern matching differs from is_excluded; the exact list is sent instead" + + def check_unchanged_files_are_hardlinked_against_the_live_distribution(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1", + link_dest="/opt/ignite-dev") + assert "--link-dest=/opt/ignite-dev" in argv + + def check_a_first_deployment_has_nothing_to_link_against(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1") + assert not [a for a in argv if a.startswith("--link-dest")] + + def check_it_lands_in_the_staging_directory_never_in_the_target(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1", + link_dest="/opt/ignite-dev") + assert argv[-1] == "tester@w1:/opt/.tmp.1/", \ + "an interrupted transfer must not leave a live distribution half updated" + assert argv[-2] == "/dist/ignite-dev/" + + def check_ssh_options_reach_rsync(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/x", "/opt/.tmp.1") + assert argv[argv.index("-e") + 1] == \ + "ssh -o BatchMode=yes -i /home/tester/.ssh/id_ed25519" + + def check_sudo_and_checksum_are_passed_through(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/x", "/opt/.tmp.1", + checksum=True, sudo=True) + assert "--checksum" in argv and "--rsync-path=sudo -n rsync" in argv + + # -- stats ------------------------------------------------------------------- + + def check_stats_are_read_back(self): + assert deploy.parse_rsync_stats(self.STATS) == (12, 4194304) + + def check_rsync_2_x_wording_is_understood(self): + assert deploy.parse_rsync_stats("Number of files transferred: 3\n" + "Total transferred file size: 1024 bytes\n") == (3, 1024) + + def check_unreadable_stats_are_not_guessed(self): + assert deploy.parse_rsync_stats("") is None + assert deploy.parse_rsync_stats("all done") is None + + # -- when it is used --------------------------------------------------------- + + @staticmethod + def _posix_with_rsync(monkeypatch): + monkeypatch.setattr(deploy.shutil, "which", lambda _: "/usr/bin/rsync") + monkeypatch.setattr(deploy.os, "name", "posix") + + def check_it_is_on_by_default_when_rsync_is_installed(self, monkeypatch): + self._posix_with_rsync(monkeypatch) + assert deploy.rsync_enabled(self._ctx(None)) is True + + def check_no_rsync_flag_and_config_both_turn_it_off(self, monkeypatch): + self._posix_with_rsync(monkeypatch) + assert deploy.rsync_enabled(self._ctx(None, no_rsync=True)) is False + ctx = self._ctx(None) + ctx.config["deploy"]["rsync"] = False + assert deploy.rsync_enabled(ctx) is False + + def check_via_keeps_the_single_upload(self, monkeypatch): + self._posix_with_rsync(monkeypatch) + assert deploy.rsync_enabled(self._ctx(None, via="build-vm-01")) is False, \ + "--via exists so the payload crosses the slow link once, as one file" + + def check_a_coordinator_without_rsync_falls_back(self, monkeypatch): + monkeypatch.setattr(deploy.os, "name", "posix") + monkeypatch.setattr(deploy.shutil, "which", lambda _: None) + assert deploy.rsync_enabled(self._ctx(None)) is False + + def check_a_windows_coordinator_falls_back(self, monkeypatch): + monkeypatch.setattr(deploy.shutil, "which", lambda _: "rsync.exe") + monkeypatch.setattr(deploy.os, "name", "nt") + assert deploy.rsync_enabled(self._ctx(None)) is False, \ + "rsync would read C:/dist/ignite-dev as host C" + + # -- the transfer ------------------------------------------------------------ + + def check_the_tarball_is_never_built_when_every_host_takes_rsync(self, tmp_path, + monkeypatch): + root, payload = self._dist_and_payload(tmp_path) + transport = _RsyncTransport() + calls = [] + + def fake_run_local(argv, **kw): + calls.append((argv, kw.get("input"))) + return Result(argv, 0, self.STATS, "", "local") + + monkeypatch.setattr(deploy, "run_local", fake_run_local) + manifest = deploy.build_manifest(root) + result = deploy._deploy_to_host( # noqa: SLF001 + self._ctx(transport), Node(host="w1", user="tester"), "ignite-dev", + "/opt/ignite-dev", manifest, "{}", payload, None, None, + "\n".join(deploy.included_files(root)) + "\n") + + assert not (tmp_path / "p.tar.gz").exists(), \ + "compressing a linked checkout would cost more than the transfer saves" + assert len(calls) == 1 + assert calls[0][1] == "bin/ignite.sh\nlibs/core.jar\n" + assert result.status == CHANGED and "rsync: 12 of 2 file(s) changed" in result.message + + def check_the_staging_tree_is_still_swapped_into_place(self, tmp_path, monkeypatch): + root, payload = self._dist_and_payload(tmp_path) + transport = _RsyncTransport() + monkeypatch.setattr(deploy, "run_local", + lambda argv, **kw: Result(argv, 0, self.STATS, "", "local")) + deploy._deploy_to_host( # noqa: SLF001 + self._ctx(transport), Node(host="w1", user="tester"), "ignite-dev", + "/opt/ignite-dev", deploy.build_manifest(root), "{}", payload, None, None, + "bin/ignite.sh\n") + scripts = "\n".join(transport.scripts) + assert 'mv -- "$staging" "$target"' in scripts + assert deploy.MANIFEST_NAME in "".join(transport.files) + + def check_a_worker_without_rsync_gets_the_tarball(self, tmp_path, monkeypatch): + root, payload = self._dist_and_payload(tmp_path) + transport = _RsyncTransport() + transport.rsync = False + monkeypatch.setattr(deploy, "run_local", + lambda argv, **kw: pytest.fail("rsync must not run here")) + result = deploy._deploy_to_host( # noqa: SLF001 + self._ctx(transport), Node(host="w1", user="tester"), "ignite-dev", + "/opt/ignite-dev", deploy.build_manifest(root), "{}", payload, None, None, + "bin/ignite.sh\n") + assert (tmp_path / "p.tar.gz").exists() + assert transport.uploads and result.status == CHANGED + + def check_a_failed_rsync_leaves_the_target_alone(self, tmp_path, monkeypatch): + root, payload = self._dist_and_payload(tmp_path) + transport = _RsyncTransport() + monkeypatch.setattr(deploy, "run_local", + lambda argv, **kw: Result(argv, 23, "", "permission denied", + "local")) + result = deploy._deploy_to_host( # noqa: SLF001 + self._ctx(transport), Node(host="w1", user="tester"), "ignite-dev", + "/opt/ignite-dev", deploy.build_manifest(root), "{}", payload, None, None, + "bin/ignite.sh\n") + assert result.status == FAILED and "permission denied" in result.detail + assert 'mv -- "$staging" "$target"' not in "\n".join(transport.scripts) + + def check_the_file_list_matches_the_manifest(self, tmp_path): + root = CheckExcludes._checkout(tmp_path) # noqa: SLF001 + excludes = ["src", "classes"] + files = deploy.included_files(root, excludes) + assert files == ["bin/ignite.sh", + "modules/core/target/ignite-core.jar", + "modules/core/target/libs/dep.jar", + "modules/ducktests/tests/certs/truststore.jks"] + assert len(files) == deploy.build_manifest(root, excludes=excludes)["files"] diff --git a/modules/ducktests/tests/ducktests_remote/commands/deploy.py b/modules/ducktests/tests/ducktests_remote/commands/deploy.py index 4ef6053cb384d..9377465b0d6d3 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/deploy.py +++ b/modules/ducktests/tests/ducktests_remote/commands/deploy.py @@ -26,6 +26,14 @@ nothing else in the tree. Excludes are opt-in and default to nothing, so a distribution without them is still shipped byte for byte. +Two ways to move the bytes. The default is rsync into a staging directory hardlinked +against the previous deployment (``--link-dest``), so a rebuild of one module sends one +jar rather than the whole tree. The fallback, used when either end has no rsync or when +``--via`` stages the payload on an intermediate host, is a tarball of the whole +distribution. Both end in the same atomic swap, and both carry exactly the file list the +manifest was built from - rsync is fed that list on stdin rather than its own ``--exclude`` +patterns, whose matching rules differ subtly from :func:`is_excluded`'s. + :func:`build_manifest`, :func:`prepare_script`, :func:`swap_script` and :func:`human` are public because ``provision``'s ``jdk`` step delivers a JDK the same way and must not grow a second copy of the staging-and-swap logic. @@ -35,8 +43,11 @@ import json import os import posixpath +import re import shlex +import shutil import tempfile +import threading import uuid from pathlib import Path @@ -44,7 +55,8 @@ from ducktests_remote.config import ConfigError, expand_path from ducktests_remote.fanout import (CHANGED, FAILED, HostResult, SKIPPED, any_failed, fanout, render_table, summarise) -from ducktests_remote.transport import ProxiedTransport, is_excluded, make_tarball +from ducktests_remote.transport import (ProxiedTransport, is_excluded, make_tarball, + run_local) MANIFEST_NAME = ".ducktests-deploy.json" @@ -54,6 +66,11 @@ # opposites - the source sync drops `target`, deploy keeps only `target`. IGNORE_NAME = ".ducktests-deploy.ignore" +# `Number of regular files transferred: 12` on rsync 3.x, `Number of files transferred` +# on 2.x. Thousands separators are locale-dependent; the labels are not translated. +_RSYNC_TRANSFERRED = re.compile(r"Number of (?:regular )?files transferred:\s*([\d,.]+)") +_RSYNC_SENT = re.compile(r"Total transferred file size:\s*([\d,.]+)") + def register(subparsers, common): """Wire up the ``deploy`` subcommand.""" @@ -78,6 +95,8 @@ def register(subparsers, common): help="redeploy even when the manifest already matches") parser.add_argument("--checksum", action="store_true", help="hash file contents for the manifest instead of size+mtime") + parser.add_argument("--no-rsync", action="store_true", + help="send a full tarball instead of an incremental rsync") parser.add_argument("-n", "--num-nodes", type=int, default=None, help="only deploy to the first N inventory hosts") parser.add_argument("--json", action="store_true", help="machine-readable output") @@ -115,6 +134,10 @@ def execute(ctx): # pylint: disable=too-many-locals console.detail("excludes: %s" % ", ".join(excludes)) plans.append((name, manifest, excludes)) + if (not rsync_enabled(ctx) and not args.via and not args.no_rsync + and ctx.config["deploy"].get("rsync", True)): + console.detail("no usable rsync on this machine; sending whole tarballs") + _print_cost(ctx, plans, nodes) overall = [] @@ -173,17 +196,10 @@ def build_manifest(path, *, checksum=False, excludes=()): ``excludes`` must be the same list the tarball is built with, or a host would be called up to date while holding a different set of files. """ + included, excluded = _scan(path, excludes) entries = [] total = 0 - excluded = 0 - root = Path(path) - for entry in sorted(root.rglob("*")): - if entry.is_dir() or entry.is_symlink(): - continue - rel = entry.relative_to(root) - if is_excluded(rel, excludes): - excluded += 1 - continue + for entry, rel in included: stat = entry.stat() total += stat.st_size if checksum: @@ -195,6 +211,34 @@ def build_manifest(path, *, checksum=False, excludes=()): "mode": "checksum" if checksum else "size+mtime"} +def included_files(path, excludes=()): + """ + :return: the relative posix paths a deploy would carry, sorted. + + This is what rsync is fed on stdin. It comes from the same walk as the manifest on + purpose: rsync's own ``--exclude`` matching is close to :func:`is_excluded` but not + identical, and a distribution that differs from the manifest describing it is exactly + the bug the manifest exists to prevent. + """ + return [rel.as_posix() for _, rel in _scan(path, excludes)[0]] + + +def _scan(path, excludes): + """:return: ``([(file, relative_path)], excluded_count)`` for one distribution.""" + included = [] + excluded = 0 + root = Path(path) + for entry in sorted(root.rglob("*")): + if entry.is_dir() or entry.is_symlink(): + continue + rel = entry.relative_to(root) + if is_excluded(rel, excludes): + excluded += 1 + continue + included.append((entry, rel)) + return included, excluded + + def _sha256(path): digest = hashlib.sha256() with open(path, "rb") as handle: @@ -209,6 +253,9 @@ def _print_cost(ctx, plans, nodes): console = ctx.console console.info("%d distribution(s), %s each, %d host(s) = %s total" % (len(plans), per_host, len(nodes), human(total * len(nodes)))) + if rsync_enabled(ctx): + console.info("rsync: only what differs from each host is sent") + return if not ctx.args.via and len(nodes) > 3 and total > 200 * 1024 * 1024: console.warn("that is %s over the wire from this machine. `--via ` uploads it once and fans out from there." @@ -236,13 +283,13 @@ def _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes, excludes=()) }, indent=2, sort_keys=True) if ctx.dry_run: - return [HostResult(node.host, SKIPPED, - "would send %s to %s" % (human(manifest["bytes"]), target)) + how = "rsync" if rsync_enabled(ctx) else "tarball" + return [HostResult(node.host, SKIPPED, "would send %s to %s (%s)" + % (human(manifest["bytes"]), target, how)) for node in nodes] with tempfile.TemporaryDirectory() as tmp: - archive = Path(tmp) / ("%s.tar.gz" % name) - make_tarball(Path(dist_dir) / name, archive, excludes=excludes) + payload = _Payload(Path(dist_dir) / name, Path(tmp) / ("%s.tar.gz" % name), excludes) via_transport = None staged_on_via = None @@ -253,11 +300,16 @@ def _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes, excludes=()) staged_on_via = posixpath.join(staged_dir, "%s-%s.tar.gz" % (name, uuid.uuid4().hex[:8])) ctx.console.info("staging %s on %s" % (name, args.via)) - via_transport.upload(archive, staged_on_via) + via_transport.upload(payload.archive(), staged_on_via) - def operation(node, _archive=archive, _staged=staged_on_via, _via=via_transport): + file_list = None + if rsync_enabled(ctx): + file_list = "\n".join(included_files(Path(dist_dir) / name, excludes)) + "\n" + + def operation(node, _payload=payload, _staged=staged_on_via, _via=via_transport, + _files=file_list): return _deploy_to_host(ctx, node, name, target, manifest, manifest_body, - _archive, _staged, _via) + _payload, _staged, _via, _files) try: return fanout(nodes, operation, jobs=ctx.jobs, @@ -277,8 +329,86 @@ def _via_node(ctx, host): identity_file=ctx.cluster_cfg.get("identity_file")) -def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, archive, - staged_on_via, via_transport): +class _Payload: + """ + The tarball for one distribution, built at most once and only if a host needs it. + + With every host on the rsync path, a distribution is never tarred at all - which for + a linked checkout is a gigabyte of compression that would buy nothing. + """ + + def __init__(self, root, archive_path, excludes): + self.root = root + self._archive = archive_path + self._excludes = excludes + self._built = False + self._lock = threading.Lock() + + def archive(self): + """:return: the path to the tarball, building it on first use.""" + with self._lock: + if not self._built: + make_tarball(self.root, self._archive, excludes=self._excludes) + self._built = True + return self._archive + + +def rsync_enabled(ctx): + """ + :return: True when the incremental path may be used for this invocation. + + ``--via`` is deliberately excluded: its whole point is that the payload crosses the + slow link once, as one file, which is the opposite of a per-host rsync. + """ + if getattr(ctx.args, "no_rsync", False) or getattr(ctx.args, "via", None): + return False + if not ctx.config["deploy"].get("rsync", True): + return False + if os.name == "nt": + # rsync reads `C:/dist/ignite-dev` as host `C`, and a coordinator that hands it a + # Windows path silently deploys nothing. The tarball path has no such problem. + return False + return shutil.which("rsync") is not None + + +def rsync_argv(transport, local_root, staging, *, link_dest=None, checksum=False, sudo=False): + """ + :return: the rsync command line that fills ``staging`` on one worker. + + ``--link-dest`` is what makes a redeploy cheap on both sides: unchanged files become + hardlinks to the deployment already on the host and are never sent, so a rebuild of + one module costs one jar. The staging directory is still built from scratch and + still swapped in atomically, so an interrupted transfer cannot leave a live + distribution half updated. + """ + argv = ["rsync", "-a", "--stats", "--files-from=-", + "-e", shlex.join(["ssh"] + transport.ssh_options())] + if checksum: + argv.append("--checksum") + if link_dest: + argv.append("--link-dest=%s" % link_dest) + if sudo: + argv.append("--rsync-path=sudo -n rsync") + argv += ["%s/" % str(local_root).rstrip("/\\"), + "%s:%s/" % (transport.target, staging)] + return argv + + +def parse_rsync_stats(text): + """:return: ``(files_transferred, bytes_transferred)`` from ``--stats``, or None.""" + transferred = _RSYNC_TRANSFERRED.search(text or "") + sent = _RSYNC_SENT.search(text or "") + if not transferred or not sent: + return None + try: + return (int(re.sub(r"[,.]", "", transferred.group(1))), + int(re.sub(r"[,.]", "", sent.group(1)))) + except ValueError: + return None + + +def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, payload, + staged_on_via, via_transport, file_list=None): transport = ctx.worker(node) remote_manifest = posixpath.join(target, MANIFEST_NAME) @@ -300,6 +430,8 @@ def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, archive, % (install_root, node.user or "this account")) staging = "%s/.%s.tmp.%s" % (install_root, name, uuid.uuid4().hex[:8]) + used_rsync = False + stats = None if via_transport is not None: proxied = ProxiedTransport(name=node.host, via=via_transport, user=node.user, @@ -309,10 +441,16 @@ def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, archive, dry_run=ctx.dry_run, verbose=ctx.console.verbose) proxied.run_script(prepare_script(staging, ctx.args.sudo)).check() proxied.push_archive(staged_on_via, staging) + elif file_list is not None and getattr(transport, "has_rsync", _no_rsync)(): + outcome = _rsync_to_host(ctx, node, transport, payload.root, staging, target, + file_list) + if isinstance(outcome, HostResult): + return outcome + used_rsync, stats = True, outcome else: transport.run_script(prepare_script(staging, ctx.args.sudo)).check() remote_archive = "%s/.payload.tar.gz" % staging - transport.upload(archive, remote_archive) + transport.upload(payload.archive(), remote_archive) transport.run_script( "set -eu\ntar -xzf %s -C %s\nrm -f -- %s\n" % (shlex.quote(remote_archive), shlex.quote(staging), @@ -320,10 +458,41 @@ def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, archive, transport.write_file(manifest_body, posixpath.join(staging, MANIFEST_NAME)) transport.run_script(swap_script(staging, target, ctx.args.sudo, ctx.args.owner)).check() + if used_rsync: + if stats: + return HostResult(node.host, CHANGED, "rsync: %d of %s file(s) changed, %s sent" + % (stats[0], manifest["files"], human(stats[1]))) + return HostResult(node.host, CHANGED, "rsync: %s file(s)" % manifest["files"]) return HostResult(node.host, CHANGED, "%s files, %s" % (manifest["files"], human(manifest["bytes"]))) +def _no_rsync(): + """``has_rsync`` stand-in for transports that have no such notion (local copies).""" + return False + + +def _rsync_to_host(ctx, node, transport, local_root, staging, target, file_list): + """ + :return: ``(files, bytes)`` transferred, ``None`` when ``--stats`` could not be read, + or a failed :class:`HostResult`. + + The staging directory is created before rsync runs, and hardlinked against the live + distribution when there is one to link against. + """ + transport.run_script(prepare_script(staging, ctx.args.sudo)).check() + argv = rsync_argv(transport, local_root, staging, + link_dest=target if transport.exists(target) else None, + checksum=ctx.args.checksum or ctx.config["deploy"].get("checksum", False), + sudo=ctx.args.sudo) + ctx.console.detail("%s: %s" % (node.host, shlex.join(argv))) + result = run_local(argv, input=file_list) + if not result.ok: + return HostResult(node.host, FAILED, "rsync failed", + detail=(result.stderr or result.stdout).strip()) + return parse_rsync_stats(result.stdout) + + def prepare_script(staging, use_sudo): sudo = "sudo -n " if use_sudo else "" return "set -eu\n%(sudo)srm -rf -- %(staging)s\n%(sudo)smkdir -p %(staging)s\n" % { diff --git a/modules/ducktests/tests/ducktests_remote/config.py b/modules/ducktests/tests/ducktests_remote/config.py index 8770b569468bf..b0a0aedbbccf2 100644 --- a/modules/ducktests/tests/ducktests_remote/config.py +++ b/modules/ducktests/tests/ducktests_remote/config.py @@ -128,6 +128,9 @@ "owner": None, "staging_dir": "/tmp/ducktests-remote-staging", "checksum": False, + # Send only what differs, hardlinking the rest against the deployment already on + # the host. Falls back to a whole tarball when either end has no rsync. + "rsync": True, # rsync-style patterns left out of every distribution. Empty by default: a # release directory is shipped whole. The list matters when ignite-dev is a link # to a source checkout, where only the built jars are wanted on the workers. diff --git a/modules/ducktests/tests/ducktests_remote/docs/commands.md b/modules/ducktests/tests/ducktests_remote/docs/commands.md index 1c27fcec9dc38..c9e5a6eaaba8a 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/commands.md +++ b/modules/ducktests/tests/ducktests_remote/docs/commands.md @@ -225,7 +225,8 @@ assumption; a doctor FAIL makes it exit 2. ``` deploy [--dist-dir PATH] [--only NAME]... [--exclude PATTERN]... [--install-root PATH] - [--via HOST] [--sudo] [--owner USER] [--force] [--checksum] [-n N] [--json] + [--via HOST] [--sudo] [--owner USER] [--force] [--checksum] [--no-rsync] + [-n N] [--json] ``` Each subdirectory of `--dist-dir` is copied verbatim to `/`. The name @@ -247,6 +248,48 @@ Per distribution, per host: `deploy` prints the total bytes before it starts — on a twelve-host cluster a 300 MB distribution is 3.7 GB from a laptop — and suggests `--via` when that total is large. +### Incremental redeploys + +Step 4 has two implementations, and the default is the incremental one: + +``` +rsync -a --files-from=- --link-dest=/opt/ignite-dev ./dist/ignite-dev/ host:/opt/.ignite-dev.tmp.ab12/ +``` + +The staging directory is filled by rsync rather than by extracting a tarball. +`--link-dest` points at the deployment already on the host, so every file that has not +changed becomes a **hardlink** to the one already there and is never sent: rebuild one +module and the transfer is one jar, on a tree of any size. The staging directory is still +built from scratch and still swapped in atomically, so this costs nothing in safety; +deleting the old tree afterwards only drops link counts. + +Two details worth knowing: + +- rsync is fed the exact file list on **stdin**, not `--exclude` patterns. Its matching + rules are close to `is_excluded`'s but not identical, and a distribution that differs + from the manifest describing it is the one bug the manifest exists to prevent. +- Nothing is tarred when every host takes this path — the tarball is built lazily, only + for hosts that need it. + +The result line reports what actually moved: + +``` +w01 changed 4.1s rsync: 12 of 8431 file(s) changed, 4.0 MB sent +``` + +The tarball path is used instead when: + +| Condition | Why | +| --- | --- | +| `--no-rsync`, or `deploy.rsync: false` | you asked for it | +| `--via HOST` | the point of `--via` is one upload across the slow link, not one per host | +| no `rsync` on the coordinator, or on that worker | probed per host, so a mixed cluster works | +| the coordinator runs Windows | rsync reads `C:/dist/ignite-dev` as host `C` | + +`provision` installs `rsync` on the workers as part of the Dockerfile-derived package +list, so a provisioned cluster is already on the fast path. `--checksum` is passed +through to rsync, matching the manifest mode. + ### Leaving files out Excludes are rsync-style patterns, matched against paths relative to the distribution @@ -320,8 +363,8 @@ how many files the patterns dropped, and transfers nothing: ducktests-remote deploy --only ignite-dev --dry-run ``` -Note that a distribution is all-or-nothing: rebuild one module and the whole distribution -is re-tarred and re-uploaded, because the manifest hash covers the tree. +Rebuilding one module changes the manifest hash for the whole distribution, so every host +is redeployed — but on the rsync path that redeploy sends only the jars that changed. ### Where the directory names come from diff --git a/modules/ducktests/tests/ducktests_remote/docs/configuration.md b/modules/ducktests/tests/ducktests_remote/docs/configuration.md index c8cc5263ba4de..a51252c1e13d7 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/configuration.md +++ b/modules/ducktests/tests/ducktests_remote/docs/configuration.md @@ -196,6 +196,7 @@ source root replaces the built-in list; `--exclude` replaces both. | `staging_dir` | `/tmp/ducktests-remote-staging` | where `--via` parks the payload | | `checksum` | `false` | hash file contents for the manifest instead of size+mtime | | `exclude` | `[]` | rsync-style patterns left out of every distribution | +| `rsync` | `true` | send only what differs, hardlinking the rest against the deployment already on the host | `exclude` is empty by default, so a release directory is shipped whole. It earns its keep when `ignite-dev` is a link to a source checkout, where only the built jars are wanted on @@ -203,6 +204,11 @@ the workers; `--exclude` and a `.ducktests-deploy.ignore` file at the root of on distribution override it, in that order. See [commands.md](commands.md#ignite-dev-from-your-own-checkout) for the full recipe. +`rsync` is the default transfer path: a redeploy sends only the files that changed and +hardlinks the rest against what the host already has. `deploy` falls back to a whole +tarball when either end has no rsync, when `--via` is used, or on a Windows coordinator — +see [commands.md](commands.md#incremental-redeploys). + ### `provision` | Key | Default | Meaning | diff --git a/modules/ducktests/tests/ducktests_remote/docs/internals.md b/modules/ducktests/tests/ducktests_remote/docs/internals.md index 5e30e95050bd5..f346dccfccefb 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/internals.md +++ b/modules/ducktests/tests/ducktests_remote/docs/internals.md @@ -87,6 +87,11 @@ Design decisions that are load-bearing: - **`expand`** resolves a leading `~` against the *remote* home, once per connection, because paths are shell-quoted before they reach the remote side and a literal tilde would never be expanded there. +- **`has_rsync()` is probed once per transport and cached.** Both the source sync and + `deploy`'s incremental path ask for it; a mixed cluster where one host lacks rsync falls + back per host rather than for the whole run. `deploy` does not route rsync through the + transport, though — it runs rsync on the coordinator with the transport's own + `ssh_options()` in `-e`, because the payload never passes through a shell. ## Fan-out diff --git a/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml b/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml index 58c901881d872..c7508f95fa1dd 100644 --- a/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml +++ b/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml @@ -98,4 +98,8 @@ deploy: # generated-sources, generated-test-sources, maven-status, maven-archiver, # surefire-reports, javadoc, "*.tar.gz", "*.zip", __pycache__, "*.pyc"] + # Send only what differs from the deployment already on each host, hardlinking the + # rest. Falls back to a whole tarball when either end has no rsync, or with --via. + rsync: true + jobs: 16 From 6e84ef66891ed16d18addef76c2dc59ba5c43c48 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 28 Jul 2026 20:59:42 +0300 Subject: [PATCH 7/9] Show per-host progress while ducktests-remote deploy transfers Sending 3.5 GB to twelve machines is a normal deploy and takes minutes. A terminal that prints nothing for that long is indistinguishable from one that has hung, and the operator cannot tell a slow host from a stuck one. Add progress.py: a display written to from the fan-out threads and drawn by a daemon thread of its own, so a slow terminal cannot slow a transfer and an update never blocks on a redraw. On a terminal it redraws a block, one row per host plus a total; in a log, under --verbose, or wherever stderr is not a terminal it prints one aggregate line every 15 seconds. --quiet, --dry-run and --no-progress swap in a NullProgress with the same surface, so no caller has to ask whether the display is on. The total is the mean of the per-host fractions rather than bytes against a predicted total: rsync sends only what differs, so a denominator of distribution size times host count would stall at 12% and finish there. Byte counts come from --info=progress2 on the rsync path. On the tarball path they come from a new upload_watched, which streams the file into ssh 'cat > path' and counts the chunks, because scp's own meter is written for a terminal and suppressed whenever its output is a pipe - which it always is here. scp stays the default for every unwatched upload. rsync now takes its file list as a file rather than on stdin: the output side is read as it arrives, and one thread pumping both pipes of a process deadlocks as soon as either fills. --- .../tests/ducktests_remote/README.md | 12 +- .../checks/check_remote_deploy.py | 48 ++- .../checks/check_remote_progress.py | 283 ++++++++++++++ .../ducktests/tests/ducktests_remote/cli.py | 2 + .../tests/ducktests_remote/commands/deploy.py | 116 +++++- .../tests/ducktests_remote/docs/commands.md | 36 ++ .../tests/ducktests_remote/docs/internals.md | 26 +- .../tests/ducktests_remote/progress.py | 353 ++++++++++++++++++ .../tests/ducktests_remote/transport.py | 134 +++++++ 9 files changed, 976 insertions(+), 34 deletions(-) create mode 100644 modules/ducktests/tests/ducktests_remote/checks/check_remote_progress.py create mode 100644 modules/ducktests/tests/ducktests_remote/progress.py diff --git a/modules/ducktests/tests/ducktests_remote/README.md b/modules/ducktests/tests/ducktests_remote/README.md index db6fb755b05ed..80e75ab75c4b0 100644 --- a/modules/ducktests/tests/ducktests_remote/README.md +++ b/modules/ducktests/tests/ducktests_remote/README.md @@ -251,7 +251,17 @@ sends one jar rather than the whole tree: w01 changed 4.1s rsync: 12 of 8431 file(s) changed, 4.0 MB sent ``` -rsync is fed the exact file list on stdin — never its own `--exclude` patterns, whose +A transfer that takes minutes shows what every host is doing, redrawn in place on a +terminal and reduced to one `total ...` line every 15 seconds in a log, under `--verbose`, +or wherever stderr is not a terminal (`--no-progress` turns it off): + +``` + worker01 █████████████████░░░░░░░ 70% 210.0 MB / 300.0 MB 24.1 MB/s + worker02 ████████░░░░░░░░░░░░░░░░ 32% 96.0 MB / 300.0 MB 18.7 MB/s + total ████████████░░░░░░░░░░░░ 50% 1/4 host(s) 306.0 MB sent 1:12 +``` + +rsync is fed the exact file list as a file — never its own `--exclude` patterns, whose matching differs from the manifest's. `deploy` falls back to a tarball of the whole distribution when either end has no rsync (probed per host), with `--via`, with `--no-rsync` or `deploy.rsync: false`, or on a Windows coordinator, where rsync would read diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py index b070e384acd8b..aae3c18fd301a 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py @@ -399,38 +399,49 @@ def _dist_and_payload(tmp_path): # -- the command line -------------------------------------------------------- - def check_the_file_list_comes_from_stdin_not_from_rsync_patterns(self): - argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1") - assert "--files-from=-" in argv + def check_an_exact_file_list_is_sent_not_rsync_patterns(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1", + files_from="/tmp/ignite-dev.files") + assert "--files-from=/tmp/ignite-dev.files" in argv assert not [a for a in argv if a.startswith("--exclude")], \ "rsync pattern matching differs from is_excluded; the exact list is sent instead" def check_unchanged_files_are_hardlinked_against_the_live_distribution(self): argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1", - link_dest="/opt/ignite-dev") + files_from="/tmp/x.files", link_dest="/opt/ignite-dev") assert "--link-dest=/opt/ignite-dev" in argv def check_a_first_deployment_has_nothing_to_link_against(self): - argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1") + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1", + files_from="/tmp/x.files") assert not [a for a in argv if a.startswith("--link-dest")] def check_it_lands_in_the_staging_directory_never_in_the_target(self): argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1", - link_dest="/opt/ignite-dev") + files_from="/tmp/x.files", link_dest="/opt/ignite-dev") assert argv[-1] == "tester@w1:/opt/.tmp.1/", \ "an interrupted transfer must not leave a live distribution half updated" assert argv[-2] == "/dist/ignite-dev/" def check_ssh_options_reach_rsync(self): - argv = deploy.rsync_argv(_RsyncTransport(), "/dist/x", "/opt/.tmp.1") + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/x", "/opt/.tmp.1", + files_from="/tmp/x.files") assert argv[argv.index("-e") + 1] == \ "ssh -o BatchMode=yes -i /home/tester/.ssh/id_ed25519" def check_sudo_and_checksum_are_passed_through(self): argv = deploy.rsync_argv(_RsyncTransport(), "/dist/x", "/opt/.tmp.1", - checksum=True, sudo=True) + files_from="/tmp/x.files", checksum=True, sudo=True) assert "--checksum" in argv and "--rsync-path=sudo -n rsync" in argv + def check_progress_is_asked_of_rsync_only_when_something_displays_it(self): + plain = deploy.rsync_argv(_RsyncTransport(), "/dist/x", "/opt/.tmp.1", + files_from="/tmp/x.files") + watched = deploy.rsync_argv(_RsyncTransport(), "/dist/x", "/opt/.tmp.1", + files_from="/tmp/x.files", progress=True) + assert "--info=progress2" not in plain + assert "--info=progress2" in watched + # -- stats ------------------------------------------------------------------- def check_stats_are_read_back(self): @@ -480,6 +491,12 @@ def check_a_windows_coordinator_falls_back(self, monkeypatch): # -- the transfer ------------------------------------------------------------ + @staticmethod + def _file_list(tmp_path, root, name="ignite-dev.files"): + path = tmp_path / name + path.write_text("\n".join(deploy.included_files(root)) + "\n", encoding="utf-8") + return path + def check_the_tarball_is_never_built_when_every_host_takes_rsync(self, tmp_path, monkeypatch): root, payload = self._dist_and_payload(tmp_path) @@ -487,20 +504,21 @@ def check_the_tarball_is_never_built_when_every_host_takes_rsync(self, tmp_path, calls = [] def fake_run_local(argv, **kw): - calls.append((argv, kw.get("input"))) + calls.append(argv) return Result(argv, 0, self.STATS, "", "local") monkeypatch.setattr(deploy, "run_local", fake_run_local) manifest = deploy.build_manifest(root) + files = self._file_list(tmp_path, root) result = deploy._deploy_to_host( # noqa: SLF001 self._ctx(transport), Node(host="w1", user="tester"), "ignite-dev", - "/opt/ignite-dev", manifest, "{}", payload, None, None, - "\n".join(deploy.included_files(root)) + "\n") + "/opt/ignite-dev", manifest, "{}", payload, None, None, files) assert not (tmp_path / "p.tar.gz").exists(), \ "compressing a linked checkout would cost more than the transfer saves" assert len(calls) == 1 - assert calls[0][1] == "bin/ignite.sh\nlibs/core.jar\n" + assert "--files-from=%s" % files in calls[0] + assert files.read_text(encoding="utf-8") == "bin/ignite.sh\nlibs/core.jar\n" assert result.status == CHANGED and "rsync: 12 of 2 file(s) changed" in result.message def check_the_staging_tree_is_still_swapped_into_place(self, tmp_path, monkeypatch): @@ -511,7 +529,7 @@ def check_the_staging_tree_is_still_swapped_into_place(self, tmp_path, monkeypat deploy._deploy_to_host( # noqa: SLF001 self._ctx(transport), Node(host="w1", user="tester"), "ignite-dev", "/opt/ignite-dev", deploy.build_manifest(root), "{}", payload, None, None, - "bin/ignite.sh\n") + self._file_list(tmp_path, root)) scripts = "\n".join(transport.scripts) assert 'mv -- "$staging" "$target"' in scripts assert deploy.MANIFEST_NAME in "".join(transport.files) @@ -525,7 +543,7 @@ def check_a_worker_without_rsync_gets_the_tarball(self, tmp_path, monkeypatch): result = deploy._deploy_to_host( # noqa: SLF001 self._ctx(transport), Node(host="w1", user="tester"), "ignite-dev", "/opt/ignite-dev", deploy.build_manifest(root), "{}", payload, None, None, - "bin/ignite.sh\n") + self._file_list(tmp_path, root)) assert (tmp_path / "p.tar.gz").exists() assert transport.uploads and result.status == CHANGED @@ -538,7 +556,7 @@ def check_a_failed_rsync_leaves_the_target_alone(self, tmp_path, monkeypatch): result = deploy._deploy_to_host( # noqa: SLF001 self._ctx(transport), Node(host="w1", user="tester"), "ignite-dev", "/opt/ignite-dev", deploy.build_manifest(root), "{}", payload, None, None, - "bin/ignite.sh\n") + self._file_list(tmp_path, root)) assert result.status == FAILED and "permission denied" in result.detail assert 'mv -- "$staging" "$target"' not in "\n".join(transport.scripts) diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_progress.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_progress.py new file mode 100644 index 0000000000000..0b6ce9f5e494f --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_progress.py @@ -0,0 +1,283 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for the progress display and the streamed transfers that feed it.""" + +import io +import sys + +from ducktests_remote.commands import deploy +from ducktests_remote.progress import (NullProgress, Progress, human_bytes, human_duration, + is_a_terminal) +from ducktests_remote.transport import LocalTransport, run_local_streaming + + +class _Stream(io.StringIO): + """A StringIO that can claim to be a terminal, with a settable encoding.""" + + def __init__(self, tty=False, encoding="utf-8"): + super().__init__() + self._tty = tty + self._encoding = encoding + + @property + def encoding(self): + """:return: the encoding the bar picks its characters from.""" + return self._encoding + + def isatty(self): + return self._tty + + +class CheckFormatting: + """The numbers an operator reads at a glance.""" + + def check_bytes_are_scaled(self): + assert human_bytes(512) == "512 B" + assert human_bytes(1024 * 1024 * 3) == "3.0 MB" + assert human_bytes(1024 ** 3 * 3.5) == "3.5 GB" + + def check_durations_grow_an_hour_field_only_when_needed(self): + assert human_duration(72) == "1:12" + assert human_duration(3672) == "1:01:12" + + +class CheckAggregate: + """The line that has to make sense on its own in a log file.""" + + @staticmethod + def _progress(hosts, **kw): + return Progress(hosts, stream=_Stream(), live=False, **kw) + + def check_it_averages_the_per_host_fractions(self): + progress = self._progress(["w1", "w2", "w3", "w4"]) + progress.done("w1") + progress.sent("w2", 50, 100) + assert "38% 1/4 host(s)" in progress.aggregate_line(), \ + "one host finished plus one half done is 3/8 of four hosts" + + def check_it_counts_finished_hosts_and_bytes_moved(self): + progress = self._progress(["w1", "w2"]) + progress.sent("w1", 1024 * 1024, 1024 * 1024 * 2) + progress.done("w2") + line = progress.aggregate_line() + assert "1/2 host(s)" in line and "1.0 MB sent" in line + + def check_a_rsync_fraction_beats_bytes_over_total(self): + progress = self._progress(["w1"]) + progress.sent("w1", 10, 1000, fraction=0.9) + assert "90% 0/1 host(s)" in progress.aggregate_line(), \ + "rsync knows what it decided not to send; sent/total does not" + + def check_an_unknown_host_is_ignored(self): + progress = self._progress(["w1"]) + progress.sent("nobody", 10, 100) + progress.done("nobody") + assert "0% 0/1 host(s)" in progress.aggregate_line() + + +class CheckRows: + """One line per host, and never more lines than a terminal can hold.""" + + @staticmethod + def _progress(hosts, **kw): + return Progress(hosts, stream=_Stream(tty=True), live=True, **kw) + + def check_a_sending_host_shows_bytes_and_a_bar(self): + progress = self._progress(["w1"]) + progress.sent("w1", 1024 * 1024 * 30, 1024 * 1024 * 100) + row = progress.rows()[0] + assert "w1" in row and " 30%" in row and "30.0 MB / 100.0 MB" in row + + def check_phases_without_bytes_still_say_what_is_happening(self): + progress = self._progress(["w1"]) + progress.phase("w1", "swapping") + assert "swapping" in progress.rows()[0] + + def check_finished_hosts_make_room_for_active_ones(self): + progress = self._progress(["w1", "w2"]) + progress.done("w1") + progress.sent("w2", 1, 2) + rows = progress.rows() + assert len(rows) == 1 and "w2" in rows[0] + + def check_a_large_cluster_is_capped_and_says_so(self): + progress = self._progress(["w%02d" % i for i in range(30)], max_rows=5) + for index in range(30): + progress.sent("w%02d" % index, 1, 10) + rows = progress.rows() + assert len(rows) == 6 and "25 more host(s)" in rows[-1] + + def check_ascii_bars_when_the_stream_cannot_encode_blocks(self): + progress = Progress(["w1"], stream=_Stream(tty=True, encoding="cp1251"), live=True) + progress.sent("w1", 1, 2) + assert "#" in progress.rows()[0] and "█" not in progress.rows()[0] + + +class CheckRendering: + """What actually reaches the terminal.""" + + def check_live_mode_redraws_in_place(self): + stream = _Stream(tty=True) + progress = Progress(["w1"], stream=stream, live=True, interval=0.01) + progress.start() + progress.sent("w1", 5, 10) + progress.close() + written = stream.getvalue() + assert "\033[2K" in written and "\033[" in written + assert written.endswith("\033[?25h"), "the cursor must be handed back" + + def check_plain_mode_prints_whole_lines_only(self): + stream = _Stream(tty=False) + progress = Progress(["w1"], stream=stream, live=False, interval=0.01, + plain_interval=0.0) + progress.start() + progress.sent("w1", 5, 10) + progress.close() + written = stream.getvalue() + assert "\033[" not in written, "a log file must not collect cursor commands" + assert "total" in written + + def check_a_broken_stream_does_not_break_the_deploy(self): + stream = _Stream(tty=True) + progress = Progress(["w1"], stream=stream, live=True, interval=0.01) + progress.start() + stream.close() + progress.sent("w1", 5, 10) + progress.close() + + def check_secrets_are_redacted_on_the_way_out(self): + class _Redactor: # pylint: disable=too-few-public-methods + @staticmethod + def redact(text): + return text.replace("w1", "***") + + stream = _Stream(tty=False) + progress = Progress(["w1"], stream=stream, live=False, interval=0.01, + plain_interval=0.0, redactor=_Redactor()) + progress.start() + progress.sent("w1", 1, 2) + progress.close() + assert "w1" not in stream.getvalue() + + +class CheckNullProgress: + """The disabled display has to accept every call the live one does.""" + + def check_every_method_is_a_no_op(self): + with NullProgress() as progress: + progress.phase("w1", "sending") + progress.sent("w1", 1, 2, fraction=0.5) + progress.done("w1", "done") + assert NullProgress().live is False + + def check_a_non_terminal_is_never_live(self): + assert is_a_terminal(_Stream(tty=False)) is False + assert is_a_terminal(io.StringIO()) is False + + +class CheckWhenItIsUsed: + """deploy decides; the display only draws.""" + + @staticmethod + def _ctx(*, dry_run=False, quiet=False, verbose=False, no_progress=False): + class _Console: # pylint: disable=too-few-public-methods + pass + + class _Args: # pylint: disable=too-few-public-methods + pass + + class _Ctx: # pylint: disable=too-few-public-methods + pass + + ctx = _Ctx() + ctx.console = _Console() + ctx.console.quiet = quiet + ctx.console.verbose = verbose + ctx.console.redactor = None + ctx.args = _Args() + ctx.args.no_progress = no_progress + ctx.dry_run = dry_run + return ctx + + @staticmethod + def _nodes(): + class _Node: # pylint: disable=too-few-public-methods + host = "w1" + + return [_Node()] + + def check_a_dry_run_reports_nothing(self): + assert isinstance(deploy.build_progress(self._ctx(dry_run=True), self._nodes()), + NullProgress) + + def check_quiet_and_no_progress_report_nothing(self): + assert isinstance(deploy.build_progress(self._ctx(quiet=True), self._nodes()), + NullProgress) + assert isinstance(deploy.build_progress(self._ctx(no_progress=True), self._nodes()), + NullProgress) + + def check_verbose_reports_without_redrawing(self, monkeypatch): + monkeypatch.setattr(deploy, "is_a_terminal", lambda _: True) + progress = deploy.build_progress(self._ctx(verbose=True), self._nodes()) + assert isinstance(progress, Progress) and progress.live is False, \ + "a redrawn block would fight with the traced command lines" + + def check_a_terminal_gets_the_live_display(self, monkeypatch): + monkeypatch.setattr(deploy, "is_a_terminal", lambda _: True) + assert deploy.build_progress(self._ctx(), self._nodes()).live is True + + +class CheckStreamedTransfers: + """The two ways bytes are counted while they move.""" + + def check_output_is_delivered_line_by_line_as_it_arrives(self): + lines = [] + script = ("import sys\n" + "sys.stdout.write(' 1,024 10% 1MB/s\\r')\n" + "sys.stdout.write(' 2,048 20% 1MB/s\\n')\n" + "sys.stdout.write('Number of regular files transferred: 2\\n')\n") + result = run_local_streaming([sys.executable, "-c", script], on_output=lines.append) + assert result.ok + assert deploy.parse_rsync_progress(lines[0]) == (1024, 0.1) + assert deploy.parse_rsync_progress(lines[1]) == (2048, 0.2) + assert deploy.parse_rsync_stats(result.stdout) is None or True + assert len(lines) == 3, "a carriage return ends a progress line just as a newline does" + + def check_a_failing_command_keeps_its_output_and_status(self): + result = run_local_streaming( + [sys.executable, "-c", "import sys; sys.stderr.write('boom'); sys.exit(3)"], + on_output=lambda _: None) + assert result.returncode == 3 and "boom" in result.stderr + + def check_progress_lines_that_are_not_progress_are_ignored(self): + assert deploy.parse_rsync_progress("sending incremental file list") is None + assert deploy.parse_rsync_progress("") is None + + def check_a_local_copy_reports_every_chunk(self, tmp_path): + source = tmp_path / "payload.tar.gz" + source.write_bytes(b"x" * (3 * 1024 * 1024 + 7)) + seen = [] + LocalTransport().upload_watched(source, str(tmp_path / "out.tar.gz"), + on_bytes=seen.append) + assert seen[-1] == source.stat().st_size + assert seen == sorted(seen) and len(seen) == 4 + assert (tmp_path / "out.tar.gz").read_bytes() == source.read_bytes() + + def check_an_unwatched_upload_is_the_plain_one(self, tmp_path): + source = tmp_path / "a" + source.write_text("body", encoding="utf-8") + LocalTransport().upload_watched(source, str(tmp_path / "b")) + assert (tmp_path / "b").read_text(encoding="utf-8") == "body" diff --git a/modules/ducktests/tests/ducktests_remote/cli.py b/modules/ducktests/tests/ducktests_remote/cli.py index 513bb62225272..57443e1ce70e4 100644 --- a/modules/ducktests/tests/ducktests_remote/cli.py +++ b/modules/ducktests/tests/ducktests_remote/cli.py @@ -190,6 +190,8 @@ def _common_parser(): help="disable ANSI colour") parser.add_argument("--fail-fast", action="store_true", default=argparse.SUPPRESS, help="abort a fan-out after the first host failure") + parser.add_argument("--no-progress", action="store_true", default=argparse.SUPPRESS, + help="do not report per-host progress during long transfers") return parser diff --git a/modules/ducktests/tests/ducktests_remote/commands/deploy.py b/modules/ducktests/tests/ducktests_remote/commands/deploy.py index 9377465b0d6d3..7e25cc75bf15b 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/deploy.py +++ b/modules/ducktests/tests/ducktests_remote/commands/deploy.py @@ -31,8 +31,10 @@ jar rather than the whole tree. The fallback, used when either end has no rsync or when ``--via`` stages the payload on an intermediate host, is a tarball of the whole distribution. Both end in the same atomic swap, and both carry exactly the file list the -manifest was built from - rsync is fed that list on stdin rather than its own ``--exclude`` -patterns, whose matching rules differ subtly from :func:`is_excluded`'s. +manifest was built from - rsync is handed that list as a file rather than its own +``--exclude`` patterns, whose matching rules differ subtly from :func:`is_excluded`'s. + +Long transfers report progress per host; see :mod:`ducktests_remote.progress`. :func:`build_manifest`, :func:`prepare_script`, :func:`swap_script` and :func:`human` are public because ``provision``'s ``jdk`` step delivers a JDK the same way and must not grow @@ -46,6 +48,7 @@ import re import shlex import shutil +import sys import tempfile import threading import uuid @@ -55,8 +58,9 @@ from ducktests_remote.config import ConfigError, expand_path from ducktests_remote.fanout import (CHANGED, FAILED, HostResult, SKIPPED, any_failed, fanout, render_table, summarise) +from ducktests_remote.progress import NullProgress, Progress, is_a_terminal from ducktests_remote.transport import (ProxiedTransport, is_excluded, make_tarball, - run_local) + run_local, run_local_streaming) MANIFEST_NAME = ".ducktests-deploy.json" @@ -71,6 +75,10 @@ _RSYNC_TRANSFERRED = re.compile(r"Number of (?:regular )?files transferred:\s*([\d,.]+)") _RSYNC_SENT = re.compile(r"Total transferred file size:\s*([\d,.]+)") +# `--info=progress2`: ` 1,234,567 35% 12.34MB/s 0:00:12`. The separators are +# locale-dependent, the layout is not. +_RSYNC_PROGRESS = re.compile(r"^\s*([\d,.]+)\s+(\d+)%\s") + def register(subparsers, common): """Wire up the ``deploy`` subcommand.""" @@ -143,7 +151,10 @@ def execute(ctx): # pylint: disable=too-many-locals overall = [] for name, manifest, excludes in plans: console.heading("%s -> %s/%s" % (name, install_root, name)) - results = _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes, excludes) + progress = build_progress(ctx, nodes) + with progress: + results = _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes, + excludes, progress) overall.extend(results) console.out(render_table(results, verbose=console.verbose)) console.out(summarise(results)) @@ -155,6 +166,22 @@ def execute(ctx): # pylint: disable=too-many-locals return EXIT_TRANSPORT if any_failed(overall) else EXIT_OK +def build_progress(ctx, nodes): + """ + :return: a :class:`Progress` for this deploy, or a :class:`NullProgress`. + + Off for ``--dry-run`` (nothing moves) and ``--quiet``. Live only on a terminal: + ``--verbose`` prints a traced command line per host, which a redrawn block would + fight with, so that combination reports one aggregate line every few seconds + instead - the same shape a CI log gets. + """ + if ctx.dry_run or ctx.console.quiet or getattr(ctx.args, "no_progress", False): + return NullProgress() + live = is_a_terminal(sys.stderr) and not ctx.console.verbose + return Progress([node.host for node in nodes], live=live, + redactor=ctx.console.redactor) + + def _distributions(dist_dir, only): names = sorted(p.name for p in dist_dir.iterdir() if p.is_dir() and not p.name.startswith(".")) @@ -271,7 +298,8 @@ def human(size): return "%.1f TB" % value -def _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes, excludes=()): +def _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes, excludes=(), + progress=None): args = ctx.args target = posixpath.join(install_root, name) manifest_body = json.dumps({ @@ -304,12 +332,14 @@ def _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes, excludes=()) file_list = None if rsync_enabled(ctx): - file_list = "\n".join(included_files(Path(dist_dir) / name, excludes)) + "\n" + file_list = Path(tmp) / ("%s.files" % name) + file_list.write_text("\n".join(included_files(Path(dist_dir) / name, excludes)) + + "\n", encoding="utf-8") def operation(node, _payload=payload, _staged=staged_on_via, _via=via_transport, _files=file_list): return _deploy_to_host(ctx, node, name, target, manifest, manifest_body, - _payload, _staged, _via, _files) + _payload, _staged, _via, _files, progress) try: return fanout(nodes, operation, jobs=ctx.jobs, @@ -371,7 +401,19 @@ def rsync_enabled(ctx): return shutil.which("rsync") is not None -def rsync_argv(transport, local_root, staging, *, link_dest=None, checksum=False, sudo=False): +def parse_rsync_progress(line): + """:return: ``(bytes_so_far, fraction)`` from one ``--info=progress2`` line, or None.""" + match = _RSYNC_PROGRESS.match(line or "") + if not match: + return None + try: + return int(re.sub(r"[,.]", "", match.group(1))), int(match.group(2)) / 100.0 + except ValueError: + return None + + +def rsync_argv(transport, local_root, staging, *, files_from, link_dest=None, + checksum=False, sudo=False, progress=False): """ :return: the rsync command line that fills ``staging`` on one worker. @@ -380,9 +422,16 @@ def rsync_argv(transport, local_root, staging, *, link_dest=None, checksum=False one module costs one jar. The staging directory is still built from scratch and still swapped in atomically, so an interrupted transfer cannot leave a live distribution half updated. + + ``files_from`` is a file holding the exact list to send, written once per + distribution. It is not stdin: the output side is read as it arrives for progress, + and a process whose stdin and stdout are both being pumped by one thread deadlocks + as soon as either pipe fills. """ - argv = ["rsync", "-a", "--stats", "--files-from=-", + argv = ["rsync", "-a", "--stats", "--files-from=%s" % files_from, "-e", shlex.join(["ssh"] + transport.ssh_options())] + if progress: + argv.append("--info=progress2") if checksum: argv.append("--checksum") if link_dest: @@ -408,8 +457,9 @@ def parse_rsync_stats(text): def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, payload, - staged_on_via, via_transport, file_list=None): + staged_on_via, via_transport, file_list=None, progress=None): transport = ctx.worker(node) + progress = progress or NullProgress() remote_manifest = posixpath.join(target, MANIFEST_NAME) if not ctx.args.force: @@ -417,6 +467,7 @@ def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, payload, if existing: try: if json.loads(existing).get("hash") == manifest["hash"]: + progress.done(node.host, "already up to date") return HostResult(node.host, SKIPPED, "already at %s" % manifest["hash"][:12]) except ValueError: @@ -425,6 +476,7 @@ def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, payload, install_root = posixpath.dirname(target) writable = transport.run(["test", "-w", install_root], check=False).ok if not writable and not ctx.args.sudo: + progress.done(node.host, "not writable") return HostResult(node.host, FAILED, "%s is not writable by %s and --sudo was not passed" % (install_root, node.user or "this account")) @@ -434,6 +486,7 @@ def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, payload, stats = None if via_transport is not None: + progress.phase(node.host, "fanning out") proxied = ProxiedTransport(name=node.host, via=via_transport, user=node.user, port=node.port, identity_file=node.identity_file, @@ -443,21 +496,29 @@ def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, payload, proxied.push_archive(staged_on_via, staging) elif file_list is not None and getattr(transport, "has_rsync", _no_rsync)(): outcome = _rsync_to_host(ctx, node, transport, payload.root, staging, target, - file_list) + file_list, progress) if isinstance(outcome, HostResult): + progress.done(node.host, "failed") return outcome used_rsync, stats = True, outcome else: transport.run_script(prepare_script(staging, ctx.args.sudo)).check() remote_archive = "%s/.payload.tar.gz" % staging - transport.upload(payload.archive(), remote_archive) + archive = payload.archive() + progress.phase(node.host, "sending") + transport.upload_watched(archive, remote_archive, + on_bytes=_watcher(progress, node.host, + os.path.getsize(archive))) + progress.phase(node.host, "extracting") transport.run_script( "set -eu\ntar -xzf %s -C %s\nrm -f -- %s\n" % (shlex.quote(remote_archive), shlex.quote(staging), shlex.quote(remote_archive))).check() + progress.phase(node.host, "swapping") transport.write_file(manifest_body, posixpath.join(staging, MANIFEST_NAME)) transport.run_script(swap_script(staging, target, ctx.args.sudo, ctx.args.owner)).check() + progress.done(node.host) if used_rsync: if stats: return HostResult(node.host, CHANGED, "rsync: %d of %s file(s) changed, %s sent" @@ -472,7 +533,13 @@ def _no_rsync(): return False -def _rsync_to_host(ctx, node, transport, local_root, staging, target, file_list): +def _watcher(progress, host, total): + """:return: an ``on_bytes`` callback that feeds one host's row.""" + return lambda sent: progress.sent(host, sent, total) + + +def _rsync_to_host(ctx, node, transport, local_root, staging, target, file_list, + progress=None): """ :return: ``(files, bytes)`` transferred, ``None`` when ``--stats`` could not be read, or a failed :class:`HostResult`. @@ -480,19 +547,36 @@ def _rsync_to_host(ctx, node, transport, local_root, staging, target, file_list) The staging directory is created before rsync runs, and hardlinked against the live distribution when there is one to link against. """ + progress = progress or NullProgress() transport.run_script(prepare_script(staging, ctx.args.sudo)).check() - argv = rsync_argv(transport, local_root, staging, + # Both display modes need the byte counts; only the display itself differs. + watched = not isinstance(progress, NullProgress) + argv = rsync_argv(transport, local_root, staging, files_from=file_list, link_dest=target if transport.exists(target) else None, checksum=ctx.args.checksum or ctx.config["deploy"].get("checksum", False), - sudo=ctx.args.sudo) + sudo=ctx.args.sudo, progress=watched) ctx.console.detail("%s: %s" % (node.host, shlex.join(argv))) - result = run_local(argv, input=file_list) + progress.phase(node.host, "sending") + if watched: + result = run_local_streaming(argv, host=node.host, + on_output=_rsync_watcher(progress, node.host)) + else: + result = run_local(argv) if not result.ok: return HostResult(node.host, FAILED, "rsync failed", detail=(result.stderr or result.stdout).strip()) return parse_rsync_stats(result.stdout) +def _rsync_watcher(progress, host): + """:return: a line callback that turns ``--info=progress2`` into one host's row.""" + def watch(line): + reading = parse_rsync_progress(line) + if reading: + progress.sent(host, reading[0], fraction=reading[1]) + return watch + + def prepare_script(staging, use_sudo): sudo = "sudo -n " if use_sudo else "" return "set -eu\n%(sudo)srm -rf -- %(staging)s\n%(sudo)smkdir -p %(staging)s\n" % { diff --git a/modules/ducktests/tests/ducktests_remote/docs/commands.md b/modules/ducktests/tests/ducktests_remote/docs/commands.md index c9e5a6eaaba8a..621777a4d8873 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/commands.md +++ b/modules/ducktests/tests/ducktests_remote/docs/commands.md @@ -33,6 +33,7 @@ Ten commands: `run`, `status`, `logs`, `fetch`, `stop`, `provision`, `deploy`, ` | `--dry-run` | print what would happen; execute nothing | | `--no-color` | disable ANSI colour | | `--fail-fast` | stop scheduling further hosts after the first failure | +| `--no-progress` | do not report per-host progress during long transfers | `--version` prints the CLI version. Everything after a bare `--` is passed straight to ducktape by `run`. @@ -277,6 +278,41 @@ The result line reports what actually moved: w01 changed 4.1s rsync: 12 of 8431 file(s) changed, 4.0 MB sent ``` +### Watching a long transfer + +Sending 3.5 GB to twelve machines takes minutes, and a terminal that prints nothing for +that long is indistinguishable from one that has hung. `deploy` shows every host while it +works, redrawn in place: + +``` + worker01 █████████████████░░░░░░░ 70% 210.0 MB / 300.0 MB 24.1 MB/s + worker02 ████████░░░░░░░░░░░░░░░░ 32% 96.0 MB / 300.0 MB 18.7 MB/s + worker03 ░░░░░░░░░░░░░░░░░░░░░░░░ extracting + total ████████████░░░░░░░░░░░░ 50% 1/4 host(s) 306.0 MB sent 1:12 +``` + +Hosts drop out of the block as they finish — they are already counted in the total — and +a cluster larger than the block ends with `... N more host(s)`. + +The total is the **mean of the per-host fractions**, not bytes against a predicted total: +rsync sends only what differs, so a denominator of "distribution size × hosts" would stop +at 12% and finish there. Each host owns an equal share of that bar. + +Where the byte counts come from: `--info=progress2` on the rsync path, and the bytes +handed to the remote `cat` on the tarball path. scp has a progress meter of its own, but +it is written for a terminal and suppressed when its output is a pipe — which it always +is here — so a watched tarball upload streams the file through `ssh 'cat > path'` and +counts the chunks itself. + +| Situation | What you get | +| --- | --- | +| a terminal | the redrawn block above | +| not a terminal (a log, CI), or `--verbose` | one `total ...` line every 15 s | +| `--quiet`, `--dry-run`, `--no-progress` | nothing | + +`--verbose` deliberately drops to the single line: it prints a traced command per host, +and a redrawn block would fight with it. + The tarball path is used instead when: | Condition | Why | diff --git a/modules/ducktests/tests/ducktests_remote/docs/internals.md b/modules/ducktests/tests/ducktests_remote/docs/internals.md index f346dccfccefb..845070004fddd 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/internals.md +++ b/modules/ducktests/tests/ducktests_remote/docs/internals.md @@ -29,6 +29,7 @@ For changing the CLI rather than using it. | `cluster.py` | inventory → `Node` list → ducktape's `cluster.json` | | `transport.py` | the only place that shells out to `ssh`/`scp` | | `fanout.py` | bounded parallel per-host execution and the result table | +| `progress.py` | the live per-host display for transfers that take minutes | | `runs.py` | run ids, run directory layout, state derivation, script rendering | | `sshdiag.py` | SSH failure classification and the administrator block | | `java.py` | JDK discovery, environment files, archive inspection | @@ -103,6 +104,25 @@ point. Statuses: `ok`, `changed`, `skipped`, `warn`, `failed`. `render_table` prints failures' detail by default and everything's detail with `-v`; `summarise` prints `9 ok, 2 failed`, ordered worst-first. +## Progress + +`Progress` is written to from the fan-out threads and drawn by one daemon thread of its +own, so a slow terminal cannot slow a transfer down and an update never blocks on a +redraw. Three rules it does not break: + +- **It never raises.** A display that takes a deploy down with it would be worse than no + display, so a stream that cannot be written to just turns the display off. +- **The block is erased before anything else prints.** `deploy` renders the result table + after the `with progress:` body, on the lines the block occupied. +- **`NullProgress` has the same surface.** Callers never test for `None` or for whether + the display is on; `--quiet`, `--dry-run` and `--no-progress` simply swap the object. + +Byte counts come from `--info=progress2` (rsync) and from `upload_watched`, which streams +a file into `ssh 'cat > path'` because scp suppresses its own meter when its output is a +pipe. `run_local_streaming` reads a child's output as it arrives, treating a carriage +return as a line ending, and drains stderr on a thread so a chatty failure cannot deadlock +the pipe. + ## Redaction `Redactor` keys on resolved **values**, not on key names. Anything coming out of `${env:}` @@ -145,8 +165,10 @@ flake8 ducktests_remote `[pytest]` in `tox.ini` collects `check_*.py` files, `Check` classes and `check_*` functions, which is why the files are named that way. `checks/fake_transport.py` provides a recording transport that simulates a small filesystem and returns canned output for -commands matching a needle. The only subprocess in the suite is the deliberate one that -proves the ducktape import boundary still holds. +commands matching a needle. Two kinds of subprocess are deliberate: the one that proves +the ducktape import boundary still holds, and the short `python -c` scripts that +`check_remote_progress.py` uses to prove output really is read as it arrives. Nothing +else spawns anything. Checks are named as sentences — `check_lists_replace_and_do_not_concatenate` — because the name is the specification and shows up in the failure output. diff --git a/modules/ducktests/tests/ducktests_remote/progress.py b/modules/ducktests/tests/ducktests_remote/progress.py new file mode 100644 index 0000000000000..c2681cddac63d --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/progress.py @@ -0,0 +1,353 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Live progress for the transfers that take minutes. + +Sending 3.5 GB to twelve machines is a normal deploy, and a terminal that prints nothing +for six minutes is indistinguishable from one that has hung. This renders what every +host is doing, from the same fan-out threads that do the work. + +Two modes and no third: + +* **live** - a redrawn block on a terminal, one row per host plus a total. +* **plain** - one aggregate line every ``plain_interval`` seconds, for a log file, a CI + job, ``--verbose`` (where a redrawn block would fight with the traced command lines) + and anything else that is not a terminal. + +The aggregate is the mean of the per-host fractions, never a byte total against a +predicted one: rsync sends only what differs, so a denominator of "the whole +distribution times the host count" would stall at 12% and finish there. Each host +contributes an equal share of the bar, scaled by how far along it is. +""" + +import os +import shutil +import sys +import threading +import time + +WAITING = "waiting" +SENDING = "sending" +DONE = "done" + +_UP = "\033[%dA" +_CLEAR_LINE = "\033[2K" +_HIDE_CURSOR = "\033[?25l" +_SHOW_CURSOR = "\033[?25h" + + +def human_bytes(size): + """:return: ``1.4 GB``; the same shape deploy prints elsewhere.""" + value = float(size) + for unit in ("B", "KB", "MB", "GB", "TB"): + if value < 1024 or unit == "TB": + return "%.0f %s" % (value, unit) if unit in ("B", "KB") else "%.1f %s" % (value, unit) + value /= 1024 + return "%.1f TB" % value + + +def human_duration(seconds): + """:return: ``4:12`` or ``1:04:12``.""" + seconds = int(max(0, seconds)) + hours, rest = divmod(seconds, 3600) + minutes, secs = divmod(rest, 60) + if hours: + return "%d:%02d:%02d" % (hours, minutes, secs) + return "%d:%02d" % (minutes, secs) + + +class _HostState: # pylint: disable=too-few-public-methods + """What one host is doing, and how far along it is.""" + + def __init__(self, name): + self.name = name + self.phase = WAITING + self.sent = 0 + self.total = 0 + self.fraction = 0.0 + self.started = None + self.message = "" + + @property + def rate(self): + """:return: bytes per second since this host started, or 0.""" + if not self.started or self.sent <= 0: + return 0 + elapsed = time.monotonic() - self.started + return self.sent / elapsed if elapsed > 0 else 0 + + +class Progress: + """ + A thread-safe progress display shared by every host in a fan-out. + + Nothing here raises: a display that breaks a deploy would be worse than no display, + so a stream that cannot be written to simply stops being written to. + """ + + def __init__(self, hosts, *, stream=None, live=None, redactor=None, + interval=0.25, plain_interval=15.0, max_rows=12, unicode_bar=None): + self.stream = stream if stream is not None else sys.stderr + self.hosts = {name: _HostState(name) for name in hosts} + self.order = list(hosts) + self.redactor = redactor + self.interval = interval + self.plain_interval = plain_interval + self.max_rows = max_rows + self.live = is_a_terminal(self.stream) if live is None else bool(live) + self.blocks = _unicode_bar(self.stream) if unicode_bar is None else bool(unicode_bar) + self.title = "" + self._lock = threading.Lock() + self._thread = None + self._stop = threading.Event() + self._drawn = 0 + self._started = time.monotonic() + self._last_plain = 0.0 + + # -- lifecycle --------------------------------------------------------------- + + def start(self, title=""): + """Begin redrawing. Safe to call when the display is disabled.""" + self.title = title + self._started = time.monotonic() + self._last_plain = time.monotonic() + if self._thread is not None: + return self + self._stop.clear() + self._thread = threading.Thread(target=self._loop, name="dtr-progress", daemon=True) + self._thread.start() + return self + + def close(self): + """Stop redrawing and leave the terminal as it was found.""" + if self._thread is None: + return + self._stop.set() + self._thread.join(timeout=2 * self.interval + 1) + self._thread = None + if self.live: + self._write(self._erase() + _SHOW_CURSOR) + + def __enter__(self): + return self.start() + + def __exit__(self, *_exc): + self.close() + return False + + # -- updates, called from the fan-out threads -------------------------------- + + def phase(self, host, phase): + """Record what ``host`` is doing now: sending, extracting, swapping.""" + with self._lock: + state = self.hosts.get(host) + if state is None: + return + state.phase = phase + if state.started is None: + state.started = time.monotonic() + + def sent(self, host, sent, total=None, fraction=None): + """ + Record transferred bytes for ``host``. + + :param fraction: 0..1 when the sender knows better than ``sent/total`` - rsync + reports its own percentage, which accounts for files it decided not to send. + """ + with self._lock: + state = self.hosts.get(host) + if state is None: + return + if state.started is None: + state.started = time.monotonic() + state.phase = SENDING + state.sent = max(state.sent, int(sent or 0)) + if total: + state.total = int(total) + if fraction is not None: + state.fraction = min(1.0, max(0.0, float(fraction))) + elif state.total: + state.fraction = min(1.0, state.sent / float(state.total)) + + def done(self, host, message=""): + """Mark ``host`` finished, however it finished.""" + with self._lock: + state = self.hosts.get(host) + if state is None: + return + state.phase = DONE + state.fraction = 1.0 + state.message = message + + # -- rendering --------------------------------------------------------------- + + def _loop(self): + while not self._stop.is_set(): + self._tick() + self._stop.wait(self.interval) + self._tick(final=True) + + def _tick(self, final=False): + if self.live: + self._write(self._frame()) + return + now = time.monotonic() + if final or now - self._last_plain >= self.plain_interval: + self._last_plain = now + line = self.aggregate_line() + if line: + self._write(line + "\n") + + def _frame(self): + lines = self.rows() + [self.aggregate_line()] + out = [_HIDE_CURSOR] + if self._drawn: + out.append(_UP % self._drawn) + for line in lines: + out.append(_CLEAR_LINE + self._fit(line) + "\n") + for _ in range(max(0, self._drawn - len(lines))): + out.append(_CLEAR_LINE + "\n") + overshoot = max(0, self._drawn - len(lines)) + if overshoot: + out.append(_UP % overshoot) + self._drawn = len(lines) + return "".join(out) + + def _erase(self): + if not self._drawn: + return "" + return (_UP % self._drawn) + (_CLEAR_LINE + "\n") * self._drawn + (_UP % self._drawn) + + def rows(self): + """:return: one rendered line per host worth showing.""" + with self._lock: + states = [self.hosts[name] for name in self.order] + active = [s for s in states if s.phase not in (DONE, WAITING)] + # Finished hosts leave the block: they are already counted in the total line, and + # on a large cluster they would otherwise crowd out the hosts still working. + candidates = active or states + shown = candidates[:self.max_rows] + width = max((len(s.name) for s in states), default=4) + rows = [self._row(state, width) for state in shown] + hidden = len(candidates) - len(shown) + if hidden > 0: + rows.append(" ... %d more host(s)" % hidden) + return rows + + def _row(self, state, width): + name = state.name.ljust(width) + if state.phase == DONE: + return " %s %s %s" % (name, self._bar(1.0), state.message or DONE) + if state.phase == WAITING: + return " %s %s %s" % (name, self._bar(0.0), WAITING) + if state.phase != SENDING: + return " %s %s %s" % (name, self._bar(state.fraction), state.phase) + detail = "%3d%% %s" % (round(state.fraction * 100), human_bytes(state.sent)) + if state.total: + detail += " / %s" % human_bytes(state.total) + if state.rate: + detail += " %s/s" % human_bytes(state.rate) + return " %s %s %s" % (name, self._bar(state.fraction), detail) + + def aggregate_line(self): + """:return: the one line that also stands on its own in a log file.""" + with self._lock: + states = [self.hosts[name] for name in self.order] + if not states: + return "" + finished = sum(1 for s in states if s.phase == DONE) + fraction = sum(s.fraction for s in states) / len(states) + moved = sum(s.sent for s in states) + width = max((len(s.name) for s in states), default=5) + text = " %s %s %3d%% %d/%d host(s) %s sent %s" % ( + "total".ljust(width), self._bar(fraction), round(fraction * 100), finished, + len(states), human_bytes(moved), + human_duration(time.monotonic() - self._started)) + return text if self.live else " ".join(text.split()) + + def _bar(self, fraction, width=24): + filled, empty = ("█", "░") if self.blocks else ("#", ".") + done = int(round(min(1.0, max(0.0, fraction)) * width)) + return filled * done + empty * (width - done) + + def _fit(self, line): + columns = terminal_width(self.stream) + return line if len(line) <= columns else line[:max(0, columns - 1)] + + def _write(self, text): + if not text: + return + if self.redactor is not None: + text = self.redactor.redact(text) + try: + self.stream.write(text) + self.stream.flush() + except (OSError, ValueError): + # A closed or broken stream must never take a deploy down with it. + self.live = False + + +class NullProgress: + """The same surface, doing nothing. Callers never test for None.""" + + live = False + + def start(self, title=""): + """:return: self, so ``with`` reads the same either way.""" + return self + + def close(self): + """Nothing to close.""" + + def __enter__(self): + return self + + def __exit__(self, *_exc): + return False + + def phase(self, host, phase): + """Ignore the update.""" + + def sent(self, host, sent, total=None, fraction=None): + """Ignore the update.""" + + def done(self, host, message=""): + """Ignore the update.""" + + +def is_a_terminal(stream): + """:return: True when ``stream`` can be redrawn in place.""" + try: + if not stream.isatty(): + return False + except (AttributeError, ValueError): + return False + return os.environ.get("TERM") != "dumb" + + +def terminal_width(stream, default=100): + """:return: usable columns, falling back when the size cannot be read.""" + try: + columns = shutil.get_terminal_size().columns + except (OSError, ValueError): + return default + del stream + return columns if columns and columns > 20 else default + + +def _unicode_bar(stream): + encoding = (getattr(stream, "encoding", "") or "").lower() + return "utf" in encoding diff --git a/modules/ducktests/tests/ducktests_remote/transport.py b/modules/ducktests/tests/ducktests_remote/transport.py index 2f13bdcf9c4bc..33afdc2113eb6 100644 --- a/modules/ducktests/tests/ducktests_remote/transport.py +++ b/modules/ducktests/tests/ducktests_remote/transport.py @@ -29,6 +29,7 @@ import subprocess import tarfile import tempfile +import threading import uuid from dataclasses import dataclass, field from fnmatch import fnmatch @@ -147,6 +148,17 @@ def run_script(self, script, **kw): def upload(self, local_path, remote_path, *, mode=None): """Copy a single local file to ``remote_path``.""" + def upload_watched(self, local_path, remote_path, *, mode=None, on_bytes=None): + """ + Copy a file, reporting bytes as they go out. + + The default is :meth:`upload` with no reporting at all; only transports that can + see the bytes leave override it. Callers therefore never have to ask whether + progress is available. + """ + del on_bytes + return self.upload(local_path, remote_path, mode=mode) + @abc.abstractmethod def download(self, remote_path, local_path): """Copy a single remote file to ``local_path``.""" @@ -212,6 +224,23 @@ def upload(self, local_path, remote_path, *, mode=None): if mode is not None: os.chmod(remote_path, mode) + def upload_watched(self, local_path, remote_path, *, mode=None, on_bytes=None): + if on_bytes is None: + return self.upload(local_path, remote_path, mode=mode) + self._trace(["cp", str(local_path), remote_path]) + if self.dry_run: + return None + Path(remote_path).parent.mkdir(parents=True, exist_ok=True) + sent = 0 + with open(local_path, "rb") as source, open(remote_path, "wb") as target: + for block in iter(lambda: source.read(CHUNK), b""): + target.write(block) + sent += len(block) + on_bytes(sent) + if mode is not None: + os.chmod(remote_path, mode) + return None + def download(self, remote_path, local_path): self._trace(["cp", remote_path, str(local_path)]) if self.dry_run: @@ -291,6 +320,27 @@ def upload(self, local_path, remote_path, *, mode=None): if mode is not None: self.run(["chmod", "%o" % mode, remote_path]).check() + def upload_watched(self, local_path, remote_path, *, mode=None, on_bytes=None): + """ + Send a file through ``ssh 'cat > path'``, counting the bytes on the way in. + + scp is the better tool and stays the default, but its progress meter is written + for a terminal and suppressed when stdout is a pipe - which it always is here. + Feeding the file to a remote ``cat`` ourselves is the only way to know how far a + 300 MB payload has got, and the write blocks on a slow link exactly as scp does. + """ + if on_bytes is None: + return self.upload(local_path, remote_path, mode=mode) + self._trace(["cat", str(local_path), ">", "%s:%s" % (self.target, remote_path)]) + if self.dry_run: + return None + command = "cat > %s" % shlex.quote(remote_path) + argv = ["ssh"] + self.ssh_options() + ["-T", self.target, command] + _stream_to_stdin(argv, local_path, on_bytes, host=self.name) + if mode is not None: + self.run(["chmod", "%o" % mode, remote_path]).check() + return None + def download(self, remote_path, local_path): self._trace(["scp", "%s:%s" % (self.target, remote_path), str(local_path)]) if self.dry_run: @@ -495,6 +545,90 @@ def run_local(argv, *, check=False, timeout=None, input=None): # noqa: A002 - m return _spawn([str(a) for a in argv], host="local", check=check, timeout=timeout, input=input) +CHUNK = 1024 * 1024 + + +def run_local_streaming(argv, *, on_output, host="local"): + """ + Run ``argv`` locally, handing each output line to ``on_output`` as it appears. + + :return: a :class:`Result` holding the whole output, so a caller can still parse a + summary the command printed at the end. + + Progress meters separate their updates with carriage returns rather than newlines, + so both count as line endings here. stderr is drained by a thread: a command that + fills the stderr pipe while nobody reads it deadlocks, and rsync is chatty on stderr + when a host misbehaves. + """ + argv = [str(a) for a in argv] + try: + # pylint: disable=consider-using-with + process = subprocess.Popen( # noqa: S603 - argv is always a list, never a string + argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0) + except FileNotFoundError as ex: + raise TransportError("%s: %s" % (argv[0], ex)) from ex + + errors = [] + drain = threading.Thread(target=lambda: errors.append(_decode(process.stderr.read())), + daemon=True) + drain.start() + + collected = [] + pending = "" + while True: + block = process.stdout.read(4096) + if not block: + break + text = _decode(block) + collected.append(text) + pending += text + pending = _emit_lines(pending, on_output) + if pending.strip(): + on_output(pending.strip()) + + process.wait() + drain.join(timeout=5) + return Result(argv, process.returncode, "".join(collected), + errors[0] if errors else "", host) + + +def _emit_lines(pending, on_output): + """:return: the unterminated tail; every complete line goes to ``on_output``.""" + pending = pending.replace("\r\n", "\n") + while True: + cut = min((i for i in (pending.find("\n"), pending.find("\r")) if i >= 0), default=-1) + if cut < 0: + return pending + line, pending = pending[:cut], pending[cut + 1:] + if line.strip(): + on_output(line.strip()) + + +def _stream_to_stdin(argv, local_path, on_bytes, *, host): + """Feed a file into ``argv``'s stdin in chunks, reporting the running total.""" + try: + # pylint: disable=consider-using-with + process = subprocess.Popen( # noqa: S603 - argv is always a list, never a string + argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except FileNotFoundError as ex: + raise TransportError("%s: %s" % (argv[0], ex)) from ex + + sent = 0 + try: + with open(local_path, "rb") as handle: + for block in iter(lambda: handle.read(CHUNK), b""): + process.stdin.write(block) + sent += len(block) + on_bytes(sent) + process.stdin.close() + except (BrokenPipeError, OSError): + # The far end died mid-transfer; its stderr says why, so fall through to wait(). + pass + _, stderr = process.communicate() + result = Result(argv, process.returncode, "", _decode(stderr), host) + return result.check() + + def build_transport(host: str, *, user=None, port=22, identity_file=None, connect_timeout=15, dry_run=False, verbose=False, printer=None) -> Transport: """:return: a :class:`LocalTransport` for ``local``, otherwise an :class:`SshTransport`.""" From 08cf7b99d6c2e0b9c6233dbdcbf048fc2bee2287 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 28 Jul 2026 21:30:23 +0300 Subject: [PATCH 8/9] Report progress for the source sync and the JDK delivery too Both move hundreds of megabytes and both were silent: the source sync is a single 170 MB upload to the runner before a run starts, and `provision --only jdk` sends a JDK to every host that lacks one. Lift build_progress out of deploy into progress.py, so one function decides for all three commands, and give upload_dir an on_progress callback: rsync grows --info=progress2 and is read as it arrives, and the tar fallback goes through upload_watched. parse_rsync_progress moves to transport.py next to the rsync invocations that produce it. Reporting is not free - rsync grows a meter, and a watched upload leaves scp for a streamed `cat` - so callers gate the wiring on progress.watching rather than passing a callback that a NullProgress would throw away. --no-progress and --quiet now really do restore the plain transfer. The JDK step reports only when there is an archive to deliver; probing for a JDK already on the host is one round trip and needs no display. --- .../tests/ducktests_remote/README.md | 7 ++- .../checks/check_remote_java.py | 45 ++++++++++++++ .../checks/check_remote_progress.py | 21 ++++--- .../checks/check_remote_sources.py | 24 ++++++++ .../checks/check_remote_transport.py | 37 +++++++++++ .../ducktests_remote/checks/fake_transport.py | 14 ++++- .../tests/ducktests_remote/commands/deploy.py | 39 ++---------- .../ducktests_remote/commands/provision.py | 34 ++++++++--- .../tests/ducktests_remote/commands/run.py | 13 +++- .../tests/ducktests_remote/docs/commands.md | 17 +++++- .../tests/ducktests_remote/docs/internals.md | 6 ++ .../tests/ducktests_remote/progress.py | 24 ++++++++ .../tests/ducktests_remote/transport.py | 61 ++++++++++++++++--- 13 files changed, 275 insertions(+), 67 deletions(-) diff --git a/modules/ducktests/tests/ducktests_remote/README.md b/modules/ducktests/tests/ducktests_remote/README.md index 80e75ab75c4b0..c70e419f31b2c 100644 --- a/modules/ducktests/tests/ducktests_remote/README.md +++ b/modules/ducktests/tests/ducktests_remote/README.md @@ -251,9 +251,10 @@ sends one jar rather than the whole tree: w01 changed 4.1s rsync: 12 of 8431 file(s) changed, 4.0 MB sent ``` -A transfer that takes minutes shows what every host is doing, redrawn in place on a -terminal and reduced to one `total ...` line every 15 seconds in a log, under `--verbose`, -or wherever stderr is not a terminal (`--no-progress` turns it off): +Every transfer that can take minutes — this one, `run`'s source sync and `provision`'s JDK +delivery — shows what each host is doing, redrawn in place on a terminal and reduced to one +`total ...` line every 15 seconds in a log, under `--verbose`, or wherever stderr is not a +terminal (`--no-progress` turns it off): ``` worker01 █████████████████░░░░░░░ 70% 210.0 MB / 300.0 MB 24.1 MB/s diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py index 17fe2358ad6a3..bd5260318aae2 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py @@ -366,6 +366,51 @@ def check_a_host_that_already_has_that_archive_is_skipped(self, tmp_path): ctx, NODE, cfg, provision.JdkPayload(plan), java.discovery_script(cfg)) assert result.status == provision.OK and not fake.uploads + def check_the_delivery_reports_its_bytes(self, tmp_path): + archive = _tarball(tmp_path, ["bin/java"]) + ctx, fake = _context(PROBE, major=21, archive=str(archive), install_root="/opt") + cfg = java.config_of(ctx) + plan = java.archive_plan(cfg.archive) + seen = [] + + class _Progress: # pylint: disable=too-few-public-methods + """Records what a real display would draw.""" + + live = False + watching = True + + @staticmethod + def phase(host, phase): + seen.append((host, phase)) + + @staticmethod + def sent(host, sent, total=None, fraction=None): + seen.append((host, "sent", sent, total)) + + @staticmethod + def done(host, message=""): + seen.append((host, "done")) + + provision._jdk_on_host( # noqa: SLF001 + ctx, NODE, cfg, provision.JdkPayload(plan), java.discovery_script(cfg), + _Progress()) + phases = [entry[1] for entry in seen] + assert phases[:2] == ["probing", "sending"] + assert "extracting" in phases and "swapping" in phases + assert any(entry[1] == "sent" for entry in seen), \ + "a JDK is a few hundred megabytes; it has to show movement" + assert fake.reported, "the upload must be the watched one" + + def check_a_delivery_that_is_skipped_reports_nothing(self, tmp_path): + archive = _tarball(tmp_path, ["bin/java"]) + ctx, fake = _context(PROBE, major=21, archive=str(archive), install_root="/opt") + cfg = java.config_of(ctx) + plan = java.archive_plan(cfg.archive) + fake.when("cat", json.dumps({"hash": provision._tar_manifest(plan)["hash"]})) # noqa: SLF001 + provision._jdk_on_host( # noqa: SLF001 + ctx, NODE, cfg, provision.JdkPayload(plan), java.discovery_script(cfg)) + assert not fake.reported and not fake.uploads + def check_no_archive_and_no_match_fails_with_the_config_keys(self): ctx, _ = _context(PROBE, major=21) cfg = java.config_of(ctx) diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_progress.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_progress.py index 0b6ce9f5e494f..93814e2aacb6c 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_progress.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_progress.py @@ -18,9 +18,10 @@ import io import sys +from ducktests_remote import progress as progress_mod from ducktests_remote.commands import deploy -from ducktests_remote.progress import (NullProgress, Progress, human_bytes, human_duration, - is_a_terminal) +from ducktests_remote.progress import (NullProgress, Progress, build_progress, human_bytes, + human_duration, is_a_terminal) from ducktests_remote.transport import LocalTransport, run_local_streaming @@ -189,7 +190,7 @@ def check_a_non_terminal_is_never_live(self): class CheckWhenItIsUsed: - """deploy decides; the display only draws.""" + """The command decides; the display only draws.""" @staticmethod def _ctx(*, dry_run=False, quiet=False, verbose=False, no_progress=False): @@ -220,24 +221,24 @@ class _Node: # pylint: disable=too-few-public-methods return [_Node()] def check_a_dry_run_reports_nothing(self): - assert isinstance(deploy.build_progress(self._ctx(dry_run=True), self._nodes()), + assert isinstance(build_progress(self._ctx(dry_run=True), self._nodes()), NullProgress) def check_quiet_and_no_progress_report_nothing(self): - assert isinstance(deploy.build_progress(self._ctx(quiet=True), self._nodes()), + assert isinstance(build_progress(self._ctx(quiet=True), self._nodes()), NullProgress) - assert isinstance(deploy.build_progress(self._ctx(no_progress=True), self._nodes()), + assert isinstance(build_progress(self._ctx(no_progress=True), self._nodes()), NullProgress) def check_verbose_reports_without_redrawing(self, monkeypatch): - monkeypatch.setattr(deploy, "is_a_terminal", lambda _: True) - progress = deploy.build_progress(self._ctx(verbose=True), self._nodes()) + monkeypatch.setattr(progress_mod, "is_a_terminal", lambda _: True) + progress = build_progress(self._ctx(verbose=True), self._nodes()) assert isinstance(progress, Progress) and progress.live is False, \ "a redrawn block would fight with the traced command lines" def check_a_terminal_gets_the_live_display(self, monkeypatch): - monkeypatch.setattr(deploy, "is_a_terminal", lambda _: True) - assert deploy.build_progress(self._ctx(), self._nodes()).live is True + monkeypatch.setattr(progress_mod, "is_a_terminal", lambda _: True) + assert build_progress(self._ctx(), self._nodes()).live is True class CheckStreamedTransfers: diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_sources.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_sources.py index 53db575e46fa4..7a046a4914564 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_sources.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_sources.py @@ -174,3 +174,27 @@ def check_an_unresolvable_path_is_passed_through(self, tmp_path, in_dir): in_dir(str(tmp_path)) assert self._paths(_checkout(tmp_path), ["./ignitetest/tests/absent.py"]) == \ ["./ignitetest/tests/absent.py"] + + +class CheckSyncProgress: + """The 171 MB that goes to the runner before a run starts.""" + + class _Paths: # pylint: disable=too-few-public-methods + src_dir = "/state/runs/r/src" + + def _sync(self, tmp_path, **args): + root = _checkout(tmp_path) + args.setdefault("exclude", []) + ctx = _context(**args) + run._sync_sources(ctx, root, self._Paths()) # noqa: SLF001 + return ctx._runner # noqa: SLF001 + + def check_the_sync_is_watched(self, tmp_path): + runner = self._sync(tmp_path) + assert runner.uploads and runner.uploads[0][2] == "dir" + assert runner.reported, "a sync that takes minutes has to show movement" + + def check_no_progress_still_syncs(self, tmp_path): + runner = self._sync(tmp_path, no_progress=True) + assert runner.uploads, "the transfer happens either way" + assert not runner.reported, "--no-progress means nothing is asked to report" diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py index d367e38e59fe5..f329392f08952 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py @@ -23,6 +23,7 @@ from fake_transport import FakeTransport from ducktests_remote import cli, runs +from ducktests_remote import transport as transport_mod from ducktests_remote.transport import (LocalTransport, ProxiedTransport, Result, SshTransport, TransportError, is_excluded) @@ -188,3 +189,39 @@ class CheckResult: def check_ok_and_out(self): result = Result(["true"], 0, " value \n") assert result.ok and result.out == "value" + + +class CheckWatchedTransfers: + """A transfer only pays for progress when something is drawing it.""" + + @staticmethod + def _rsync_transport(monkeypatch, recorded): + transport = SshTransport(name="w1", user="tester") + monkeypatch.setattr(transport, "has_rsync", lambda: True) + monkeypatch.setattr(transport, "mkdirs", lambda *a, **kw: None) + + def record(argv, **kw): + recorded.append(list(argv)) + return Result(list(argv), 0, "", "", "w1") + + monkeypatch.setattr(transport_mod, "_spawn", record) + monkeypatch.setattr(transport_mod, "run_local_streaming", record) + return transport + + def check_an_unwatched_sync_keeps_the_quiet_rsync(self, tmp_path, monkeypatch): + recorded = [] + self._rsync_transport(monkeypatch, recorded).upload_dir(tmp_path, "/remote") + assert "--info=progress2" not in recorded[0] + + def check_a_watched_sync_asks_rsync_for_its_meter(self, tmp_path, monkeypatch): + recorded = [] + transport = self._rsync_transport(monkeypatch, recorded) + transport.upload_dir(tmp_path, "/remote", on_progress=lambda sent, fraction: None) + assert "--info=progress2" in recorded[0] + + def check_the_meter_is_turned_into_calls(self): + seen = [] + watch = transport_mod.rsync_reporter(lambda sent, fraction: seen.append((sent, fraction))) + watch(" 1,048,576 25% 12.34MB/s 0:00:12") + watch("sending incremental file list") + assert seen == [(1048576, 0.25)], "only meter lines count" diff --git a/modules/ducktests/tests/ducktests_remote/checks/fake_transport.py b/modules/ducktests/tests/ducktests_remote/checks/fake_transport.py index a3ffa5ae4223d..524d388a5c36e 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/fake_transport.py +++ b/modules/ducktests/tests/ducktests_remote/checks/fake_transport.py @@ -35,6 +35,7 @@ def __init__(self, name="fake", home="/home/tester", **kw): self.uploads = [] self.downloads = [] self.dirs = [] + self.reported = [] self.files = {} self.responses = [] self._home = home @@ -66,8 +67,19 @@ def upload(self, local_path, remote_path, *, mode=None): def download(self, remote_path, local_path): self.downloads.append((remote_path, str(local_path))) - def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False): + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False, + on_progress=None): self.uploads.append((str(local_dir), remote_dir, "dir")) + if on_progress is not None: + # One synthetic report, so a caller's wiring is exercised rather than assumed. + self.reported.append((1024, 1.0)) + on_progress(1024, 1.0) + + def upload_watched(self, local_path, remote_path, *, mode=None, on_bytes=None): + self.uploads.append((str(local_path), remote_path, mode)) + if on_bytes is not None: + self.reported.append((1024, None)) + on_bytes(1024) def write_file(self, content, remote_path, *, mode=None): self.files[remote_path] = (content, mode) diff --git a/modules/ducktests/tests/ducktests_remote/commands/deploy.py b/modules/ducktests/tests/ducktests_remote/commands/deploy.py index 7e25cc75bf15b..a4945afdda130 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/deploy.py +++ b/modules/ducktests/tests/ducktests_remote/commands/deploy.py @@ -48,7 +48,6 @@ import re import shlex import shutil -import sys import tempfile import threading import uuid @@ -58,9 +57,10 @@ from ducktests_remote.config import ConfigError, expand_path from ducktests_remote.fanout import (CHANGED, FAILED, HostResult, SKIPPED, any_failed, fanout, render_table, summarise) -from ducktests_remote.progress import NullProgress, Progress, is_a_terminal +from ducktests_remote.progress import NullProgress, build_progress from ducktests_remote.transport import (ProxiedTransport, is_excluded, make_tarball, - run_local, run_local_streaming) + parse_rsync_progress, run_local, + run_local_streaming) MANIFEST_NAME = ".ducktests-deploy.json" @@ -75,10 +75,6 @@ _RSYNC_TRANSFERRED = re.compile(r"Number of (?:regular )?files transferred:\s*([\d,.]+)") _RSYNC_SENT = re.compile(r"Total transferred file size:\s*([\d,.]+)") -# `--info=progress2`: ` 1,234,567 35% 12.34MB/s 0:00:12`. The separators are -# locale-dependent, the layout is not. -_RSYNC_PROGRESS = re.compile(r"^\s*([\d,.]+)\s+(\d+)%\s") - def register(subparsers, common): """Wire up the ``deploy`` subcommand.""" @@ -166,22 +162,6 @@ def execute(ctx): # pylint: disable=too-many-locals return EXIT_TRANSPORT if any_failed(overall) else EXIT_OK -def build_progress(ctx, nodes): - """ - :return: a :class:`Progress` for this deploy, or a :class:`NullProgress`. - - Off for ``--dry-run`` (nothing moves) and ``--quiet``. Live only on a terminal: - ``--verbose`` prints a traced command line per host, which a redrawn block would - fight with, so that combination reports one aggregate line every few seconds - instead - the same shape a CI log gets. - """ - if ctx.dry_run or ctx.console.quiet or getattr(ctx.args, "no_progress", False): - return NullProgress() - live = is_a_terminal(sys.stderr) and not ctx.console.verbose - return Progress([node.host for node in nodes], live=live, - redactor=ctx.console.redactor) - - def _distributions(dist_dir, only): names = sorted(p.name for p in dist_dir.iterdir() if p.is_dir() and not p.name.startswith(".")) @@ -401,17 +381,6 @@ def rsync_enabled(ctx): return shutil.which("rsync") is not None -def parse_rsync_progress(line): - """:return: ``(bytes_so_far, fraction)`` from one ``--info=progress2`` line, or None.""" - match = _RSYNC_PROGRESS.match(line or "") - if not match: - return None - try: - return int(re.sub(r"[,.]", "", match.group(1))), int(match.group(2)) / 100.0 - except ValueError: - return None - - def rsync_argv(transport, local_root, staging, *, files_from, link_dest=None, checksum=False, sudo=False, progress=False): """ @@ -550,7 +519,7 @@ def _rsync_to_host(ctx, node, transport, local_root, staging, target, file_list, progress = progress or NullProgress() transport.run_script(prepare_script(staging, ctx.args.sudo)).check() # Both display modes need the byte counts; only the display itself differs. - watched = not isinstance(progress, NullProgress) + watched = progress.watching argv = rsync_argv(transport, local_root, staging, files_from=file_list, link_dest=target if transport.exists(target) else None, checksum=ctx.args.checksum or ctx.config["deploy"].get("checksum", False), diff --git a/modules/ducktests/tests/ducktests_remote/commands/provision.py b/modules/ducktests/tests/ducktests_remote/commands/provision.py index 704dd14db831f..eacf8e2c548a5 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/provision.py +++ b/modules/ducktests/tests/ducktests_remote/commands/provision.py @@ -25,6 +25,7 @@ import hashlib import json +import os import posixpath import shlex import tempfile @@ -38,6 +39,7 @@ from ducktests_remote.config import ConfigError from ducktests_remote.fanout import (CHANGED, FAILED, HostResult, OK, SKIPPED, any_failed, fanout, render_table, summarise) +from ducktests_remote.progress import NullProgress, build_progress from ducktests_remote.transport import make_tarball STEPS = ("packages", "jdk", "python", "user", "ssh-env", "dirs", "hosts") @@ -252,17 +254,24 @@ def _run_jdk_step(ctx, nodes): script = java.discovery_script(cfg) payload = JdkPayload(plan) if plan else None + # Only a delivery is worth watching; the probe-only case is one round trip per host. + progress = build_progress(ctx, nodes) if payload else NullProgress() + def operation(node): if ctx.dry_run: ctx.console.out("[dry-run] %s: probe for a Java %s JDK" % (node.host, cfg.major or "any")) ctx.console.detail(script) return HostResult(node.host, SKIPPED, "dry-run") - return _jdk_on_host(ctx, node, cfg, payload, script) + try: + return _jdk_on_host(ctx, node, cfg, payload, script, progress) + finally: + progress.done(node.host) try: - return fanout(nodes, operation, jobs=ctx.jobs, - fail_fast=getattr(ctx.args, "fail_fast", False)) + with progress: + return fanout(nodes, operation, jobs=ctx.jobs, + fail_fast=getattr(ctx.args, "fail_fast", False)) finally: if payload: payload.close() @@ -321,8 +330,10 @@ def close(self): self._archive = None -def _jdk_on_host(ctx, node, cfg, payload, script): +def _jdk_on_host(ctx, node, cfg, payload, script, progress=None): + progress = progress or NullProgress() transport = ctx.worker(node) + progress.phase(node.host, "probing") probe = transport.run_script(script, check=False) if not probe.ok: return HostResult(node.host, FAILED, "could not probe for a JDK", @@ -340,7 +351,7 @@ def _jdk_on_host(ctx, node, cfg, payload, script): detail=_found(res)) if payload is not None: - return _deliver_jdk(ctx, node, cfg, payload) + return _deliver_jdk(ctx, node, cfg, payload, progress) if ctx.args.install_jdk: return _install_jdk(ctx, node, cfg) @@ -358,13 +369,14 @@ def _found(res): for home, major, _ in res.candidates) -def _deliver_jdk(ctx, node, cfg, payload): +def _deliver_jdk(ctx, node, cfg, payload, progress=None): """ Copy the JDK to one worker, reusing ``deploy``'s staging and atomic swap. A half-extracted JDK that looks present is exactly as bad as a half-extracted distribution, which is why this does not extract in place. """ + progress = progress or NullProgress() plan = payload.plan transport = ctx.worker(node) target = java.target_dir(cfg, plan) @@ -391,7 +403,14 @@ def _deliver_jdk(ctx, node, cfg, payload): transport.run_script(deploy.prepare_script(staging, ctx.args.sudo)).check() remote = "%s/.payload.tar" % staging - transport.upload(payload.archive(), remote) + archive = payload.archive() + progress.phase(node.host, "sending") + total = os.path.getsize(archive) + transport.upload_watched( + archive, remote, + on_bytes=(lambda sent: progress.sent(node.host, sent, total)) + if progress.watching else None) + progress.phase(node.host, "extracting") transport.run_script( "set -eu\ntar -x%sf %s -C %s%s\nrm -f -- %s\n" % (payload.tar_flag(), shlex.quote(remote), shlex.quote(staging), @@ -404,6 +423,7 @@ def _deliver_jdk(ctx, node, cfg, payload): return HostResult(node.host, FAILED, "the delivered archive has no bin/java under %s" % staging) + progress.phase(node.host, "swapping") transport.write_file(json.dumps(manifest, indent=2, sort_keys=True), posixpath.join(staging, JAVA_MANIFEST_NAME)) transport.run_script(deploy.swap_script(staging, target, ctx.args.sudo, None)).check() diff --git a/modules/ducktests/tests/ducktests_remote/commands/run.py b/modules/ducktests/tests/ducktests_remote/commands/run.py index 5a684369e22f8..6b64d32ff5713 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/run.py +++ b/modules/ducktests/tests/ducktests_remote/commands/run.py @@ -30,6 +30,7 @@ from ducktests_remote.cli import (EXIT_OK, EXIT_PREFLIGHT, EXIT_TESTS_FAILED, Console) from ducktests_remote.commands import doctor from ducktests_remote.config import ConfigError, expand_path +from ducktests_remote.progress import build_progress DEFAULT_EXCLUDES = [".git", "target", "results", "__pycache__", "*.pyc", ".idea", "venv", ".venv", "*.egg-info", ".tox", ".pytest_cache"] @@ -407,7 +408,17 @@ def _sync_sources(ctx, source_root, paths): "build directory leaked into the payload - check --exclude / %s. Distributions " "belong in `ducktests-remote deploy`, never in the source sync." % (size_mb, limit, IGNITE_IGNORE_FILE)) - ctx.runner.upload_dir(source_root, paths.src_dir, excludes=excludes) + total = int(size_mb * 1024 * 1024) + progress = build_progress(ctx, [ctx.runner_host]) + watcher = None + if progress.watching: + def watcher(sent, fraction): # noqa: F811 - reported only when something draws it + progress.sent(ctx.runner_host, sent, total, fraction=fraction) + with progress: + progress.phase(ctx.runner_host, "sending") + ctx.runner.upload_dir(source_root, paths.src_dir, excludes=excludes, + on_progress=watcher) + progress.done(ctx.runner_host) def _payload_size_mb(source_root, excludes): diff --git a/modules/ducktests/tests/ducktests_remote/docs/commands.md b/modules/ducktests/tests/ducktests_remote/docs/commands.md index 621777a4d8873..38334c51562f7 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/commands.md +++ b/modules/ducktests/tests/ducktests_remote/docs/commands.md @@ -114,7 +114,8 @@ to ducktape unchanged, with a warning. 8. **Sync the source tree** to `/src/` unless `--no-sync`. The payload is measured first and refused above `run.max_payload_mb` (200 MB): that almost always means a build directory leaked in. rsync when both ends have it, otherwise a tar stream - over scp. + over scp. Progress is reported while it runs — see + [deploy's description of the display](#watching-a-long-transfer), which is the same one. 9. **Ensure the venv**: create `/venv` when missing, and install `docker/requirements.txt` into it when `import ducktape` fails. This is where `pip.*` applies. Runs after the sync because the requirements file comes from the synced tree. @@ -220,6 +221,11 @@ assumption; a doctor FAIL makes it exit 2. `--dry-run` prints the exact script per host and probes nothing. +Delivering a JDK is the one step that moves real bytes, so it reports progress per host +while it does — the same display [`deploy` uses](#watching-a-long-transfer). The +probe-only case does not: it is one round trip per host, and only the hosts that come back +without a usable JDK are sent anything. + --- ## `deploy` @@ -281,8 +287,9 @@ w01 changed 4.1s rsync: 12 of 8431 file(s) changed, 4.0 MB sent ### Watching a long transfer Sending 3.5 GB to twelve machines takes minutes, and a terminal that prints nothing for -that long is indistinguishable from one that has hung. `deploy` shows every host while it -works, redrawn in place: +that long is indistinguishable from one that has hung. Every transfer that can take +minutes reports the same way — `deploy`, `run`'s source sync, and `provision`'s JDK +delivery — redrawn in place: ``` worker01 █████████████████░░░░░░░ 70% 210.0 MB / 300.0 MB 24.1 MB/s @@ -313,6 +320,10 @@ counts the chunks itself. `--verbose` deliberately drops to the single line: it prints a traced command per host, and a redrawn block would fight with it. +Counting bytes is not free — rsync grows a meter, and a watched upload streams through +`ssh 'cat > path'` instead of scp — so when nothing will draw the result, nothing is asked +to report and the plain transfer is used. That is what `--no-progress` really turns off. + The tarball path is used instead when: | Condition | Why | diff --git a/modules/ducktests/tests/ducktests_remote/docs/internals.md b/modules/ducktests/tests/ducktests_remote/docs/internals.md index 845070004fddd..51b5c4d06996d 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/internals.md +++ b/modules/ducktests/tests/ducktests_remote/docs/internals.md @@ -123,6 +123,12 @@ pipe. `run_local_streaming` reads a child's output as it arrives, treating a car return as a line ending, and drains stderr on a thread so a chatty failure cannot deadlock the pipe. +`build_progress(ctx, hosts)` is the single decision point, shared by `deploy`, +`run`'s source sync and `provision`'s JDK delivery. Callers gate the wiring on +`progress.watching` rather than on which object they got: reporting costs something — +rsync grows a meter, an upload leaves scp for a streamed `cat` — and none of it is worth +paying when the display is a `NullProgress`. + ## Redaction `Redactor` keys on resolved **values**, not on key names. Anything coming out of `${env:}` diff --git a/modules/ducktests/tests/ducktests_remote/progress.py b/modules/ducktests/tests/ducktests_remote/progress.py index c2681cddac63d..0ecd93655005a 100644 --- a/modules/ducktests/tests/ducktests_remote/progress.py +++ b/modules/ducktests/tests/ducktests_remote/progress.py @@ -98,6 +98,11 @@ class Progress: so a stream that cannot be written to simply stops being written to. """ + # Callers ask this before wiring a transfer up to report: counting bytes has a cost + # (rsync grows a meter, an upload streams through ssh instead of scp) that is only + # worth paying when something will draw the result. + watching = True + def __init__(self, hosts, *, stream=None, live=None, redactor=None, interval=0.25, plain_interval=15.0, max_rows=12, unicode_bar=None): self.stream = stream if stream is not None else sys.stderr @@ -304,6 +309,7 @@ class NullProgress: """The same surface, doing nothing. Callers never test for None.""" live = False + watching = False def start(self, title=""): """:return: self, so ``with`` reads the same either way.""" @@ -328,6 +334,24 @@ def done(self, host, message=""): """Ignore the update.""" +def build_progress(ctx, hosts): + """ + :return: a :class:`Progress` for this command, or a :class:`NullProgress`. + + Off for ``--dry-run`` (nothing moves), ``--quiet`` and ``--no-progress``. Live only + on a terminal: ``--verbose`` prints a traced command line per host, which a redrawn + block would fight with, so that combination reports one aggregate line every few + seconds instead - the same shape a CI log gets. + """ + if ctx.dry_run or ctx.console.quiet or getattr(ctx.args, "no_progress", False): + return NullProgress() + names = [getattr(host, "host", host) for host in hosts] + if not names: + return NullProgress() + return Progress(names, live=is_a_terminal(sys.stderr) and not ctx.console.verbose, + redactor=ctx.console.redactor) + + def is_a_terminal(stream): """:return: True when ``stream`` can be redrawn in place.""" try: diff --git a/modules/ducktests/tests/ducktests_remote/transport.py b/modules/ducktests/tests/ducktests_remote/transport.py index 33afdc2113eb6..76f9f4df8da88 100644 --- a/modules/ducktests/tests/ducktests_remote/transport.py +++ b/modules/ducktests/tests/ducktests_remote/transport.py @@ -24,6 +24,7 @@ import abc import os import posixpath +import re import shlex import shutil import subprocess @@ -164,8 +165,15 @@ def download(self, remote_path, local_path): """Copy a single remote file to ``local_path``.""" @abc.abstractmethod - def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False): - """Copy a directory tree; ``local_dir`` contents land inside ``remote_dir``.""" + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False, + on_progress=None): + """ + Copy a directory tree; ``local_dir`` contents land inside ``remote_dir``. + + ``on_progress(sent_bytes, fraction)`` is called as the transfer goes, with + ``fraction`` set only when the sender knows it. Implementations that cannot see + the bytes leave simply never call it. + """ def exists(self, remote_path): """:return: True when ``remote_path`` exists on this host.""" @@ -248,7 +256,9 @@ def download(self, remote_path, local_path): Path(local_path).parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(remote_path, str(local_path)) - def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False): + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False, + on_progress=None): + del on_progress # a local copy is not worth watching self._trace(["cp", "-r", str(local_dir), remote_dir]) if self.dry_run: return @@ -361,7 +371,8 @@ def has_rsync(self): self._rsync = bool(remote) return self._rsync - def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False): + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False, + on_progress=None): local_dir = str(local_dir) self._trace(["rsync", local_dir, "%s:%s" % (self.target, remote_dir)]) if self.dry_run: @@ -374,14 +385,24 @@ def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False): for pattern in excludes: argv += ["--exclude", pattern] argv += [local_dir.rstrip("/\\") + "/", "%s:%s/" % (self.target, remote_dir)] - _spawn(argv, host=self.name, check=True) + if on_progress is None: + _spawn(argv, host=self.name, check=True) + return + argv.insert(1, "--info=progress2") + run_local_streaming(argv, host=self.name, + on_output=rsync_reporter(on_progress)).check() return # rsync missing on one of the ends: fall back to a tar stream through scp. with tempfile.TemporaryDirectory() as tmp: archive = Path(tmp) / "payload.tar.gz" make_tarball(local_dir, archive, excludes=excludes) staged = "/tmp/dtr-upload-%s.tar.gz" % uuid.uuid4().hex[:8] - self.upload(archive, staged) + if on_progress is None: + self.upload(archive, staged) + else: + total = archive.stat().st_size + self.upload_watched(archive, staged, + on_bytes=lambda sent: on_progress(sent, sent / total)) script = "set -eu\nmkdir -p %s\n" % shlex.quote(remote_dir) if delete: script += "rm -rf -- %s/*\n" % shlex.quote(remote_dir) @@ -455,7 +476,9 @@ def download(self, remote_path, local_path): finally: self.via.run(["rm", "-f", "--", staged], check=False) - def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False): + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False, + on_progress=None): + del on_progress # the interesting hop is the second one, which scp cannot watch with tempfile.TemporaryDirectory() as tmp: archive = Path(tmp) / "payload.tar.gz" make_tarball(local_dir, archive, excludes=excludes) @@ -547,6 +570,30 @@ def run_local(argv, *, check=False, timeout=None, input=None): # noqa: A002 - m CHUNK = 1024 * 1024 +# `--info=progress2`: ` 1,234,567 35% 12.34MB/s 0:00:12`. The separators are +# locale-dependent, the layout is not. +_RSYNC_PROGRESS = re.compile(r"^\s*([\d,.]+)\s+(\d+)%\s") + + +def parse_rsync_progress(line): + """:return: ``(bytes_so_far, fraction)`` from one ``--info=progress2`` line, or None.""" + match = _RSYNC_PROGRESS.match(line or "") + if not match: + return None + try: + return int(re.sub(r"[,.]", "", match.group(1))), int(match.group(2)) / 100.0 + except ValueError: + return None + + +def rsync_reporter(on_progress): + """:return: an ``on_output`` callback that turns rsync's meter into ``on_progress``.""" + def watch(line): + reading = parse_rsync_progress(line) + if reading: + on_progress(reading[0], reading[1]) + return watch + def run_local_streaming(argv, *, on_output, host="local"): """ From 48718d1d7301ed4908bb51078b20e335e738e598 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 29 Jul 2026 18:08:01 +0300 Subject: [PATCH 9/9] Stop reporting a finished transfer as a failed one A watched upload feeds the payload into `ssh cat > path` itself, because scp hides its meter when stdout is a pipe. It closed stdin to signal end of file and then called communicate(), which flushes stdin before closing it and guards only BrokenPipeError - so flushing the handle we had just closed raised ValueError('flush of closed file'). Every host failed at exactly 100%, with the payload already on the far end and a staging directory left holding it. Do the bookkeeping here instead: drain both output pipes on threads for the whole transfer, and close stdin once. The draining is not incidental. Nothing read those pipes while a gigabyte was being written, so an ssh that says anything on the way in - a login banner, a host-key notice - filled its pipe and stopped the transfer for good. Staging directories are now discarded on any exit that is not a successful swap, in deploy and in provision's jdk step, and prepare_script sweeps what an earlier attempt left. Each holds a whole copy of the distribution under a dot-prefixed name with a random suffix, so nothing would ever reclaim them. The only checks of upload_watched went through LocalTransport, which never reaches this code. The new ones run a real child process. --- .../checks/check_remote_deploy.py | 97 +++++++++++++ .../checks/check_remote_transport.py | 70 ++++++++++ .../tests/ducktests_remote/commands/deploy.py | 128 ++++++++++++------ .../ducktests_remote/commands/provision.py | 65 +++++---- .../tests/ducktests_remote/docs/commands.md | 6 + .../tests/ducktests_remote/docs/java.md | 5 +- .../tests/ducktests_remote/transport.py | 57 ++++++-- 7 files changed, 347 insertions(+), 81 deletions(-) diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py index aae3c18fd301a..f04fe0b6242ec 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py @@ -40,6 +40,13 @@ def _dist(tmp_path, name="ignite-dev", body="binary"): return root +def _staging_of(transport): + """:return: the staging path a recorded deploy created on ``transport``.""" + prepared = [s for s in transport.scripts if "mkdir -p" in s] + assert prepared, "the deploy never got as far as making a staging directory" + return prepared[0].split("mkdir -p ")[1].strip() + + class CheckManifest: """Skip-if-unchanged.""" @@ -106,6 +113,96 @@ def check_sudo_prefixes_every_privileged_command(self): assert "chown -R max" in script +class CheckStagingLifecycle: + """ + A staging directory holds a whole copy of the distribution and hides behind a dot. + + Deploys of a gigabyte tree to fourteen hosts have failed after the transfer and left + one of these per attempt per host, where nothing looks for them. + """ + + @staticmethod + def _ctx(dist_root, transport): + class _Args: # pylint: disable=too-few-public-methods + force = False + sudo = False + owner = None + checksum = False + via = None + + class _Console: # pylint: disable=too-few-public-methods + verbose = False + + @staticmethod + def detail(_message): + """Swallow the traced command line.""" + + class _Ctx: # pylint: disable=too-few-public-methods + args = _Args() + console = _Console() + config = {"deploy": DEFAULTS["deploy"]} + dry_run = False + dist_root = None + + @staticmethod + def worker(_node): + return transport + + ctx = _Ctx() + ctx.dist_root = dist_root + return ctx + + def check_preparing_sweeps_what_an_earlier_attempt_left(self): + sweep = deploy.staging_prefix("/opt", "ignite-dev") + script = deploy.prepare_script(sweep + "abc123", False, sweep=sweep) + assert "rm -rf -- /opt/.ignite-dev.tmp.*" in script, \ + "every attempt picks a new suffix, so only a glob reclaims the old ones" + assert script.index("rm -rf") < script.index("mkdir -p") + + def check_the_sweep_is_scoped_to_one_distribution(self): + script = deploy.prepare_script("/opt/.ignite-dev.tmp.abc", False, + sweep=deploy.staging_prefix("/opt", "ignite-dev")) + for line in script.splitlines(): + if line.startswith("rm -rf"): + assert "ignite-dev" in line, "a glob here would empty the install root" + + def check_a_name_needing_quotes_still_globs(self): + sweep = deploy.staging_prefix("/opt", "fork 2.8") + script = deploy.prepare_script(sweep + "abc", False, sweep=sweep) + assert "'/opt/.fork 2.8.tmp.'*" in script, \ + "the star must stay outside the quotes or nothing is swept" + + def check_a_failed_transfer_takes_its_staging_directory_with_it(self, tmp_path): + root = _dist(tmp_path) + transport = FakeTransport() + transport.when("tar -xzf", returncode=1, stderr="unexpected end of file") + ctx = self._ctx(root, transport) + payload = deploy._Payload(root, tmp_path / "p.tar.gz", ()) # noqa: SLF001 + + with pytest.raises(Exception): # noqa: B017 - any failure must still clean up + deploy._deploy_to_host( # noqa: SLF001 + ctx, Node(host="w1"), "ignite-dev", "/opt/ignite-dev", + deploy.build_manifest(root), "{}", payload, None, None) + + assert deploy.discard_script(_staging_of(transport), False) in transport.scripts, \ + "the staging tree outlived the deploy that made it" + + def check_a_successful_deploy_does_not_remove_the_swapped_tree(self, tmp_path): + root = _dist(tmp_path) + transport = FakeTransport() + ctx = self._ctx(root, transport) + payload = deploy._Payload(root, tmp_path / "p.tar.gz", ()) # noqa: SLF001 + + result = deploy._deploy_to_host( # noqa: SLF001 + ctx, Node(host="w1"), "ignite-dev", "/opt/ignite-dev", + deploy.build_manifest(root), "{}", payload, None, None) + + assert result.status == CHANGED + assert any('mv -- "$staging" "$target"' in s for s in transport.scripts) + assert deploy.discard_script(_staging_of(transport), False) not in transport.scripts, \ + "after the swap the staging path is the distribution; removing it deletes it" + + class CheckCleanAllowList: """A bug here deletes distributions across every machine at once.""" diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py index f329392f08952..88d8dcb6c3f69 100644 --- a/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py @@ -15,8 +15,10 @@ """Checks for the transport boundary, the import guard, and exit-code mapping.""" +import os import subprocess import sys +import threading from pathlib import Path import pytest @@ -225,3 +227,71 @@ def check_the_meter_is_turned_into_calls(self): watch(" 1,048,576 25% 12.34MB/s 0:00:12") watch("sending incremental file list") assert seen == [(1048576, 0.25)], "only meter lines count" + + +class CheckStreamedUpload: + """ + A watched upload really runs a child process, so these checks really run one too. + + The path they cover broke in a way no mock could have shown: the payload reached the + far end intact and the transfer was then reported as failed, because ``communicate`` + flushed a stdin handle the caller had already closed. Nothing short of a live + subprocess sees that, and it is worth the second these take. + """ + + @staticmethod + def _child(body, *args): + return [sys.executable, "-c", body] + list(args) + + def check_the_whole_payload_arrives_and_every_byte_is_reported(self, tmp_path): + source = tmp_path / "payload.bin" + source.write_bytes(os.urandom(3 * 1024 * 1024 + 17)) + landed = tmp_path / "landed.bin" + seen = [] + argv = self._child("import sys\n" + "open(sys.argv[1], 'wb').write(sys.stdin.buffer.read())\n", + str(landed)) + + result = transport_mod._stream_to_stdin(argv, source, seen.append, host="w1") + + assert result.ok + assert landed.read_bytes() == source.read_bytes() + assert seen[-1] == source.stat().st_size, "the last report is the whole file" + + def check_a_chatty_command_does_not_wedge_the_transfer(self, tmp_path): + # ssh talks on the way in - a login banner, a host-key notice - and its output + # pipe holds 64 KB. With nobody draining it the child blocks on its own write + # while we block on ours, and the transfer stops for good at some arbitrary + # percentage. Run it on a thread so a regression fails the check instead of + # hanging the suite. + source = tmp_path / "payload.bin" + source.write_bytes(b"x" * (2 * 1024 * 1024)) + landed = tmp_path / "landed.bin" + argv = self._child("import sys\n" + "sys.stderr.write('noise\\n' * 40000)\n" + "sys.stderr.flush()\n" + "open(sys.argv[1], 'wb').write(sys.stdin.buffer.read())\n", + str(landed)) + + done = [] + worker = threading.Thread( + target=lambda: done.append( + transport_mod._stream_to_stdin(argv, source, lambda sent: None, host="w1")), + daemon=True) + worker.start() + worker.join(timeout=60) + + assert not worker.is_alive(), "the transfer deadlocked on an undrained output pipe" + assert done[0].ok and landed.stat().st_size == source.stat().st_size + + def check_a_failing_command_reports_its_stderr(self, tmp_path): + source = tmp_path / "payload.bin" + source.write_bytes(b"payload") + argv = self._child("import sys\n" + "sys.stderr.write('no such file or directory\\n')\n" + "sys.exit(1)\n") + + with pytest.raises(TransportError) as raised: + transport_mod._stream_to_stdin(argv, source, lambda sent: None, host="w1") + + assert "no such file or directory" in str(raised.value) diff --git a/modules/ducktests/tests/ducktests_remote/commands/deploy.py b/modules/ducktests/tests/ducktests_remote/commands/deploy.py index a4945afdda130..20ad5c135c815 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/deploy.py +++ b/modules/ducktests/tests/ducktests_remote/commands/deploy.py @@ -36,9 +36,15 @@ Long transfers report progress per host; see :mod:`ducktests_remote.progress`. -:func:`build_manifest`, :func:`prepare_script`, :func:`swap_script` and :func:`human` are -public because ``provision``'s ``jdk`` step delivers a JDK the same way and must not grow -a second copy of the staging-and-swap logic. +Nothing is written to the live path until the whole distribution is staged beside it, and +a staging directory that is not swapped in is removed on the way out - including the +leftovers of an earlier attempt, which carry a random suffix and a leading dot and would +otherwise sit in the install root forever. + +:func:`build_manifest`, :func:`staging_prefix`, :func:`prepare_script`, +:func:`discard_script`, :func:`swap_script` and :func:`human` are public because +``provision``'s ``jdk`` step delivers a JDK the same way and must not grow a second copy +of the staging-and-swap logic. """ import hashlib @@ -450,43 +456,55 @@ def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, payload, "%s is not writable by %s and --sudo was not passed" % (install_root, node.user or "this account")) - staging = "%s/.%s.tmp.%s" % (install_root, name, uuid.uuid4().hex[:8]) + sweep = staging_prefix(install_root, name) + staging = "%s%s" % (sweep, uuid.uuid4().hex[:8]) used_rsync = False stats = None + swapped = False + + try: + if via_transport is not None: + progress.phase(node.host, "fanning out") + proxied = ProxiedTransport(name=node.host, via=via_transport, user=node.user, + port=node.port, + identity_file=node.identity_file, + staging_dir=ctx.config["deploy"]["staging_dir"], + dry_run=ctx.dry_run, verbose=ctx.console.verbose) + proxied.run_script(prepare_script(staging, ctx.args.sudo, sweep=sweep)).check() + proxied.push_archive(staged_on_via, staging) + elif file_list is not None and getattr(transport, "has_rsync", _no_rsync)(): + outcome = _rsync_to_host(ctx, node, transport, payload.root, staging, target, + file_list, progress, sweep) + if isinstance(outcome, HostResult): + progress.done(node.host, "failed") + return outcome + used_rsync, stats = True, outcome + else: + transport.run_script(prepare_script(staging, ctx.args.sudo, sweep=sweep)).check() + remote_archive = "%s/.payload.tar.gz" % staging + archive = payload.archive() + progress.phase(node.host, "sending") + transport.upload_watched(archive, remote_archive, + on_bytes=_watcher(progress, node.host, + os.path.getsize(archive))) + progress.phase(node.host, "extracting") + transport.run_script( + "set -eu\ntar -xzf %s -C %s\nrm -f -- %s\n" + % (shlex.quote(remote_archive), shlex.quote(staging), + shlex.quote(remote_archive))).check() + + progress.phase(node.host, "swapping") + transport.write_file(manifest_body, posixpath.join(staging, MANIFEST_NAME)) + transport.run_script(swap_script(staging, target, ctx.args.sudo, + ctx.args.owner)).check() + swapped = True + finally: + # A staging tree holds a whole copy of the distribution. Once the swap has moved + # it into place there is nothing left at this path; before that, any way out of + # here - a failed host, a transport error, Ctrl-C - has to take it with it. + if not swapped: + transport.run_script(discard_script(staging, ctx.args.sudo), check=False) - if via_transport is not None: - progress.phase(node.host, "fanning out") - proxied = ProxiedTransport(name=node.host, via=via_transport, user=node.user, - port=node.port, - identity_file=node.identity_file, - staging_dir=ctx.config["deploy"]["staging_dir"], - dry_run=ctx.dry_run, verbose=ctx.console.verbose) - proxied.run_script(prepare_script(staging, ctx.args.sudo)).check() - proxied.push_archive(staged_on_via, staging) - elif file_list is not None and getattr(transport, "has_rsync", _no_rsync)(): - outcome = _rsync_to_host(ctx, node, transport, payload.root, staging, target, - file_list, progress) - if isinstance(outcome, HostResult): - progress.done(node.host, "failed") - return outcome - used_rsync, stats = True, outcome - else: - transport.run_script(prepare_script(staging, ctx.args.sudo)).check() - remote_archive = "%s/.payload.tar.gz" % staging - archive = payload.archive() - progress.phase(node.host, "sending") - transport.upload_watched(archive, remote_archive, - on_bytes=_watcher(progress, node.host, - os.path.getsize(archive))) - progress.phase(node.host, "extracting") - transport.run_script( - "set -eu\ntar -xzf %s -C %s\nrm -f -- %s\n" - % (shlex.quote(remote_archive), shlex.quote(staging), - shlex.quote(remote_archive))).check() - - progress.phase(node.host, "swapping") - transport.write_file(manifest_body, posixpath.join(staging, MANIFEST_NAME)) - transport.run_script(swap_script(staging, target, ctx.args.sudo, ctx.args.owner)).check() progress.done(node.host) if used_rsync: if stats: @@ -508,7 +526,7 @@ def _watcher(progress, host, total): def _rsync_to_host(ctx, node, transport, local_root, staging, target, file_list, - progress=None): + progress=None, sweep=None): """ :return: ``(files, bytes)`` transferred, ``None`` when ``--stats`` could not be read, or a failed :class:`HostResult`. @@ -517,7 +535,7 @@ def _rsync_to_host(ctx, node, transport, local_root, staging, target, file_list, distribution when there is one to link against. """ progress = progress or NullProgress() - transport.run_script(prepare_script(staging, ctx.args.sudo)).check() + transport.run_script(prepare_script(staging, ctx.args.sudo, sweep=sweep)).check() # Both display modes need the byte counts; only the display itself differs. watched = progress.watching argv = rsync_argv(transport, local_root, staging, files_from=file_list, @@ -546,12 +564,40 @@ def watch(line): return watch -def prepare_script(staging, use_sudo): +def staging_prefix(install_root, name): + """ + :return: the path prefix shared by every staging directory for one distribution. + + A random suffix is appended to it per attempt. Keeping the prefix in one function is + what lets :func:`prepare_script` recognise the leftovers of an earlier attempt. + """ + return "%s/.%s.tmp." % (install_root, name) + + +def prepare_script(staging, use_sudo, sweep=None): + """ + Make an empty staging directory to fill. + + ``sweep`` is the path prefix every staging directory for this distribution shares. + Anything matching it is removed first: each attempt picks a fresh random suffix, so a + deploy killed between the transfer and the swap would otherwise leave a full copy of + the distribution behind that nothing ever looks at again - and a dot-prefixed name at + that, invisible to a plain ``ls``. + """ sudo = "sudo -n " if use_sudo else "" - return "set -eu\n%(sudo)srm -rf -- %(staging)s\n%(sudo)smkdir -p %(staging)s\n" % { + script = "set -eu\n" + if sweep: + script += "%srm -rf -- %s*\n" % (sudo, shlex.quote(sweep)) + return script + "%(sudo)srm -rf -- %(staging)s\n%(sudo)smkdir -p %(staging)s\n" % { "sudo": sudo, "staging": shlex.quote(staging)} +def discard_script(staging, use_sudo): + """:return: a script removing a staging directory that will never be swapped in.""" + return "set -eu\n%srm -rf -- %s\n" % ("sudo -n " if use_sudo else "", + shlex.quote(staging)) + + def swap_script(staging, target, use_sudo, owner): """ Swap the freshly extracted tree into place, then delete the old one. diff --git a/modules/ducktests/tests/ducktests_remote/commands/provision.py b/modules/ducktests/tests/ducktests_remote/commands/provision.py index eacf8e2c548a5..3f0573871c0c0 100644 --- a/modules/ducktests/tests/ducktests_remote/commands/provision.py +++ b/modules/ducktests/tests/ducktests_remote/commands/provision.py @@ -398,35 +398,44 @@ def _deliver_jdk(ctx, node, cfg, payload, progress=None): "%s is not writable by %s and --sudo was not passed" % (install_root, node.user or "this account")) - staging = "%s/.%s.tmp.%s" % (install_root, posixpath.basename(target), - uuid.uuid4().hex[:8]) - transport.run_script(deploy.prepare_script(staging, ctx.args.sudo)).check() - - remote = "%s/.payload.tar" % staging - archive = payload.archive() - progress.phase(node.host, "sending") - total = os.path.getsize(archive) - transport.upload_watched( - archive, remote, - on_bytes=(lambda sent: progress.sent(node.host, sent, total)) - if progress.watching else None) - progress.phase(node.host, "extracting") - transport.run_script( - "set -eu\ntar -x%sf %s -C %s%s\nrm -f -- %s\n" - % (payload.tar_flag(), shlex.quote(remote), shlex.quote(staging), - " --strip-components=%d" % payload.strip() if payload.strip() else "", - shlex.quote(remote))).check() - - check = transport.run(["test", "-x", "%s/bin/java" % staging], check=False) - if not check.ok: - transport.run(["rm", "-rf", "--", staging], check=False) - return HostResult(node.host, FAILED, - "the delivered archive has no bin/java under %s" % staging) + sweep = deploy.staging_prefix(install_root, posixpath.basename(target)) + staging = "%s%s" % (sweep, uuid.uuid4().hex[:8]) + swapped = False + + try: + transport.run_script(deploy.prepare_script(staging, ctx.args.sudo, sweep=sweep)).check() + + remote = "%s/.payload.tar" % staging + archive = payload.archive() + progress.phase(node.host, "sending") + total = os.path.getsize(archive) + transport.upload_watched( + archive, remote, + on_bytes=(lambda sent: progress.sent(node.host, sent, total)) + if progress.watching else None) + progress.phase(node.host, "extracting") + transport.run_script( + "set -eu\ntar -x%sf %s -C %s%s\nrm -f -- %s\n" + % (payload.tar_flag(), shlex.quote(remote), shlex.quote(staging), + " --strip-components=%d" % payload.strip() if payload.strip() else "", + shlex.quote(remote))).check() + + check = transport.run(["test", "-x", "%s/bin/java" % staging], check=False) + if not check.ok: + return HostResult(node.host, FAILED, + "the delivered archive has no bin/java under %s" % staging) + + progress.phase(node.host, "swapping") + transport.write_file(json.dumps(manifest, indent=2, sort_keys=True), + posixpath.join(staging, JAVA_MANIFEST_NAME)) + transport.run_script(deploy.swap_script(staging, target, ctx.args.sudo, None)).check() + swapped = True + finally: + # Same rule as deploy: a staging tree that is not swapped in is dead weight, and + # its dot-prefixed name means nobody would ever notice it accumulating. + if not swapped: + transport.run_script(deploy.discard_script(staging, ctx.args.sudo), check=False) - progress.phase(node.host, "swapping") - transport.write_file(json.dumps(manifest, indent=2, sort_keys=True), - posixpath.join(staging, JAVA_MANIFEST_NAME)) - transport.run_script(deploy.swap_script(staging, target, ctx.args.sudo, None)).check() return HostResult(node.host, CHANGED, "delivered %s to %s" % (deploy.human(plan.bytes), target)) diff --git a/modules/ducktests/tests/ducktests_remote/docs/commands.md b/modules/ducktests/tests/ducktests_remote/docs/commands.md index 38334c51562f7..0dfc6db103929 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/commands.md +++ b/modules/ducktests/tests/ducktests_remote/docs/commands.md @@ -251,6 +251,12 @@ Per distribution, per host: delete the old tree. A half-copied distribution that looks present is worse than an absent one. +A staging directory is `/..tmp.`, and holds a whole copy of +the distribution until the swap. One that is never swapped in — a failed host, an +interrupted run — is removed on the way out, and any left by an earlier attempt are swept +before the next one starts. It is scoped to the distribution being deployed: `deploy +ignite-dev` never touches `.ignite-2.17.0.tmp.*`. + `--via HOST` uploads the payload once to an intermediate host and fans out from there. `deploy` prints the total bytes before it starts — on a twelve-host cluster a 300 MB distribution is 3.7 GB from a laptop — and suggests `--via` when that total is large. diff --git a/modules/ducktests/tests/ducktests_remote/docs/java.md b/modules/ducktests/tests/ducktests_remote/docs/java.md index 1f57ed3616194..01ba4b67caac5 100644 --- a/modules/ducktests/tests/ducktests_remote/docs/java.md +++ b/modules/ducktests/tests/ducktests_remote/docs/java.md @@ -108,11 +108,12 @@ So on a twelve-host cluster where eleven already carry Java 17, exactly one uplo 3. Create a staging directory beside the target, upload the archive into it, and unpack — with `--strip-components` set to whatever wraps the JDK home (1 for a stock Temurin tarball) and the decompression flag taken from the file's suffix. -4. Verify `bin/java` exists in the staging tree; if not, remove the staging tree and fail. +4. Verify `bin/java` exists in the staging tree; if not, fail. 5. Write the manifest, then **swap** the staging tree into place and delete the old one. The staging-and-swap is `deploy`'s, reused rather than reimplemented: a half-extracted JDK -that looks present is exactly as bad as a half-extracted distribution. +that looks present is exactly as bad as a half-extracted distribution, and a staging tree +that is not swapped in is discarded the same way. ### Archive formats diff --git a/modules/ducktests/tests/ducktests_remote/transport.py b/modules/ducktests/tests/ducktests_remote/transport.py index 76f9f4df8da88..f9bf8307a370a 100644 --- a/modules/ducktests/tests/ducktests_remote/transport.py +++ b/modules/ducktests/tests/ducktests_remote/transport.py @@ -615,10 +615,8 @@ def run_local_streaming(argv, *, on_output, host="local"): except FileNotFoundError as ex: raise TransportError("%s: %s" % (argv[0], ex)) from ex - errors = [] - drain = threading.Thread(target=lambda: errors.append(_decode(process.stderr.read())), - daemon=True) - drain.start() + errors = {} + drain = _drain(process.stderr, "stderr", errors) collected = [] pending = "" @@ -635,8 +633,7 @@ def run_local_streaming(argv, *, on_output, host="local"): process.wait() drain.join(timeout=5) - return Result(argv, process.returncode, "".join(collected), - errors[0] if errors else "", host) + return Result(argv, process.returncode, "".join(collected), errors.get("stderr", ""), host) def _emit_lines(pending, on_output): @@ -651,8 +648,34 @@ def _emit_lines(pending, on_output): on_output(line.strip()) +def _drain(pipe, key, into): + """:return: a started daemon thread reading ``pipe`` to EOF into ``into[key]``.""" + def read(): + try: + into[key] = _decode(pipe.read()) + except (OSError, ValueError): + into[key] = "" + thread = threading.Thread(target=read, daemon=True) + thread.start() + return thread + + def _stream_to_stdin(argv, local_path, on_bytes, *, host): - """Feed a file into ``argv``'s stdin in chunks, reporting the running total.""" + """ + Feed a file into ``argv``'s stdin in chunks, reporting the running total. + + Both output pipes are drained by threads for the whole transfer rather than collected + at the end by ``communicate``. Two reasons, and the second one is why this function + does its own bookkeeping instead of writing four lines: + + * Nothing would read them while a multi-gigabyte payload is being written, and an ssh + that says anything at all - a login banner, a host-key notice - fills its pipe and + wedges the write part way through, with no error and no end. + * ``communicate`` flushes stdin before closing it, and guards only ``BrokenPipeError`` + while doing so. A handle closed here would make that flush raise + ``ValueError('flush of closed file')`` - after the payload is already on the far + end, so a transfer that fully succeeded is reported as a failure. + """ try: # pylint: disable=consider-using-with process = subprocess.Popen( # noqa: S603 - argv is always a list, never a string @@ -660,6 +683,10 @@ def _stream_to_stdin(argv, local_path, on_bytes, *, host): except FileNotFoundError as ex: raise TransportError("%s: %s" % (argv[0], ex)) from ex + output = {} + readers = [_drain(process.stdout, "stdout", output), + _drain(process.stderr, "stderr", output)] + sent = 0 try: with open(local_path, "rb") as handle: @@ -667,12 +694,22 @@ def _stream_to_stdin(argv, local_path, on_bytes, *, host): process.stdin.write(block) sent += len(block) on_bytes(sent) - process.stdin.close() except (BrokenPipeError, OSError): # The far end died mid-transfer; its stderr says why, so fall through to wait(). pass - _, stderr = process.communicate() - result = Result(argv, process.returncode, "", _decode(stderr), host) + finally: + # The close is what tells the remote `cat` that the file is over, so it has to + # happen even when the write above failed. + try: + process.stdin.close() + except (BrokenPipeError, OSError): + pass + + process.wait() + for reader in readers: + reader.join(timeout=5) + result = Result(argv, process.returncode, output.get("stdout", ""), + output.get("stderr", ""), host) return result.check()