diff --git a/modules/ducktests/tests/MANIFEST.in b/modules/ducktests/tests/MANIFEST.in index 6fcceb37144dd..523152cba3824 100644 --- a/modules/ducktests/tests/MANIFEST.in +++ b/modules/ducktests/tests/MANIFEST.in @@ -14,3 +14,5 @@ # limitations under the License. recursive-include ignitetest **.j2 +recursive-include ducktests_remote/templates *.tmpl +recursive-include ducktests_remote/examples *.yaml diff --git a/modules/ducktests/tests/docker/requirements.txt b/modules/ducktests/tests/docker/requirements.txt index aba4f25a86d1d..57492847d3c99 100644 --- a/modules/ducktests/tests/docker/requirements.txt +++ b/modules/ducktests/tests/docker/requirements.txt @@ -16,3 +16,6 @@ filelock==3.8.2 ducktape==0.13.0 looseversion==1.3.0 +# ducktests_remote (the `ducktests-remote` CLI) parses YAML config and profiles. +# It is the only runtime dependency that package has beyond the standard library. +PyYAML==6.0.2 diff --git a/modules/ducktests/tests/ducktests_remote/README.md b/modules/ducktests/tests/ducktests_remote/README.md new file mode 100644 index 0000000000000..c70e419f31b2c --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/README.md @@ -0,0 +1,571 @@ + + +# ducktests-remote + +Run Apache Ignite ducktests against a real VM cluster, from wherever you happen to be. + +`tests/docker/run_tests.sh` covers the local Docker flow. This covers the other one: a +cluster of real machines, driven by a long `ducktape ...` command line that until now +only existed inside a Jenkins job. + +**This file is the tour. [`docs/`](docs/index.md) is the manual** — every command and +config key, and exactly what happens in what order. Start at +[docs/index.md](docs/index.md), which routes questions to the document that answers them: +[concepts](docs/concepts.md) · [configuration](docs/configuration.md) · +[commands](docs/commands.md) · [java](docs/java.md) · [runs](docs/runs.md) · +[troubleshooting](docs/troubleshooting.md) · [internals](docs/internals.md). + +## The model, in three sentences + +The **coordinator** is the machine where this CLI runs — your laptop, a VM inside the +cluster, or a Jenkins agent. The **runner** is the host where the `ducktape` process +itself lives; it may be the coordinator (`--runner local`) or an SSH host. The +**workers** are the cluster hosts that ducktape drives over SSH and that actually run +Ignite nodes. + +All run state lives on the runner, so any coordinator can inspect, follow or stop a run +that a different coordinator started. Everything the CLI executes goes through a single +`Transport` abstraction, so `--runner local` and `--runner build-vm-01` take identical +code paths. + +## Install + +```bash +cd modules/ducktests/tests +pip install -e . # provides the `ducktests-remote` console script +``` + +Runtime dependencies: the standard library and `PyYAML`. The CLI deliberately **never +imports ducktape** — it drives ducktape on the runner, so it stays installable on a +coordinator that has none. There is a unit check that fails if that ever changes. + +### When PyPI is not reachable + +Only the runner ever runs pip: it creates the venv from `docker/requirements.txt`, and +`run --install-sources` installs the synced sources into it. Workers need no Python at +all. Point pip at your mirror once, in the config: + +```yaml +pip: + index_url: https://nexus.corp/repository/pypi/simple + extra_index_url: [https://nexus.corp/repository/pypi-internal/simple] + trusted_host: [nexus.corp] # plain http, or a certificate you cannot fix + cert: /etc/pki/tls/certs/corp-ca.pem + timeout: 60 + retries: 5 +``` + +or per invocation: `--pip-index-url`, `--pip-extra-index-url`, `--pip-trusted-host`, +`--pip-timeout`, `--pip-cert`. The flags become literal pip arguments — `--dry-run` shows +them — and they apply to *every* pip command the CLI runs, including the build +dependencies of `--install-sources`. + +`cert` is a **runner-side** path, exactly like `identity_file`: it is opened by pip on the +runner, so a file that exists on your laptop proves nothing. + +An index URL usually carries a token. Write it as `${env:NEXUS_URL}` and it is resolved on +the coordinator at launch and masked in everything the CLI prints; credentials typed +directly into a config file are masked too, on output, but they are then sitting in a file. + +The coordinator's own install obeys the same flags, in pip's own spelling: + +```bash +pip install --index-url https://nexus.corp/repository/pypi/simple -e . +``` + +`provision --only packages` uses apt/dnf and is *not* covered by any of this. Which OS +repositories a VM talks to is set when the image is built; this tool does not pretend to +manage it. + +## Quickstart + +Every command accepts `--dry-run`, and `--dry-run` is genuinely side-effect free: it +prints the commands it would run and the files it would generate, including the rendered +`run.sh`, the `cluster.json`, and a redacted `globals.json`. Start there. + +### 1. Describe your cluster + +Copy `examples/cluster.yaml` to `~/.ducktests-remote/config.yaml` and edit it. The +minimum: + +```yaml +cluster: + name: lab + user: max # your account; there is no `ducker` on a real VM + identity_file: ~/.ssh/id_rsa # path AS THE RUNNER SEES IT + runner: build-vm-01 # or "local" + nodes: + - host: node[01-12].dc.local +``` + +### 2. Find out what is missing + +```bash +ducktests-remote doctor +``` + +`doctor` probes the coordinator, the runner and every worker in parallel, and never +stops at the first failure. When hosts are unusable it ends with a copy-pasteable +**"what to ask your administrator"** block naming the hosts, the account, and the exact +line to append to `authorized_keys`. + +### 3. Run + +```bash +ducktests-remote run ./ignitetest/tests/smoke_test.py +``` + +Test paths are the ones you already use with `docker/run_tests.sh`: relative to +`modules/ducktests/tests`. Run the command from anywhere inside the checkout - the +repository root is found by walking up, and paths relative to it work too. + +## The three coordinators + +**Laptop.** The runner is remote, so the key ducktape uses has to live on the runner. +Agent forwarding will not do: a detached run outlives your SSH session, and the agent +dies with it. + +```bash +ducktests-remote keys push # installs the identity on the runner, authorises it on the workers +ducktests-remote doctor +ducktests-remote run -t ./ignitetest/tests/smoke_test.py --detach +ducktests-remote logs -f # reattach later, from anywhere +``` + +**A VM inside the cluster.** Set `cluster.runner: local`. The venv is created under +`state_root` on first run and populated from `docker/requirements.txt`. + +```bash +ducktests-remote --runner local doctor +ducktests-remote --runner local run ./ignitetest/ +``` + +**Jenkins agent.** Use `--detach` plus `status --json`, and read the exit code. + +```bash +ducktests-remote --profile ise-perf run -t "$TC_PATHS" --detach +ducktests-remote status --json > status.json +``` + +## Migrating from the Jenkins one-liner + +Do it in two steps, and keep a working run at each one. + +**Step 1 — paste the blob verbatim.** Whatever JSON the Jenkins job passes to +`--globals`, hand it over unchanged: + +```bash +ducktests-remote run \ + --globals-json '{"project":"ise","ignite_versions":["ise-0-32"],"ssl":{"enabled":true}}' \ + --cluster-file ./49_cluster.json \ + -t ./isetest/perftests/ +``` + +`--cluster-file` is uploaded byte for byte, so an existing hand-written cluster file +keeps working. `--globals-file` reads the same JSON from a file. + +**Step 2 — split it into a profile.** Move the keys into +`~/.ducktests-remote/profiles/ise-perf.yaml`, replacing every secret with a placeholder: + +```yaml +globals: + project: ise + ignite_versions: ["ise-0-32"] + ssl: {enabled: true} + authentication: + enabled: true + username: ${env:ISE_USER} + password: ${env:ISE_PASSWORD} +``` + +Then `ducktests-remote --profile ise-perf run -t ./isetest/perftests/`. Compare the two +with `--dry-run` until the rendered `globals.json` matches, and delete the blob. + +`${env:NAME}` and `${file:PATH}` are resolved on the coordinator at launch, in `globals` +and in every other section alike — `cluster.user`, `pip.index_url`, `java.home`. A missing +variable is a hard error naming the variable and the file it came from — never an empty +string, never a run that fails on authentication three hours later. + +Layering, later winning: built-in defaults → `~/.ducktests-remote/config.yaml` → +`--config` files → `--profile` files → `DTR_*` environment → command-line flags → +`-g KEY=VALUE`. Dicts deep-merge; **lists replace**, so a later layer can shrink one. +`-g` values are parsed as JSON when they parse, so `-g ssl.enabled=true` is a boolean and +`-g project=ise` is a string. + +## Secrets + +- The composed `globals.json` is written to the run directory with mode `0600`. +- Any value resolved from `${env:}` or `${file:}` is registered with a redactor and + replaced with `***` in everything the CLI prints — including `--dry-run` output, log + streaming and error messages. Redaction is keyed on the *value*, so a password that + leaks into an unrelated field is still caught. Key-name matching is only a fallback. +- `fetch` always excludes `globals.json`. +- The example profiles in `examples/` contain no real hostnames, addresses, accounts or + passwords. Keep it that way: this directory is in a public Apache repository. + +## Deploying distributions + +`deploy` is deliberately dumb. Each subdirectory of `--dist-dir` is copied verbatim to +`/`; the name is never interpreted or checked against version +parsing, so you name the directories to match what the tests expect. + +``` +dist/ +├── ignite-dev/ -> /opt/ignite-dev +├── ignite-2.17.0/ -> /opt/ignite-2.17.0 +└── ise-0-32/ -> /opt/ise-0-32 +``` + +```bash +ducktests-remote deploy --dry-run # plan and total bytes, transfers nothing +ducktests-remote deploy --only ignite-dev +ducktests-remote deploy --via build-vm-01 # upload once, fan out from there +ducktests-remote deploy --sudo --owner max # when /opt is root-owned +``` + +Each host gets a `.ducktests-deploy.json` manifest (sorted paths + sizes + mtimes; +`--checksum` hashes contents instead). Hosts whose manifest already matches are skipped +unless `--force`. The staging directory is filled and then swapped into place, because a +half-copied distribution that looks present is worse than an absent one. + +By default the staging directory is filled by **rsync**, hardlinked with `--link-dest` +against the deployment already on the host, so a redeploy after rebuilding one module +sends one jar rather than the whole tree: + +``` +w01 changed 4.1s rsync: 12 of 8431 file(s) changed, 4.0 MB sent +``` + +Every transfer that can take minutes — this one, `run`'s source sync and `provision`'s JDK +delivery — shows what each host is doing, redrawn in place on a terminal and reduced to one +`total ...` line every 15 seconds in a log, under `--verbose`, or wherever stderr is not a +terminal (`--no-progress` turns it off): + +``` + worker01 █████████████████░░░░░░░ 70% 210.0 MB / 300.0 MB 24.1 MB/s + worker02 ████████░░░░░░░░░░░░░░░░ 32% 96.0 MB / 300.0 MB 18.7 MB/s + total ████████████░░░░░░░░░░░░ 50% 1/4 host(s) 306.0 MB sent 1:12 +``` + +rsync is fed the exact file list as a file — never its own `--exclude` patterns, whose +matching differs from the manifest's. `deploy` falls back to a tarball of the whole +distribution when either end has no rsync (probed per host), with `--via`, with +`--no-rsync` or `deploy.rsync: false`, or on a Windows coordinator, where rsync would read +`C:/dist/ignite-dev` as a host named `C`. Nothing is tarred at all when every host takes +the rsync path. + +On a twelve-host cluster a 300 MB distribution is 3.7 GB over the wire from a laptop. +`deploy` prints that total before it starts, and suggests `--via`. + +### `ignite-dev` from your own checkout + +`ignite-dev` must look like a *built source tree* on the worker, not like a release: +`IgniteSpec` classpaths `modules//target` and `modules//target/libs`, +`bin/ignite.sh` is run from the same home, and SSL tests read +`modules/ducktests/tests/certs`. Everything else in a checkout is ballast — a bare copy +of a built repository is well over a gigabyte, most of it `.git` and sources the workers +never open. + +Link the distribution to your checkout, and trim it with excludes: + +```bash +mkdir -p ~/dist +ln -sfn ~/dev/ignite ~/dist/ignite-dev # relink any time; deploy follows it +``` + +```yaml +deploy: + dist_dir: ~/dist + exclude: [.git, .idea, src, docs, assembly, classes, test-classes, + generated-sources, generated-test-sources, maven-status, maven-archiver, + surefire-reports, javadoc, "*.tar.gz", "*.zip", __pycache__, "*.pyc"] +``` + +```bash +mvn package -pl :ignite-ducktests -am -DskipTests # you build, however you like +ducktests-remote deploy --only ignite-dev --dry-run # payload size + files dropped +ducktests-remote deploy --only ignite-dev +``` + +Excludes are the same rsync-style patterns the source sync uses, and default to nothing. +Three sources, most specific first: `--exclude PATTERN`, a `.ducktests-deploy.ignore` file +at the root of one distribution, then `deploy.exclude`. That file is deliberately *not* +named `.ducktestsignore`: when `ignite-dev` links to a checkout, the distribution root and +the source root are the same directory, and the two lists are opposites — the source sync +drops `target`, `deploy` keeps little else. The manifest is built from the filtered list, +so an `already at ` skip always means the host holds what was actually sent. + +Symlinks *inside* a distribution are not followed: they are shipped as links and arrive +dangling. Only the top-level link is resolved. + +### Where the directory names come from + +`ignitetest` resolves a distribution home as `/`, where `product` +is `str(IgniteVersion(version))` (`services/utils/path.py`, `services/utils/ignite_aware.py`). +`IgniteVersion.__str__` **normalises**, so: + +| `ignite_versions` entry | directory under `/opt` | +| --- | --- | +| `dev` | `ignite-dev` | +| `2.17.0` | `ignite-2.17.0` | +| `ise-0-32` | `ise-0-32` | +| `ise--6` | `ise-6` — note the collapsed dash | + +A fork can override `product`, so `doctor` reports a missing directory as a WARN listing +what it *did* find under the install root rather than failing on a guessed mapping. + +## Provisioning + +`modules/ducktests/tests/docker/Dockerfile` is the source of truth for what a prepared +node looks like; the package list in `config.py` is derived from it and carries a comment +saying so. `provision` is not Ansible and must not become it. + +```bash +ducktests-remote provision --dry-run # recommended first invocation +ducktests-remote provision --sudo --only packages --only dirs +ducktests-remote provision --only ssh-env +ducktests-remote provision --sudo --write-hosts +``` + +| Step | What it does | +| --- | --- | +| `packages` | Installs the Dockerfile's system utilities. Detects apt/dnf/yum; an unknown package manager is a clear failure, not a guess. Needs `--sudo`. | +| `jdk` | Resolves a JDK of `java.major` per host and, when none is there, delivers `java.archive`. See "Choosing the JDK". | +| `python` | Verifies only. Workers do not need Python — ducktape drives them over plain SSH. The runner's venv is created by `run`. | +| `user` | `--create-user NAME` plus `--authorize-key`. Not run by default; most operators use their own account. Needs `--sudo`. | +| `ssh-env` | Points the workers' non-interactive `PATH`/`JAVA_HOME` at that JDK, then proves it. **The one that is easiest to forget.** | +| `dirs` | Creates and chowns `/mnt/service` and the install root. Needs `--sudo`. | +| `hosts` | `--write-hosts` rewrites only the block between `# BEGIN ducktests-remote` and `# END ducktests-remote` in `/etc/hosts`. Needs `--sudo`. | + +Anything needing root goes through `sudo -n`. If that fails, the step is reported as +`no-sudo`, skipped, and the remaining steps still run — a partial provision with an +honest report beats an all-or-nothing failure. `provision` always finishes by running +the `doctor` checks, so it ends with evidence rather than an assumption. + +### Why `ssh-env` matters + +ducktape runs every command over **non-interactive** SSH, where `~/.profile` is not +sourced. A `java` that works fine when you log in by hand is simply absent during a test +run, and the failure surfaces as an unrelated timeout. The Dockerfile solves this with +`PermitUserEnvironment yes` plus `~/.ssh/environment`; this step does the same, adds a +`~/.bashrc` fallback, and then proves it by running `java -version` non-interactively. + +## Choosing the JDK + +### Why `PATH` matters more than `JAVA_HOME` + +`ignitetest` reaches a JVM four different ways, and only one of them respects `JAVA_HOME`: + +| Consumer | Mechanism | +| --- | --- | +| `ignite.sh`, via `IgniteSpec.envs()` | honours `JAVA_HOME` | +| `jvm_utils.java_version()` → `java -version` | bare `java`, so `PATH` | +| `services/kafka/kafka.py` → `nohup java …` | bare `java`, so `PATH` | +| `jmx_utils` → `java -jar jmxterm.jar` | bare `java`, so `PATH` | + +Setting `JAVA_HOME` therefore changes what `ignite.sh` uses and nothing else. Both are +set, and what a fresh non-interactive session actually gets is then verified rather than +assumed. + +### Configuration + +```yaml +java: + major: 17 # derived from the Dockerfile's eclipse-temurin:17 + home: /opt/jdk-17.0.11 # optional: use exactly this, no search + search_paths: [/opt, /usr/lib/jvm, /usr/java] + archive: ~/jdk/OpenJDK17U-jdk_x64_linux_hotspot.tar.gz + install_root: /opt # defaults to cluster.install_root + ssh_environment: true + bashrc: true +``` + +`provision --only jdk` resolves one JDK per host, in this order: + +1. **`java.home`** — verified, and a host that does not have it is a failure naming that + host. Explicit means explicit; falling back would defeat the point of saying it. +2. **the JVM already on the non-interactive `PATH`**, when its major matches. Nothing is + installed on a VM that is already correct. +3. **a JDK under `search_paths`** — `/opt/jdk-17.0.11`, `/usr/lib/jvm/java-17-openjdk`. + Highest patch level wins, compared numerically, so `17.0.11` beats `17.0.9`. +4. **`java.archive`**, delivered from the coordinator to the hosts that got this far — + and only to those. A JDK tarball (`.tar.gz`, `.tgz`, `.tar`, `.tar.bz2`, `.tar.xz`) or + an unpacked directory. Whatever wraps the JDK home inside the archive is stripped, so + both a stock Temurin tarball and a hand-repacked one land as `/opt/jdk-17.0.11+9`. Bad + archives (no `bin/java` anywhere, a macOS build with `Contents/Home`, a zip) fail on + the coordinator, before anything is copied to twelve machines. +5. otherwise a failure listing every JDK that *was* found. `--install-jdk` (with `--sudo`) + adds the distribution's own package as a last rung. + +Delivery reuses `deploy`: staging plus an atomic swap, and a `.ducktests-java.json` +manifest so a host that already has the JDK is skipped. `--force` re-delivers. + +```bash +ducktests-remote provision --dry-run --only jdk # what each host would resolve to +ducktests-remote provision --only jdk --only ssh-env # resolve, install, then point PATH at it +ducktests-remote provision --only jdk --java-archive ~/jdk/temurin17.tar.gz +``` + +`provision --only ssh-env` runs the same ladder, so it can be used on its own and still +points at the JDK you asked for rather than at whatever `java` happens to be first. + +### Making it stick + +Both files are written, from one resolved value, in one step: + +- **`~/.ssh/environment`** — what the Dockerfile does. Silently ignored unless sshd carries + `PermitUserEnvironment yes`; `provision` says so when it does not. +- **`~/.bashrc`** — a marked block at the **top** of the file, above the + `case $- in *i*) ;; *) return;; esac` guard the stock file opens with. That guard exists + precisely because bash *does* source `~/.bashrc` for non-interactive ssh commands. It + does nothing when the account's login shell is not bash. + +Then a fresh connection is opened and asked what it got. That answer is the result: if it +is still the wrong JVM, the step fails there, not three hours into a run. + +`doctor` judges the same thing — what a non-interactive session gets — and a major that +does not match `java.major` is a **FAIL**, which stops `run` at preflight (exit 2). A +matching version from a JDK other than an explicitly configured `java.home` is a WARN: the +tests will run, but the pin is not in effect. + +## Privileges the tests actually need + +Grepped from the `ignitetest` sources, not assumed: + +- **Ordinary unprivileged account** for everything except the two suites below. It needs + write access to `persistent_root` (default `/mnt/service`) and read access to + `install_root` (default `/opt`). +- **Passwordless `sudo` for `iptables`** only, and only for the network-segmentation + suites: `ignitetest/tests/discovery_test.py` and + `ignitetest/tests/cellular_affinity_test.py`. They reach `sudo iptables`, + `iptables-save` and `iptables-restore` through `IgniteAwareService.drop_network` + (`services/utils/ignite_aware.py`). +- **Write access to `install_root`** only if you use `deploy`; otherwise `deploy --sudo`. + +Nothing else in `ignitetest` needs root. If your administrator is offering you a +privileged account you do not need, this is the list to show them. + +## Cleaning up + +```bash +ducktests-remote clean --dry-run # prints exactly what it would kill and remove +ducktests-remote clean +``` + +Kills processes matching `clean.process_pattern` (default `org.apache.ignite`, which +covers `CommandLineStartup`, `CdcCommandLineStartup`, `IgniteAwareApplicationService` and +`KafkaToIgniteCommandLineStartup`), then removes `clean.paths` (default `/mnt/service`). + +Every path is checked against `clean.allowed_roots` before it is sent anywhere, and the +roots themselves are not removable. A bug here would delete distributions across every +machine at once, so the rule is deliberately blunt. + +`stop` runs `clean` afterwards unless you pass `--no-clean`. + +## Ctrl-C during `--follow` + +`run` launches detached from second zero and then attaches to the log. Therefore: + +- **Ctrl-C detaches. It does not stop the run.** The CLI prints how to reattach and how + to stop. +- A **second Ctrl-C within 3 seconds** offers to stop the run (and, in a non-interactive + shell, simply detaches). +- `--detach` skips the following entirely. + +Reattach with `ducktests-remote logs -f`; stop with +`ducktests-remote stop `. + +## Run directory + +On the runner, under `/runs//`: + +``` +meta.json run id, coordinator, start time, test paths, redacted config summary +cluster.json what ducktape was given +globals.json composed globals, mode 0600 +run.sh the exact command; ssh in and rerun it to reproduce by hand +launch.sh wrapper that waits on run.sh and records exit_code +pid, pgid for stop +exit_code written when the process ends +ducktape.log combined stdout and stderr +results/ ducktape --results-root, with ducktape's own `latest` symlink inside +``` + +`/runs/latest` points at the newest run. Run ids look like +`max-20260727-141233-9f2a`. + +## Exit codes + +| Code | Meaning | +| --- | --- | +| 0 | success | +| 1 | usage or configuration error | +| 2 | preflight failed | +| 3 | reserved (this deployment has a single runner and takes no cluster lease) | +| 4 | ducktape ran and reported test failures | +| 5 | transport or infrastructure error | +| 130 | interrupted by the operator | + +4 and 5 are deliberately distinct: Jenkins needs "tests failed" separate from "the +cluster is broken". Note that ducktape itself exits `1` both for test failures and for +its own startup errors, so a `4` means "ducktape ran and exited non-zero" — the log +distinguishes the two. + +There is no cluster lease. This deployment has one runner and one operator; queueing is +the job of whatever schedules these runs. + +## Troubleshooting + +**Stale Ignite JVMs.** The single most common source of baffling failures on a shared +cluster: a previous run was killed and its JVMs are still holding ports and +`/mnt/service`. `doctor` reports this as a FAIL with the host list. Fix with +`ducktests-remote clean --dry-run` and then `clean`. + +**`identity_file` is a runner-side path.** It is the path *ducktape* will open, on the +runner. If the runner is not the coordinator, a file that exists on your laptop proves +nothing. `doctor` checks it on the runner and reports mode; `keys push` installs it. + +**Agent forwarding does not survive a detached run.** `ssh -A` gives you an agent for the +lifetime of your session. A run that lasts hours outlives it, and every subsequent +worker connection then fails. Use a real key file on the runner. + +**`java: command not found`, or the wrong JDK, deep inside a test.** Non-interactive SSH +does not source `~/.profile`. Run `provision --only jdk --only ssh-env`. If it still +reports the wrong JVM afterwards, sshd is ignoring `~/.ssh/environment` *and* the login +shell is not bash; set `java.home` to a JDK the site already puts on the default `PATH`. + +**Discovery failures with no useful message.** Workers that cannot resolve each other's +hostnames fail inside discovery. `doctor` runs an N-way resolution probe; `provision +--write-hosts` is the escape hatch when cluster DNS cannot be fixed. + +**"source payload is N MB, above the limit".** A build directory leaked into the sync. +Distributions go through `deploy`, never through the source sync. Adjust `--exclude` or +add a `.ducktestsignore` file at the source root. + +## Development + +```bash +cd modules/ducktests/tests +pytest ducktests_remote/checks # unit only: no network, no Docker, no ducktape +flake8 ducktests_remote +``` + +The checks live in `ducktests_remote/checks/` and are named `check_*.py` with `Check` +classes and `check_*` methods, because that is what `[pytest]` in `tox.ini` collects. +`checks/fake_transport.py` provides a recording transport; nothing in the unit checks +touches a network or a real process except the deliberate subprocess in the import guard. diff --git a/modules/ducktests/tests/ducktests_remote/__init__.py b/modules/ducktests/tests/ducktests_remote/__init__.py new file mode 100644 index 0000000000000..ddd5782f052f3 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/__init__.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +ducktests-remote: run Apache Ignite ducktests against a real VM cluster. + +This package deliberately never imports ``ducktape``. The CLI is a coordinator-side +orchestrator: it renders artifacts and drives the ``ducktape`` process that lives on the +runner. Keeping the dependency out means the CLI stays installable on a laptop that has +no ducktape at all, and lets the two be upgraded independently. +""" + +__version__ = "0.1.0" diff --git a/modules/ducktests/tests/ducktests_remote/__main__.py b/modules/ducktests/tests/ducktests_remote/__main__.py new file mode 100644 index 0000000000000..52a9da8c830ed --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/__main__.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Entry point for ``python -m ducktests_remote``.""" + +import sys + +from ducktests_remote.cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_cluster.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_cluster.py new file mode 100644 index 0000000000000..729abf68f38d5 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_cluster.py @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for the inventory and the generated ducktape cluster file.""" + +import json + +import pytest + +from ducktests_remote.cluster import (cluster_json, dumps, expand_hosts, load_nodes, + select_nodes) +from ducktests_remote.config import ConfigError + + +def _cfg(nodes, **kw): + base = {"user": "tester", "port": 22, "identity_file": "/home/tester/.ssh/id_rsa", + "nodes": nodes} + base.update(kw) + return base + + +class CheckRangeExpansion: + """``node[01-12].dc.local`` shorthand.""" + + def check_zero_padding_follows_the_lower_bound(self): + assert expand_hosts("node[01-03].dc.local") == [ + "node01.dc.local", "node02.dc.local", "node03.dc.local"] + + def check_unpadded_range(self): + assert expand_hosts("node[8-11]") == ["node8", "node9", "node10", "node11"] + + def check_a_plain_hostname_is_left_alone(self): + assert expand_hosts("10.0.0.13") == ["10.0.0.13"] + + def check_forty_nine_nodes(self): + assert len(expand_hosts("node[01-49].dc.local")) == 49 + + def check_two_nodes(self): + assert len(expand_hosts("node[01-02].dc.local")) == 2 + + def check_inverted_range_is_rejected(self): + with pytest.raises(ConfigError): + expand_hosts("node[9-2]") + + +class CheckInventory: + """Loading nodes from the config section.""" + + def check_bare_string_shorthand(self): + nodes = load_nodes(_cfg(["10.0.0.13"])) + assert nodes[0].host == "10.0.0.13" and nodes[0].user == "tester" + + def check_per_host_user_override(self): + nodes = load_nodes(_cfg([{"host": "a"}, {"host": "b", "user": "other"}])) + assert [n.user for n in nodes] == ["tester", "other"] + + def check_externally_routable_ip_falls_back_to_the_hostname(self): + nodes = load_nodes(_cfg([{"host": "a"}, {"host": "b", "ip": "10.0.0.2"}])) + assert nodes[0].externally_routable_ip == "a" + assert nodes[1].externally_routable_ip == "10.0.0.2" + + def check_duplicate_hosts_are_rejected(self): + with pytest.raises(ConfigError): + load_nodes(_cfg(["a", "a"])) + + def check_unknown_node_key_is_rejected(self): + with pytest.raises(ConfigError): + load_nodes(_cfg([{"host": "a", "usr": "typo"}])) + + def check_ip_cannot_be_combined_with_a_range(self): + with pytest.raises(ConfigError): + load_nodes(_cfg([{"host": "n[1-3]", "ip": "10.0.0.1"}])) + + +class CheckSelection: + """``--num-nodes``, the analogue of IGNITE_NUM_CONTAINERS.""" + + def check_truncation_takes_the_first_n(self): + nodes = load_nodes(_cfg(["a", "b", "c", "d"])) + assert [n.host for n in select_nodes(nodes, 2)] == ["a", "b"] + + def check_none_means_everything(self): + nodes = load_nodes(_cfg(["a", "b"])) + assert len(select_nodes(nodes, None)) == 2 + + def check_too_many_names_the_inventory_size(self): + nodes = load_nodes(_cfg(["a", "b"])) + with pytest.raises(ConfigError) as ex: + select_nodes(nodes, 5) + assert "2 hosts" in str(ex.value) + + def check_zero_is_rejected(self): + with pytest.raises(ConfigError): + select_nodes(load_nodes(_cfg(["a"])), 0) + + +class CheckClusterFile: + """The schema ducktape's JsonCluster reads.""" + + def check_shape_matches_ducktape(self): + nodes = load_nodes(_cfg([{"host": "node01.dc.local", "ip": "10.0.0.11"}])) + payload = cluster_json(nodes) + entry = payload["nodes"][0] + assert entry["externally_routable_ip"] == "10.0.0.11" + # RemoteAccountSSHConfig(host, hostname, user, port, password, identityfile) + assert set(entry["ssh_config"]) == { + "host", "hostname", "user", "port", "identityfile", "password"} + assert entry["ssh_config"]["port"] == 22 + assert entry["ssh_config"]["identityfile"] == "/home/tester/.ssh/id_rsa" + + def check_runner_side_identity_fallback(self): + nodes = load_nodes(_cfg(["a"], identity_file=None)) + payload = cluster_json(nodes, identity_file="/runner/side/key") + assert payload["nodes"][0]["ssh_config"]["identityfile"] == "/runner/side/key" + + def check_two_and_forty_nine_have_the_same_shape(self): + small = cluster_json(load_nodes(_cfg(["n[01-02]"]))) + large = cluster_json(load_nodes(_cfg(["n[01-49]"]))) + assert len(small["nodes"]) == 2 and len(large["nodes"]) == 49 + assert small["nodes"][0].keys() == large["nodes"][0].keys() + + def check_empty_inventory_is_rejected(self): + with pytest.raises(ConfigError): + cluster_json([]) + + def check_output_is_valid_json(self): + rendered = dumps(cluster_json(load_nodes(_cfg(["a", "b"])))) + assert len(json.loads(rendered)["nodes"]) == 2 diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_config.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_config.py new file mode 100644 index 0000000000000..715628d9b9c80 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_config.py @@ -0,0 +1,198 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for configuration discovery, layering and validation.""" + +import pytest + +from ducktests_remote.config import (ConfigError, coerce_scalar, deep_merge, env_overrides, + get_dotted, load_config, parse_document, set_dotted, + validate) + + +class CheckMerge: + """Layering semantics.""" + + def check_dicts_merge_recursively(self): + base = {"a": {"x": 1, "y": 2}, "b": 1} + overlay = {"a": {"y": 3, "z": 4}} + assert deep_merge(base, overlay) == {"a": {"x": 1, "y": 3, "z": 4}, "b": 1} + + def check_lists_replace_and_do_not_concatenate(self): + merged = deep_merge({"v": ["a", "b", "c"]}, {"v": ["d"]}) + assert merged["v"] == ["d"], "a later layer must be able to shrink a list" + + def check_base_is_not_mutated(self): + base = {"a": {"x": 1}} + deep_merge(base, {"a": {"x": 2}}) + assert base == {"a": {"x": 1}} + + +class CheckDocuments: + """Parser selection by content, not by extension.""" + + def check_json_body(self): + assert parse_document('{"cluster": {"name": "lab"}}') == {"cluster": {"name": "lab"}} + + def check_yaml_body(self): + assert parse_document("cluster:\n name: lab\n") == {"cluster": {"name": "lab"}} + + def check_broken_body_names_the_source(self): + with pytest.raises(ConfigError) as ex: + parse_document("cluster: [unclosed", source="profile.yaml") + assert "profile.yaml" in str(ex.value) + + +class CheckValidation: + """Unknown keys are a hard error with a suggestion.""" + + def check_unknown_top_level_key_is_rejected(self): + with pytest.raises(ConfigError) as ex: + validate({"clustr": {}}) + assert "clustr" in str(ex.value) and "cluster" in str(ex.value) + + def check_unknown_nested_key_is_rejected(self): + with pytest.raises(ConfigError) as ex: + validate({"cluster": {"instal_root": "/opt"}}) + assert "cluster.instal_root" in str(ex.value) + + def check_globals_are_free_form(self): + validate({"globals": {"anything_at_all": {"nested": 1}}}) + + def check_parameters_are_free_form(self): + validate({"parameters": {"whatever": 1}}) + + +class CheckDotted: + """Dotted path helpers.""" + + def check_set_creates_intermediate_dicts(self): + target = {} + set_dotted(target, "a.b.c", 1) + assert target == {"a": {"b": {"c": 1}}} + + def check_get_returns_default_for_missing(self): + assert get_dotted({"a": {"b": 1}}, "a.z", "fallback") == "fallback" + + def check_scalar_coercion(self): + assert coerce_scalar("true") is True + assert coerce_scalar("12") == 12 + assert coerce_scalar("ise") == "ise" + + +class CheckEnvironment: + """DTR_* overrides.""" + + def check_double_underscore_is_a_path_separator(self): + overlay = env_overrides({"DTR_CLUSTER__RUNNER": "build-vm-01"}) + assert overlay == {"cluster": {"runner": "build-vm-01"}} + + def check_single_underscores_survive_inside_a_key(self): + overlay = env_overrides({"DTR_RUN__MAX_PAYLOAD_MB": "50"}) + assert overlay == {"run": {"max_payload_mb": 50}} + + def check_aliases(self): + assert env_overrides({"DTR_RUNNER": "vm"}) == {"cluster": {"runner": "vm"}} + + def check_secret_variables_are_not_treated_as_config(self): + # Profiles interpolate ${env:DTR_...}; those must not become config paths. + assert env_overrides({"DTR_ISE_PASSWORD": "hunter2"}) == {} + + +class CheckLoad: + """Whole-stack layering.""" + + def check_later_config_file_wins(self, tmp_path): + first = tmp_path / "a.yaml" + first.write_text("cluster:\n name: one\n port: 2222\n", encoding="utf-8") + second = tmp_path / "b.yaml" + second.write_text("cluster:\n name: two\n", encoding="utf-8") + config = load_config(config_files=[first, second], environ={}, user_config=None) + assert config["cluster"]["name"] == "two" + assert config["cluster"]["port"] == 2222, "unrelated keys survive the overlay" + + def check_flags_beat_environment(self, tmp_path): + config = load_config(environ={"DTR_RUNNER": "from-env"}, + overrides={"cluster": {"runner": "from-flag"}}, + user_config=None) + assert config["cluster"]["runner"] == "from-flag" + + def check_user_defaults_to_the_coordinator_account_not_ducker(self): + config = load_config(environ={}, user_config=None) + assert config["cluster"]["user"], "an ssh user must always be resolved" + assert config["cluster"]["user"] != "ducker", \ + "nothing may default to the Docker image's account" + + def check_missing_config_file_is_reported(self): + with pytest.raises(ConfigError) as ex: + load_config(config_files=["/nonexistent/nope.yaml"], environ={}, user_config=None) + assert "not found" in str(ex.value) + + +class CheckRenamedKeys: + """Keys that moved out of `provision` when pip and java got sections of their own.""" + + def check_the_old_pip_index_key_is_rejected_with_a_hint(self): + with pytest.raises(ConfigError) as ex: + validate({"provision": {"pip_index_url": "https://nexus.invalid/simple"}}) + assert "provision.pip_index_url" in str(ex.value) + + def check_the_old_jdk_major_key_is_rejected(self): + with pytest.raises(ConfigError) as ex: + validate({"provision": {"jdk_major": 17}}) + assert "provision.jdk_major" in str(ex.value) + + def check_the_new_sections_validate(self): + validate({"pip": {"index_url": "https://nexus.invalid/simple", "timeout": 60}, + "java": {"major": 17, "home": "/opt/jdk-17", "bashrc": False}}) + + def check_a_typo_in_the_new_sections_is_caught(self): + with pytest.raises(ConfigError) as ex: + validate({"java": {"majr": 17}}) + assert "java.majr" in str(ex.value) and "major" in str(ex.value) + + +class CheckInterpolation: + """``${env:}`` / ``${file:}`` outside the globals section.""" + + def check_a_placeholder_in_a_config_section_is_resolved(self): + # examples/cluster.yaml has always advertised this; before pip.* needed it, the + # placeholder was passed through literally and ssh went looking for a host named + # "${env:DTR_RUNNER}". + config = load_config(environ={"DTR_TEST_RUNNER": "build-vm-01"}, user_config=None, + overrides={"cluster": {"runner": "${env:DTR_TEST_RUNNER}"}}) + assert config["cluster"]["runner"] == "build-vm-01" + + def check_a_missing_variable_names_the_variable(self): + with pytest.raises(ConfigError) as ex: + load_config(environ={}, user_config=None, + overrides={"pip": {"index_url": "${env:DTR_ABSENT_INDEX}"}}) + assert "DTR_ABSENT_INDEX" in str(ex.value) + + def check_a_resolved_value_is_registered_for_redaction(self): + from ducktests_remote.globals_builder import Redactor # noqa: PLC0415 + + redactor = Redactor() + load_config(environ={"DTR_TEST_INDEX": "https://bob:s3cret@nexus.invalid/simple"}, + user_config=None, redactor=redactor, + overrides={"pip": {"index_url": "${env:DTR_TEST_INDEX}"}}) + assert redactor.redact("index is https://bob:s3cret@nexus.invalid/simple") \ + == "index is ***" + + def check_globals_are_left_for_the_globals_builder(self): + # Resolved there, per layer, so the error can name the profile it came from. + config = load_config(environ={}, user_config=None, + overrides={"globals": {"password": "${env:DTR_ABSENT}"}}) + assert config["globals"]["password"] == "${env:DTR_ABSENT}" diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py new file mode 100644 index 0000000000000..f04fe0b6242ec --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_deploy.py @@ -0,0 +1,668 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for the deploy manifest, the clean allow-list, and provision idempotency.""" + +import json +import os +import tarfile +import time + +import pytest +from fake_transport import FakeTransport + +from ducktests_remote.cluster import Node +from ducktests_remote.commands import clean as clean_cmd +from ducktests_remote.commands import deploy, provision +from ducktests_remote.config import DEFAULTS, ConfigError +from ducktests_remote.fanout import CHANGED, FAILED, OK, HostResult, fanout, summarise +from ducktests_remote.transport import Result, make_tarball + + +def _dist(tmp_path, name="ignite-dev", body="binary"): + root = tmp_path / name + (root / "bin").mkdir(parents=True) + (root / "bin" / "ignite.sh").write_text(body, encoding="utf-8") + (root / "libs").mkdir() + (root / "libs" / "core.jar").write_text("jar", encoding="utf-8") + return root + + +def _staging_of(transport): + """:return: the staging path a recorded deploy created on ``transport``.""" + prepared = [s for s in transport.scripts if "mkdir -p" in s] + assert prepared, "the deploy never got as far as making a staging directory" + return prepared[0].split("mkdir -p ")[1].strip() + + +class CheckManifest: + """Skip-if-unchanged.""" + + def check_identical_trees_hash_the_same(self, tmp_path): + first = _dist(tmp_path / "a") + second = _dist(tmp_path / "b") + os.utime(second / "bin" / "ignite.sh", + (os.stat(first / "bin" / "ignite.sh").st_atime, + os.stat(first / "bin" / "ignite.sh").st_mtime)) + os.utime(second / "libs" / "core.jar", + (os.stat(first / "libs" / "core.jar").st_atime, + os.stat(first / "libs" / "core.jar").st_mtime)) + assert deploy.build_manifest(first)["hash"] == deploy.build_manifest(second)["hash"] + + def check_changed_content_changes_the_hash(self, tmp_path): + dist = _dist(tmp_path / "a") + before = deploy.build_manifest(dist, checksum=True)["hash"] + (dist / "bin" / "ignite.sh").write_text("different", encoding="utf-8") + assert deploy.build_manifest(dist, checksum=True)["hash"] != before + + def check_changed_mtime_changes_the_default_hash(self, tmp_path): + dist = _dist(tmp_path / "a") + before = deploy.build_manifest(dist)["hash"] + os.utime(dist / "bin" / "ignite.sh", (time.time() + 60, time.time() + 60)) + assert deploy.build_manifest(dist)["hash"] != before + + def check_manifest_records_size_and_count(self, tmp_path): + manifest = deploy.build_manifest(_dist(tmp_path / "a")) + assert manifest["files"] == 2 and manifest["bytes"] > 0 + assert manifest["mode"] == "size+mtime" + + def check_checksum_mode_is_recorded(self, tmp_path): + assert deploy.build_manifest(_dist(tmp_path / "a"), + checksum=True)["mode"] == "checksum" + + +class CheckSkipLogic: + """The decision the fan-out makes per host, given a remote manifest.""" + + @staticmethod + def _remote_says(hash_value): + transport = FakeTransport() + transport.when("cat", stdout=json.dumps({"hash": hash_value})) + return transport + + def check_matching_manifest_means_skip(self): + transport = self._remote_says("abc") + existing = json.loads(transport.read_file("/opt/x/.ducktests-deploy.json")) + assert existing["hash"] == "abc" + + def check_manifest_filename_is_stable(self): + assert deploy.MANIFEST_NAME == ".ducktests-deploy.json" + + def check_swap_removes_the_old_tree_only_after_the_move(self, tmp_path): + script = deploy.swap_script("/opt/.x.tmp.1", "/opt/x", False, None) + move_index = script.index('mv -- "$staging" "$target"') + remove_index = script.index('rm -rf -- "$old"') + assert move_index < remove_index, \ + "a half-copied distribution that looks present is worse than an absent one" + + def check_sudo_prefixes_every_privileged_command(self): + script = deploy.swap_script("/opt/.x.tmp.1", "/opt/x", True, "max") + assert script.count("sudo -n ") >= 3 + assert "chown -R max" in script + + +class CheckStagingLifecycle: + """ + A staging directory holds a whole copy of the distribution and hides behind a dot. + + Deploys of a gigabyte tree to fourteen hosts have failed after the transfer and left + one of these per attempt per host, where nothing looks for them. + """ + + @staticmethod + def _ctx(dist_root, transport): + class _Args: # pylint: disable=too-few-public-methods + force = False + sudo = False + owner = None + checksum = False + via = None + + class _Console: # pylint: disable=too-few-public-methods + verbose = False + + @staticmethod + def detail(_message): + """Swallow the traced command line.""" + + class _Ctx: # pylint: disable=too-few-public-methods + args = _Args() + console = _Console() + config = {"deploy": DEFAULTS["deploy"]} + dry_run = False + dist_root = None + + @staticmethod + def worker(_node): + return transport + + ctx = _Ctx() + ctx.dist_root = dist_root + return ctx + + def check_preparing_sweeps_what_an_earlier_attempt_left(self): + sweep = deploy.staging_prefix("/opt", "ignite-dev") + script = deploy.prepare_script(sweep + "abc123", False, sweep=sweep) + assert "rm -rf -- /opt/.ignite-dev.tmp.*" in script, \ + "every attempt picks a new suffix, so only a glob reclaims the old ones" + assert script.index("rm -rf") < script.index("mkdir -p") + + def check_the_sweep_is_scoped_to_one_distribution(self): + script = deploy.prepare_script("/opt/.ignite-dev.tmp.abc", False, + sweep=deploy.staging_prefix("/opt", "ignite-dev")) + for line in script.splitlines(): + if line.startswith("rm -rf"): + assert "ignite-dev" in line, "a glob here would empty the install root" + + def check_a_name_needing_quotes_still_globs(self): + sweep = deploy.staging_prefix("/opt", "fork 2.8") + script = deploy.prepare_script(sweep + "abc", False, sweep=sweep) + assert "'/opt/.fork 2.8.tmp.'*" in script, \ + "the star must stay outside the quotes or nothing is swept" + + def check_a_failed_transfer_takes_its_staging_directory_with_it(self, tmp_path): + root = _dist(tmp_path) + transport = FakeTransport() + transport.when("tar -xzf", returncode=1, stderr="unexpected end of file") + ctx = self._ctx(root, transport) + payload = deploy._Payload(root, tmp_path / "p.tar.gz", ()) # noqa: SLF001 + + with pytest.raises(Exception): # noqa: B017 - any failure must still clean up + deploy._deploy_to_host( # noqa: SLF001 + ctx, Node(host="w1"), "ignite-dev", "/opt/ignite-dev", + deploy.build_manifest(root), "{}", payload, None, None) + + assert deploy.discard_script(_staging_of(transport), False) in transport.scripts, \ + "the staging tree outlived the deploy that made it" + + def check_a_successful_deploy_does_not_remove_the_swapped_tree(self, tmp_path): + root = _dist(tmp_path) + transport = FakeTransport() + ctx = self._ctx(root, transport) + payload = deploy._Payload(root, tmp_path / "p.tar.gz", ()) # noqa: SLF001 + + result = deploy._deploy_to_host( # noqa: SLF001 + ctx, Node(host="w1"), "ignite-dev", "/opt/ignite-dev", + deploy.build_manifest(root), "{}", payload, None, None) + + assert result.status == CHANGED + assert any('mv -- "$staging" "$target"' in s for s in transport.scripts) + assert deploy.discard_script(_staging_of(transport), False) not in transport.scripts, \ + "after the swap the staging path is the distribution; removing it deletes it" + + +class CheckCleanAllowList: + """A bug here deletes distributions across every machine at once.""" + + def check_default_paths_are_accepted(self): + assert clean_cmd.validated_paths(DEFAULTS["clean"]) == ["/mnt/service"] + + def check_path_outside_the_allow_list_is_rejected(self): + with pytest.raises(ConfigError): + clean_cmd.validated_paths({"paths": ["/opt/ignite-dev"], + "allowed_roots": ["/mnt"]}) + + def check_root_itself_is_rejected(self): + with pytest.raises(ConfigError): + clean_cmd.validated_paths({"paths": ["/"], "allowed_roots": ["/mnt"]}) + + def check_the_allowed_root_itself_is_not_removable(self): + with pytest.raises(ConfigError): + clean_cmd.validated_paths({"paths": ["/mnt"], "allowed_roots": ["/mnt"]}) + + def check_relative_paths_are_rejected(self): + with pytest.raises(ConfigError): + clean_cmd.validated_paths({"paths": ["service"], "allowed_roots": ["/mnt"]}) + + def check_traversal_is_normalised_away(self): + with pytest.raises(ConfigError): + clean_cmd.validated_paths({"paths": ["/mnt/../opt/ignite-dev"], + "allowed_roots": ["/mnt"]}) + + def check_dry_run_script_kills_and_removes_nothing(self): + script = clean_cmd._script("org.apache.ignite", ["/mnt/service"], # noqa: SLF001 + dry_run=True) + assert "dry=1" in script + assert 'if [ "$dry" -eq 0 ]; then rm -rf -- "$d"; fi' in script + + +class CheckProvisionIdempotency: + """Running a step twice reports ``changed`` and then ``ok``.""" + + def check_changed_then_ok(self): + first = FakeTransport() + first.when("bash", stdout="CHANGED installed: rsync jq\n") + second = FakeTransport() + second.when("bash", stdout="all 10 packages present\n") + + def classify(transport): + result = transport.run_script("script", check=False) + return CHANGED if "CHANGED" in result.stdout else OK + + assert classify(first) == CHANGED + assert classify(second) == OK + + def check_hosts_step_rewrites_only_between_its_markers(self): + class _Args: # pylint: disable=too-few-public-methods + write_hosts = True + + class _Ctx: # pylint: disable=too-few-public-methods + args = _Args() + config = {"provision": DEFAULTS["provision"]} + + from ducktests_remote.cluster import Node # pylint: disable=import-outside-toplevel + script = provision._hosts_script( # noqa: SLF001 + _Ctx(), [Node(host="node01"), Node(host="node02", ip="10.0.0.2")]) + assert provision.HOSTS_BEGIN in script and provision.HOSTS_END in script + assert "awk" in script and "/etc/hosts" in script + assert "10.0.0.2 node02" in script + assert "node01 node01" in script + assert "> /etc/hosts" not in script.replace('> "$tmp"', ""), \ + "the whole file must never be truncated" + + def check_package_list_is_derived_from_the_dockerfile(self): + packages = DEFAULTS["provision"]["packages"] + for expected in ("rsync", "unzip", "curl", "jq", "iptables", "net-tools", "coreutils"): + assert expected in packages + + def check_user_step_is_not_run_by_default(self): + class _Args: # pylint: disable=too-few-public-methods + only = [] + skip = [] + create_user = None + write_hosts = False + + steps = provision._selected_steps(_Args()) # noqa: SLF001 + assert "user" not in steps, "most operators use their own existing account" + assert "hosts" not in steps + assert "ssh-env" in steps + + def check_only_selects_exactly_one_step(self): + class _Args: # pylint: disable=too-few-public-methods + only = ["packages"] + skip = [] + create_user = None + write_hosts = False + + assert provision._selected_steps(_Args()) == ["packages"] # noqa: SLF001 + + +class CheckFanout: + """Per-host isolation and reporting.""" + + def check_one_failure_does_not_abort_the_batch(self): + def operation(host): + if host == "b": + raise RuntimeError("boom") + return HostResult(host, OK) + + results = fanout(["a", "b", "c"], operation, jobs=4) + assert [r.status for r in results] == [OK, FAILED, OK] + assert summarise(results) == "1 failed, 2 ok" + + def check_fail_fast_stops_scheduling(self): + def operation(host): + if host == "a": + return HostResult(host, FAILED, "no") + return HostResult(host, OK) + + results = fanout(["a", "b", "c"], operation, jobs=1, fail_fast=True) + assert results[0].status == FAILED + assert all(r.status != OK for r in results[1:]) + + def check_two_and_forty_nine_hosts_produce_the_same_shape(self): + small = fanout(["a", "b"], lambda h: HostResult(h, OK), jobs=4) + large = fanout(["h%02d" % i for i in range(49)], lambda h: HostResult(h, OK), jobs=8) + assert {type(r) for r in small} == {type(r) for r in large} + assert len(small) == 2 and len(large) == 49 + + def check_empty_inventory_is_not_an_error(self): + assert fanout([], lambda h: None) == [] + + +class CheckExcludes: + """``ignite-dev`` is normally a link to a checkout; only the built jars are wanted.""" + + @staticmethod + def _checkout(tmp_path): + """A tree shaped like an Ignite source root after a build.""" + root = tmp_path / "ignite-dev" + for rel in ("bin/ignite.sh", + "modules/core/src/main/java/Ignite.java", + "modules/core/target/classes/Ignite.class", + "modules/core/target/ignite-core.jar", + "modules/core/target/libs/dep.jar", + "modules/ducktests/tests/certs/truststore.jks"): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rel, encoding="utf-8") + return root + + @staticmethod + def _ctx(tmp_path, exclude=None, config_exclude=None): + class _Args: # pylint: disable=too-few-public-methods + pass + + class _Ctx: # pylint: disable=too-few-public-methods + pass + + ctx = _Ctx() + ctx.args = _Args() + ctx.args.exclude = list(exclude or []) + ctx.config = {"deploy": {**DEFAULTS["deploy"], "exclude": list(config_exclude or [])}} + ctx.dist_dir = tmp_path + return ctx + + def check_manifest_leaves_out_excluded_files(self, tmp_path): + root = self._checkout(tmp_path) + whole = deploy.build_manifest(root) + filtered = deploy.build_manifest(root, excludes=["src", "classes"]) + assert filtered["files"] == whole["files"] - 2 + assert filtered["excluded"] == 2 + assert filtered["bytes"] < whole["bytes"] + assert filtered["hash"] != whole["hash"] + + def check_the_tarball_holds_exactly_what_the_manifest_counted(self, tmp_path): + root = self._checkout(tmp_path) + excludes = ["src", "classes"] + archive = tmp_path / "payload.tar.gz" + make_tarball(root, archive, excludes=excludes) + with tarfile.open(archive) as tar: + names = sorted(m.name for m in tar.getmembers()) + assert names == ["bin/ignite.sh", + "modules/core/target/ignite-core.jar", + "modules/core/target/libs/dep.jar", + "modules/ducktests/tests/certs/truststore.jks"] + assert len(names) == deploy.build_manifest(root, excludes=excludes)["files"], \ + "a host called up to date must hold the same files the tarball carries" + + def check_no_excludes_ships_the_tree_whole(self, tmp_path): + root = self._checkout(tmp_path) + manifest = deploy.build_manifest(root) + assert manifest["excluded"] == 0 and manifest["files"] == 6 + + def check_command_line_beats_the_ignore_file_and_the_config(self, tmp_path): + root = self._checkout(tmp_path) + (root / deploy.IGNORE_NAME).write_text("classes\n", encoding="utf-8") + ctx = self._ctx(tmp_path, exclude=["src"], config_exclude=["target"]) + assert deploy.resolve_excludes(ctx, root) == ["src", deploy.IGNORE_NAME] + + def check_the_ignore_file_beats_the_config(self, tmp_path): + root = self._checkout(tmp_path) + (root / deploy.IGNORE_NAME).write_text( + "# only the jars are wanted on the workers\n" + "src\n" + "\n" + "classes\n", encoding="utf-8") + ctx = self._ctx(tmp_path, config_exclude=["target"]) + assert deploy.resolve_excludes(ctx, root) == ["src", "classes", deploy.IGNORE_NAME] + + def check_the_ignore_file_is_never_shipped(self, tmp_path): + root = self._checkout(tmp_path) + (root / deploy.IGNORE_NAME).write_text("src\n", encoding="utf-8") + ctx = self._ctx(tmp_path) + excludes = deploy.resolve_excludes(ctx, root) + archive = tmp_path / "payload.tar.gz" + make_tarball(root, archive, excludes=excludes) + with tarfile.open(archive) as tar: + assert deploy.IGNORE_NAME not in [m.name for m in tar.getmembers()] + + def check_the_config_applies_when_nothing_more_specific_exists(self, tmp_path): + root = self._checkout(tmp_path) + ctx = self._ctx(tmp_path, config_exclude=["src", "classes"]) + assert deploy.resolve_excludes(ctx, root) == ["src", "classes"] + + def check_the_default_is_no_filtering(self, tmp_path): + assert deploy.resolve_excludes(self._ctx(tmp_path), self._checkout(tmp_path)) == [] + assert DEFAULTS["deploy"]["exclude"] == [] + + +class _RsyncTransport(FakeTransport): + """A worker whose rsync is present, with the ssh details rsync needs.""" + + rsync = True + + @property + def target(self): + return "tester@w1" + + def ssh_options(self, *, for_scp=False): # pylint: disable=unused-argument + return ["-o", "BatchMode=yes", "-i", "/home/tester/.ssh/id_ed25519"] + + def has_rsync(self): + """:return: whether this worker is on the incremental path.""" + return self.rsync + + +class CheckRsyncFastPath: + """Rebuild one module, send one jar.""" + + STATS = ("Number of files: 1,234 (reg: 1,200, dir: 34)\n" + "Number of regular files transferred: 12\n" + "Total file size: 524,288,000 bytes\n" + "Total transferred file size: 4,194,304 bytes\n") + + @staticmethod + def _args(**kw): + class _Args: # pylint: disable=too-few-public-methods + force = False + sudo = False + owner = None + checksum = False + no_rsync = False + via = None + + args = _Args() + for key, value in kw.items(): + setattr(args, key, value) + return args + + @classmethod + def _ctx(cls, transport, **kw): + class _Console: # pylint: disable=too-few-public-methods + verbose = False + + @staticmethod + def detail(message): + """Swallow the traced command line.""" + + class _Ctx: # pylint: disable=too-few-public-methods + pass + + ctx = _Ctx() + ctx.args = cls._args(**kw) + ctx.config = {"deploy": dict(DEFAULTS["deploy"])} + ctx.console = _Console() + ctx.dry_run = False + ctx.worker = lambda node: transport + return ctx + + @staticmethod + def _dist_and_payload(tmp_path): + root = _dist(tmp_path / "d") + return root, deploy._Payload(root, tmp_path / "p.tar.gz", ()) # noqa: SLF001 + + # -- the command line -------------------------------------------------------- + + def check_an_exact_file_list_is_sent_not_rsync_patterns(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1", + files_from="/tmp/ignite-dev.files") + assert "--files-from=/tmp/ignite-dev.files" in argv + assert not [a for a in argv if a.startswith("--exclude")], \ + "rsync pattern matching differs from is_excluded; the exact list is sent instead" + + def check_unchanged_files_are_hardlinked_against_the_live_distribution(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1", + files_from="/tmp/x.files", link_dest="/opt/ignite-dev") + assert "--link-dest=/opt/ignite-dev" in argv + + def check_a_first_deployment_has_nothing_to_link_against(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1", + files_from="/tmp/x.files") + assert not [a for a in argv if a.startswith("--link-dest")] + + def check_it_lands_in_the_staging_directory_never_in_the_target(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/ignite-dev", "/opt/.tmp.1", + files_from="/tmp/x.files", link_dest="/opt/ignite-dev") + assert argv[-1] == "tester@w1:/opt/.tmp.1/", \ + "an interrupted transfer must not leave a live distribution half updated" + assert argv[-2] == "/dist/ignite-dev/" + + def check_ssh_options_reach_rsync(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/x", "/opt/.tmp.1", + files_from="/tmp/x.files") + assert argv[argv.index("-e") + 1] == \ + "ssh -o BatchMode=yes -i /home/tester/.ssh/id_ed25519" + + def check_sudo_and_checksum_are_passed_through(self): + argv = deploy.rsync_argv(_RsyncTransport(), "/dist/x", "/opt/.tmp.1", + files_from="/tmp/x.files", checksum=True, sudo=True) + assert "--checksum" in argv and "--rsync-path=sudo -n rsync" in argv + + def check_progress_is_asked_of_rsync_only_when_something_displays_it(self): + plain = deploy.rsync_argv(_RsyncTransport(), "/dist/x", "/opt/.tmp.1", + files_from="/tmp/x.files") + watched = deploy.rsync_argv(_RsyncTransport(), "/dist/x", "/opt/.tmp.1", + files_from="/tmp/x.files", progress=True) + assert "--info=progress2" not in plain + assert "--info=progress2" in watched + + # -- stats ------------------------------------------------------------------- + + def check_stats_are_read_back(self): + assert deploy.parse_rsync_stats(self.STATS) == (12, 4194304) + + def check_rsync_2_x_wording_is_understood(self): + assert deploy.parse_rsync_stats("Number of files transferred: 3\n" + "Total transferred file size: 1024 bytes\n") == (3, 1024) + + def check_unreadable_stats_are_not_guessed(self): + assert deploy.parse_rsync_stats("") is None + assert deploy.parse_rsync_stats("all done") is None + + # -- when it is used --------------------------------------------------------- + + @staticmethod + def _posix_with_rsync(monkeypatch): + monkeypatch.setattr(deploy.shutil, "which", lambda _: "/usr/bin/rsync") + monkeypatch.setattr(deploy.os, "name", "posix") + + def check_it_is_on_by_default_when_rsync_is_installed(self, monkeypatch): + self._posix_with_rsync(monkeypatch) + assert deploy.rsync_enabled(self._ctx(None)) is True + + def check_no_rsync_flag_and_config_both_turn_it_off(self, monkeypatch): + self._posix_with_rsync(monkeypatch) + assert deploy.rsync_enabled(self._ctx(None, no_rsync=True)) is False + ctx = self._ctx(None) + ctx.config["deploy"]["rsync"] = False + assert deploy.rsync_enabled(ctx) is False + + def check_via_keeps_the_single_upload(self, monkeypatch): + self._posix_with_rsync(monkeypatch) + assert deploy.rsync_enabled(self._ctx(None, via="build-vm-01")) is False, \ + "--via exists so the payload crosses the slow link once, as one file" + + def check_a_coordinator_without_rsync_falls_back(self, monkeypatch): + monkeypatch.setattr(deploy.os, "name", "posix") + monkeypatch.setattr(deploy.shutil, "which", lambda _: None) + assert deploy.rsync_enabled(self._ctx(None)) is False + + def check_a_windows_coordinator_falls_back(self, monkeypatch): + monkeypatch.setattr(deploy.shutil, "which", lambda _: "rsync.exe") + monkeypatch.setattr(deploy.os, "name", "nt") + assert deploy.rsync_enabled(self._ctx(None)) is False, \ + "rsync would read C:/dist/ignite-dev as host C" + + # -- the transfer ------------------------------------------------------------ + + @staticmethod + def _file_list(tmp_path, root, name="ignite-dev.files"): + path = tmp_path / name + path.write_text("\n".join(deploy.included_files(root)) + "\n", encoding="utf-8") + return path + + def check_the_tarball_is_never_built_when_every_host_takes_rsync(self, tmp_path, + monkeypatch): + root, payload = self._dist_and_payload(tmp_path) + transport = _RsyncTransport() + calls = [] + + def fake_run_local(argv, **kw): + calls.append(argv) + return Result(argv, 0, self.STATS, "", "local") + + monkeypatch.setattr(deploy, "run_local", fake_run_local) + manifest = deploy.build_manifest(root) + files = self._file_list(tmp_path, root) + result = deploy._deploy_to_host( # noqa: SLF001 + self._ctx(transport), Node(host="w1", user="tester"), "ignite-dev", + "/opt/ignite-dev", manifest, "{}", payload, None, None, files) + + assert not (tmp_path / "p.tar.gz").exists(), \ + "compressing a linked checkout would cost more than the transfer saves" + assert len(calls) == 1 + assert "--files-from=%s" % files in calls[0] + assert files.read_text(encoding="utf-8") == "bin/ignite.sh\nlibs/core.jar\n" + assert result.status == CHANGED and "rsync: 12 of 2 file(s) changed" in result.message + + def check_the_staging_tree_is_still_swapped_into_place(self, tmp_path, monkeypatch): + root, payload = self._dist_and_payload(tmp_path) + transport = _RsyncTransport() + monkeypatch.setattr(deploy, "run_local", + lambda argv, **kw: Result(argv, 0, self.STATS, "", "local")) + deploy._deploy_to_host( # noqa: SLF001 + self._ctx(transport), Node(host="w1", user="tester"), "ignite-dev", + "/opt/ignite-dev", deploy.build_manifest(root), "{}", payload, None, None, + self._file_list(tmp_path, root)) + scripts = "\n".join(transport.scripts) + assert 'mv -- "$staging" "$target"' in scripts + assert deploy.MANIFEST_NAME in "".join(transport.files) + + def check_a_worker_without_rsync_gets_the_tarball(self, tmp_path, monkeypatch): + root, payload = self._dist_and_payload(tmp_path) + transport = _RsyncTransport() + transport.rsync = False + monkeypatch.setattr(deploy, "run_local", + lambda argv, **kw: pytest.fail("rsync must not run here")) + result = deploy._deploy_to_host( # noqa: SLF001 + self._ctx(transport), Node(host="w1", user="tester"), "ignite-dev", + "/opt/ignite-dev", deploy.build_manifest(root), "{}", payload, None, None, + self._file_list(tmp_path, root)) + assert (tmp_path / "p.tar.gz").exists() + assert transport.uploads and result.status == CHANGED + + def check_a_failed_rsync_leaves_the_target_alone(self, tmp_path, monkeypatch): + root, payload = self._dist_and_payload(tmp_path) + transport = _RsyncTransport() + monkeypatch.setattr(deploy, "run_local", + lambda argv, **kw: Result(argv, 23, "", "permission denied", + "local")) + result = deploy._deploy_to_host( # noqa: SLF001 + self._ctx(transport), Node(host="w1", user="tester"), "ignite-dev", + "/opt/ignite-dev", deploy.build_manifest(root), "{}", payload, None, None, + self._file_list(tmp_path, root)) + assert result.status == FAILED and "permission denied" in result.detail + assert 'mv -- "$staging" "$target"' not in "\n".join(transport.scripts) + + def check_the_file_list_matches_the_manifest(self, tmp_path): + root = CheckExcludes._checkout(tmp_path) # noqa: SLF001 + excludes = ["src", "classes"] + files = deploy.included_files(root, excludes) + assert files == ["bin/ignite.sh", + "modules/core/target/ignite-core.jar", + "modules/core/target/libs/dep.jar", + "modules/ducktests/tests/certs/truststore.jks"] + assert len(files) == deploy.build_manifest(root, excludes=excludes)["files"] diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_globals.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_globals.py new file mode 100644 index 0000000000000..509250428c647 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_globals.py @@ -0,0 +1,143 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for globals composition, interpolation and secret redaction.""" + +import json + +import pytest + +from ducktests_remote.config import ConfigError +from ducktests_remote.globals_builder import (Redactor, build, dumps, load_raw_layer, + parse_kv_override) + +SECRET = "s3cr3t-passphrase" + + +class CheckLayering: + """Deep merge, list replacement, override precedence.""" + + def check_later_layer_wins_per_key(self): + composed, _ = build([("base", {"ssl": {"enabled": False}, "project": "ignite"}), + ("profile", {"ssl": {"enabled": True}})]) + assert composed == {"ssl": {"enabled": True}, "project": "ignite"} + + def check_lists_replace(self): + composed, _ = build([("base", {"ignite_versions": ["2.16.0", "2.17.0"]}), + ("profile", {"ignite_versions": ["dev"]})]) + assert composed["ignite_versions"] == ["dev"] + + def check_raw_jenkins_blob_is_a_usable_base_layer(self): + blob = '{"project": "ise", "ignite_versions": ["ise-0-32"]}' + raw = load_raw_layer(blob, None) + composed, _ = build([("blob", raw), ("profile", {"project": "ignite"})]) + assert composed == {"project": "ignite", "ignite_versions": ["ise-0-32"]} + + def check_raw_blob_must_be_an_object(self): + with pytest.raises(ConfigError): + load_raw_layer("[1, 2, 3]", None) + + +class CheckOverrides: + """``-g a.b.c=value`` mechanics.""" + + def check_dotted_path_nests(self): + composed, _ = build([], ["ssl.key_store.path=/tmp/ks"]) + assert composed == {"ssl": {"key_store": {"path": "/tmp/ks"}}} + + def check_json_values_are_coerced(self): + composed, _ = build([], ["ssl.enabled=true", "count=3", "project=ise"]) + assert composed["ssl"]["enabled"] is True + assert composed["count"] == 3 + assert composed["project"] == "ise", "a bare word stays a string" + + def check_override_beats_the_layers(self): + composed, _ = build([("profile", {"project": "ignite"})], ["project=ise"]) + assert composed["project"] == "ise" + + def check_missing_equals_is_an_error(self): + with pytest.raises(ConfigError): + parse_kv_override("just-a-key") + + +class CheckInterpolation: + """``${env:}`` and ``${file:}``.""" + + def check_env_is_resolved(self): + composed, _ = build([("p", {"authentication": {"password": "${env:ISE_PASSWORD}"}})], + environ={"ISE_PASSWORD": SECRET}) + assert composed["authentication"]["password"] == SECRET + + def check_missing_env_names_the_variable_and_the_file(self): + with pytest.raises(ConfigError) as ex: + build([("profile-ise.yaml", {"p": "${env:NOT_SET_ANYWHERE}"})], environ={}) + message = str(ex.value) + assert "NOT_SET_ANYWHERE" in message and "profile-ise.yaml" in message + + def check_missing_env_never_becomes_an_empty_string(self): + with pytest.raises(ConfigError): + build([("p", {"password": "${env:ABSENT}"})], environ={}) + + def check_file_is_read_and_trimmed(self, tmp_path): + path = tmp_path / "pass.txt" + path.write_text(SECRET + "\n", encoding="utf-8") + composed, _ = build([("p", {"password": "${file:%s}" % path})], environ={}) + assert composed["password"] == SECRET + + def check_missing_file_is_an_error(self, tmp_path): + with pytest.raises(ConfigError): + build([("p", {"password": "${file:%s}" % (tmp_path / "nope")})], environ={}) + + def check_placeholders_inside_lists_are_resolved(self): + composed, _ = build([("p", {"versions": ["${env:V}"]})], environ={"V": "ise-0-32"}) + assert composed["versions"] == ["ise-0-32"] + + +class CheckRedaction: + """A resolved secret must not survive anywhere the CLI writes.""" + + def _composed(self): + redactor = Redactor() + composed, _ = build([("p", {"authentication": {"password": "${env:ISE_PASSWORD}"}, + "note": "connect with %s please" % SECRET})], + redactor=redactor, environ={"ISE_PASSWORD": SECRET}) + return composed, redactor + + def check_value_based_redaction_catches_an_unrelated_field(self): + composed, redactor = self._composed() + masked = redactor.redact_structure(composed) + assert SECRET not in json.dumps(masked) + assert masked["note"] == "connect with *** please" + + def check_rendered_output_is_clean(self): + composed, redactor = self._composed() + rendered = dumps(redactor.redact_structure(composed)) + assert SECRET not in rendered + + def check_arbitrary_text_is_redacted(self): + _, redactor = self._composed() + line = "ssh failed: tried password %s" % SECRET + assert SECRET not in redactor.redact(line) + + def check_key_name_fallback_masks_unresolved_secrets(self): + redactor = Redactor() + masked = redactor.redact_structure({"password": "typed-inline-not-from-env"}) + assert masked["password"] == "***" + + def check_the_written_globals_file_still_contains_the_real_value(self): + # Redaction is for output only; ducktape needs the real credentials on the runner, + # which is why globals.json is written with mode 0600 and excluded from fetch. + composed, _ = self._composed() + assert composed["authentication"]["password"] == SECRET diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py new file mode 100644 index 0000000000000..bd5260318aae2 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_java.py @@ -0,0 +1,493 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for JDK resolution, the environment files, and the archive inspection.""" + +import io +import json +import tarfile + +import pytest + +from fake_transport import FakeTransport + +from ducktests_remote import java +from ducktests_remote.cli import Console, Context +from ducktests_remote.cluster import Node +from ducktests_remote.commands import doctor, provision +from ducktests_remote.config import ConfigError, load_config + +PROBE = """java_requested=17 +java=openjdk version "11.0.19" 2023-04-18 +java_path=/usr/bin/java +java_path_version=11.0.19 +java_path_major=11 +java_path_real=/usr/lib/jvm/java-11-openjdk/bin/java +java_env_home= +java_candidates=/usr/lib/jvm/java-11-openjdk:11:11.0.19,/opt/jdk-17.0.9:17:17.0.9,\ +/opt/jdk-17.0.11:17:17.0.11 +""" + +MATCHING = """java_requested=17 +java=openjdk version "17.0.11" 2024-04-16 +java_path=/usr/bin/java +java_path_version=17.0.11 +java_path_major=17 +java_path_real=/opt/jdk-17.0.11/bin/java +java_env_home=/opt/jdk-17.0.11 +java_candidates=/opt/jdk-17.0.11:17:17.0.11 +""" + + +def _cfg(**kw): + kw.setdefault("major", 17) + kw.setdefault("search_paths", ["/opt", "/usr/lib/jvm"]) + return java.JavaConfig(**kw) + + +class CheckVersionParsing: + """Mirrors ignitetest's jvm_utils.java_major_version.""" + + @pytest.mark.parametrize("version,major", [ + ("1.8.0_292", 8), ("11.0.19", 11), ("17.0.11+9", 17), ("21", 21), ("11-ea", 11), + ("", None), ("garbage", None), + ]) + def check_major(self, version, major): + assert java.major_of(version) == major + + def check_patch_levels_order_numerically(self): + assert java.version_key("17.0.9") < java.version_key("17.0.11") + + +class CheckResolution: + """The ladder, exercised against recorded probe output.""" + + def check_the_newest_matching_jdk_is_chosen(self): + res = java.parse_probe("w1", PROBE, _cfg()) + assert res.home == "/opt/jdk-17.0.11", "17.0.9 sorts after 17.0.11 as a string" + assert res.source == java.SEARCH + assert res.path_matches(17) is False, "PATH still points at Java 11" + + def check_a_matching_java_on_path_is_used_as_is(self): + res = java.parse_probe("w1", MATCHING, _cfg()) + assert res.source == java.CURRENT + assert res.home == "/opt/jdk-17.0.11" + assert res.path_matches(17) and res.home_in_effect + + def check_an_explicit_home_short_circuits_the_search(self): + res = java.parse_probe("w1", PROBE, _cfg(home="/opt/jdk-17.0.9")) + assert res.home == "/opt/jdk-17.0.9" and res.source == java.EXPLICIT + + def check_an_explicit_home_that_is_absent_selects_nothing(self): + # Deliberately not a fallback: an explicit java.home that silently resolves to a + # different JVM is worse than a failure naming the host. + assert not java.parse_probe("w1", PROBE, _cfg(home="/opt/nope")).selected + + def check_no_matching_jdk_selects_nothing_but_reports_what_is_there(self): + res = java.parse_probe("w1", PROBE, _cfg(major=21)) + assert not res.selected + assert ("/opt/jdk-17.0.11", 17, "17.0.11") in res.candidates + + def check_no_requested_version_accepts_whatever_is_there(self): + res = java.parse_probe("w1", PROBE, _cfg(major=None)) + assert res.home == "/usr/lib/jvm/java-11-openjdk" and res.source == java.CURRENT + + def check_a_symlinked_java_still_counts_as_the_selected_home(self): + res = java.parse_probe("w1", MATCHING, _cfg()) + assert res.home_in_effect, "/usr/bin/java resolves into the selected home" + + def check_an_empty_probe_is_survivable(self): + res = java.parse_probe("w1", "", _cfg()) + assert not res.selected and res.path_major is None + + +class CheckDiscoveryScript: + """The generated shell has to be safe to run from doctor.""" + + def check_paths_are_quoted(self): + script = java.discovery_script(_cfg(search_paths=["/opt/my jdks"], + home="/opt/vendor jdk")) + assert "'/opt/my jdks'" in script and "explicit='/opt/vendor jdk'" in script + + def check_it_never_writes_anything(self): + script = java.discovery_script(_cfg()) + for mutation in ("mkdir", "rm ", "mv ", "install", ">>", "chmod"): + assert mutation not in script, "discovery must be read-only: %r" % mutation + + def check_it_always_exits_zero(self): + # One unusable host must not abort the fan-out; the status is in the fields. + assert java.discovery_script(_cfg()).rstrip().endswith("exit 0") + + +class CheckEnvScript: + """Both environment files, written from one resolved value.""" + + def check_both_files_are_written(self): + script = java.env_script(_cfg(), "/opt/jdk-17") + assert "~/.ssh/environment" in script and "~/.bashrc" in script + + def check_each_file_can_be_switched_off(self): + # Composed in Python rather than guarded at runtime, so --dry-run shows only what + # the step will really do - and so a disabled section cannot print a note about + # itself as the fallback. + without_bashrc = java.env_script(_cfg(bashrc=False), "/opt/jdk-17") + assert "~/.bashrc" not in without_bashrc + assert "java.bashrc is off" in without_bashrc, "and the note says so" + without_ssh = java.env_script(_cfg(ssh_environment=False), "/opt/jdk-17") + assert "~/.ssh/environment" not in without_ssh and "~/.bashrc" in without_ssh + + def check_switching_both_off_is_refused(self): + with pytest.raises(ConfigError): + java.env_script(_cfg(ssh_environment=False, bashrc=False), "/opt/jdk-17") + + def check_the_bashrc_block_goes_above_the_interactivity_guard(self): + # The stock ~/.bashrc returns early for non-interactive shells, which is exactly + # the case ducktape runs in, so appending would be writing to /dev/null. + script = java.env_script(_cfg(), "/opt/jdk-17") + assert script.index(java.BLOCK_BEGIN) < script.index("awk 'BEGIN{skip=0}"), \ + "the block is emitted before the existing file is appended to it" + + def check_the_jdk_is_prepended_to_path_not_appended(self): + assert 'want_path="PATH=$jh/bin:$base_path' in java.env_script(_cfg(), "/opt/jdk-17") + + def check_the_path_is_not_grown_on_every_run(self): + # A host that honours ~/.ssh/environment feeds the composed PATH straight back in. + script = java.env_script(_cfg(), "/opt/jdk-17", ["/opt/venv/bin"]) + assert 'strip="$jh/bin:/opt/venv/bin"' in script + + def check_it_refuses_a_home_without_java(self): + assert '[ -x "$jh/bin/java" ]' in java.env_script(_cfg(), "/opt/not-a-jdk") + + +class CheckVerifyScript: + """What a fresh session gets is the authority, so it must be parseable.""" + + def check_it_reports_the_fields_the_resolver_reads(self): + script = java.verify_script() + for key in ("java_path", "java_path_version", "java_path_major", "java_env_home"): + assert "say %s " % key in script + + def check_its_output_round_trips(self): + verified = """java=openjdk version "17.0.11" 2024-04-16 +java_path=/usr/bin/java +java_path_version=17.0.11 +java_path_major=17 +java_path_real=/opt/jdk-17.0.11/bin/java +""" + assert java.parse_probe("w1", verified, _cfg()).path_matches(17) + + +def _tarball(tmp_path, names, top="jdk-17.0.11+9"): + path = tmp_path / "jdk.tar.gz" + with tarfile.open(path, "w:gz") as tar: + for name in names: + full = "%s/%s" % (top, name) if top else name + info = tarfile.TarInfo(full) + info.size = 4 + tar.addfile(info, io.BytesIO(b"data")) + return path + + +class CheckArchivePlan: + """A bad archive must fail on the coordinator, before it is copied anywhere.""" + + def check_a_single_top_level_directory_is_stripped(self, tmp_path): + plan = java.archive_plan(_tarball(tmp_path, ["bin/java", "lib/modules"])) + assert plan.strip == 1 and plan.top_level == "jdk-17.0.11+9" + assert plan.name == "jdk-17.0.11+9", "the target directory takes the JDK's own name" + assert plan.bytes == 8 + + def check_a_macos_build_is_refused_by_name(self, tmp_path): + # It unpacks perfectly well and then fails on every worker, so it has to be + # caught here rather than on twelve hosts at once. + with pytest.raises(ConfigError) as ex: + java.archive_plan(_tarball(tmp_path, ["Contents/Home/bin/java"])) + assert "macOS" in str(ex.value) and "Linux x64" in str(ex.value) + + def check_an_archive_without_any_java_is_refused(self, tmp_path): + with pytest.raises(ConfigError) as ex: + java.archive_plan(_tarball(tmp_path, ["lib/modules", "release"])) + assert "does not contain bin/java" in str(ex.value) + + def check_a_flat_archive_is_not_stripped(self, tmp_path): + plan = java.archive_plan(_tarball(tmp_path, ["bin/java", "lib/modules"], top="")) + assert plan.strip == 0 and plan.top_level is None + assert plan.name == "jdk" + + def check_a_deeper_layout_is_stripped_to_the_java_home(self, tmp_path): + # `tar czf` of a directory that itself holds the unpacked JDK. The wrapper is + # not a reason to refuse the archive; it is a reason to strip one level more. + plan = java.archive_plan( + _tarball(tmp_path, ["jdk-17.0.11+9/bin/java", "jdk-17.0.11+9/lib/modules"], + top="openjdk-17")) + assert plan.strip == 2 and plan.top_level == "openjdk-17/jdk-17.0.11+9" + assert plan.name == "jdk-17.0.11+9", "the target takes the JDK's own name" + + def check_extra_entries_beside_the_jdk_do_not_confuse_the_plan(self, tmp_path): + # AppleDouble junk, a stray LICENSE, a second top-level file: none of it changes + # where bin/java is. + path = tmp_path / "jdk.tar.gz" + with tarfile.open(path, "w:gz") as tar: + for name in ("._jdk-17.0.11", "LICENSE", "jdk-17.0.11/bin/java"): + info = tarfile.TarInfo(name) + info.size = 4 + tar.addfile(info, io.BytesIO(b"data")) + plan = java.archive_plan(str(path)) + assert plan.strip == 1 and plan.top_level == "jdk-17.0.11" + + def check_the_shallowest_java_wins(self, tmp_path): + # A bundled JRE deeper in the tree must not drag the strip depth with it. + plan = java.archive_plan( + _tarball(tmp_path, ["bin/java", "legal/jre/bin/java"], top="jdk-17")) + assert plan.strip == 1 and plan.top_level == "jdk-17" + + def check_the_tar_flag_follows_the_suffix(self, tmp_path): + # The worker-side `tar` is told how to decompress; guessing gzip for an .xz + # archive passes every coordinator-side check and fails on every host. + gz = java.archive_plan(_tarball(tmp_path, ["bin/java"])) + assert gz.tar_flag == "z" + + xz_path = tmp_path / "jdk.tar.xz" + with tarfile.open(xz_path, "w:xz") as tar: + info = tarfile.TarInfo("jdk-17/bin/java") + info.size = 4 + tar.addfile(info, io.BytesIO(b"data")) + assert java.archive_plan(str(xz_path)).tar_flag == "J" + + def check_a_directory_source_is_accepted(self, tmp_path): + home = tmp_path / "jdk-17" + (home / "bin").mkdir(parents=True) + (home / "bin" / "java").write_text("#!/bin/sh\n", encoding="utf-8") + plan = java.archive_plan(str(home)) + assert plan.kind == "dir" and plan.name == "jdk-17" + + def check_a_directory_without_bin_java_is_refused(self, tmp_path): + (tmp_path / "empty").mkdir() + with pytest.raises(ConfigError): + java.archive_plan(str(tmp_path / "empty")) + + def check_a_zip_names_the_supported_formats(self, tmp_path): + path = tmp_path / "jdk.zip" + path.write_bytes(b"PK\x03\x04") + with pytest.raises(ConfigError) as ex: + java.archive_plan(str(path)) + assert ".tar.gz" in str(ex.value) + + def check_a_missing_archive_names_the_path(self, tmp_path): + with pytest.raises(ConfigError) as ex: + java.archive_plan(str(tmp_path / "absent.tar.gz")) + assert "absent.tar.gz" in str(ex.value) + + def check_the_target_directory_can_be_renamed(self, tmp_path): + plan = java.archive_plan(_tarball(tmp_path, ["bin/java"])) + assert java.target_dir(_cfg(install_root="/opt", name="jdk-17"), plan) == "/opt/jdk-17" + assert java.target_dir(_cfg(install_root="/opt"), plan) == "/opt/jdk-17.0.11+9" + + +def _facts(text): + return {line.split("=", 1)[0]: line.split("=", 1)[1] + for line in text.strip().splitlines() if "=" in line} + + +def _context(probe_output, **java_cfg): + """A context whose single worker answers the JDK probe with ``probe_output``.""" + class Args: # pylint: disable=too-few-public-methods + """Minimal stand-in for the parsed command line.""" + + args = Args() + args.dry_run = False + args.sudo = False + args.force = False + args.install_jdk = False + args.num_nodes = None + + config = load_config(user_config=None) + config["java"].update(java_cfg) + config["cluster"]["nodes"] = [{"host": "w1"}] + + ctx = Context(config, args, Console(color=False)) + fake = FakeTransport(name="w1") + fake.when("java_requested", probe_output) + ctx._workers["w1"] = fake # noqa: SLF001 - the point of the fake + return ctx, fake + + +NODE = Node(host="w1", user="max", port=22, identity_file=None) + + +class CheckDelivery: + """`provision --only jdk`, on one host, with no network and no processes.""" + + def check_a_host_that_already_matches_is_left_alone(self): + ctx, fake = _context(MATCHING) + result = provision._jdk_on_host( # noqa: SLF001 + ctx, NODE, java.config_of(ctx), None, java.discovery_script(java.config_of(ctx))) + assert result.status == provision.OK + assert not fake.uploads, "nothing may be sent to a host that is already correct" + + def check_a_host_without_a_match_gets_the_archive(self, tmp_path): + archive = _tarball(tmp_path, ["bin/java", "lib/modules"]) + ctx, fake = _context(PROBE, major=21, archive=str(archive), install_root="/opt") + cfg = java.config_of(ctx) + plan = java.archive_plan(cfg.archive) + result = provision._jdk_on_host( # noqa: SLF001 + ctx, NODE, cfg, provision.JdkPayload(plan), java.discovery_script(cfg)) + + assert result.status == provision.CHANGED + assert fake.uploads, "the archive has to reach the host" + scripts = "\n".join(fake.scripts) + assert "--strip-components=1" in scripts, "a Temurin tarball has one top-level dir" + assert "/opt/jdk-17.0.11+9" in scripts, "the swap targets the JDK's own name" + manifest = [body for path, (body, _) in fake.files.items() + if path.endswith(provision.JAVA_MANIFEST_NAME)] + assert manifest and json.loads(manifest[0])["hash"] + + def check_a_host_that_already_has_that_archive_is_skipped(self, tmp_path): + archive = _tarball(tmp_path, ["bin/java"]) + ctx, fake = _context(PROBE, major=21, archive=str(archive), install_root="/opt") + cfg = java.config_of(ctx) + plan = java.archive_plan(cfg.archive) + digest = provision._tar_manifest(plan)["hash"] # noqa: SLF001 + fake.when("cat", json.dumps({"hash": digest})) + + result = provision._jdk_on_host( # noqa: SLF001 + ctx, NODE, cfg, provision.JdkPayload(plan), java.discovery_script(cfg)) + assert result.status == provision.OK and not fake.uploads + + def check_the_delivery_reports_its_bytes(self, tmp_path): + archive = _tarball(tmp_path, ["bin/java"]) + ctx, fake = _context(PROBE, major=21, archive=str(archive), install_root="/opt") + cfg = java.config_of(ctx) + plan = java.archive_plan(cfg.archive) + seen = [] + + class _Progress: # pylint: disable=too-few-public-methods + """Records what a real display would draw.""" + + live = False + watching = True + + @staticmethod + def phase(host, phase): + seen.append((host, phase)) + + @staticmethod + def sent(host, sent, total=None, fraction=None): + seen.append((host, "sent", sent, total)) + + @staticmethod + def done(host, message=""): + seen.append((host, "done")) + + provision._jdk_on_host( # noqa: SLF001 + ctx, NODE, cfg, provision.JdkPayload(plan), java.discovery_script(cfg), + _Progress()) + phases = [entry[1] for entry in seen] + assert phases[:2] == ["probing", "sending"] + assert "extracting" in phases and "swapping" in phases + assert any(entry[1] == "sent" for entry in seen), \ + "a JDK is a few hundred megabytes; it has to show movement" + assert fake.reported, "the upload must be the watched one" + + def check_a_delivery_that_is_skipped_reports_nothing(self, tmp_path): + archive = _tarball(tmp_path, ["bin/java"]) + ctx, fake = _context(PROBE, major=21, archive=str(archive), install_root="/opt") + cfg = java.config_of(ctx) + plan = java.archive_plan(cfg.archive) + fake.when("cat", json.dumps({"hash": provision._tar_manifest(plan)["hash"]})) # noqa: SLF001 + provision._jdk_on_host( # noqa: SLF001 + ctx, NODE, cfg, provision.JdkPayload(plan), java.discovery_script(cfg)) + assert not fake.reported and not fake.uploads + + def check_no_archive_and_no_match_fails_with_the_config_keys(self): + ctx, _ = _context(PROBE, major=21) + cfg = java.config_of(ctx) + result = provision._jdk_on_host(ctx, NODE, cfg, None, # noqa: SLF001 + java.discovery_script(cfg)) + assert result.status == provision.FAILED + assert "java.archive" in result.message and "java.home" in result.message + + def check_a_missing_explicit_home_is_never_papered_over(self, tmp_path): + # Even with an archive available: java.home means that JDK, not a similar one. + archive = _tarball(tmp_path, ["bin/java"]) + ctx, fake = _context(PROBE, home="/opt/vendor-jdk", archive=str(archive)) + cfg = java.config_of(ctx) + result = provision._jdk_on_host( # noqa: SLF001 + ctx, NODE, cfg, provision.JdkPayload(java.archive_plan(cfg.archive)), + java.discovery_script(cfg)) + assert result.status == provision.FAILED and "/opt/vendor-jdk" in result.message + assert not fake.uploads + + +class CheckSshEnvStep: + """The verification, not the edit, decides the outcome.""" + + def check_a_session_that_still_gets_the_wrong_jvm_fails(self): + ctx, fake = _context(PROBE) # PATH java is 11, selected is 17 + fake.when("say java_path", PROBE) # ... and stays 11 after the write + results = provision._run_ssh_env_step(ctx, [NODE]) # noqa: SLF001 + assert results[0].status == provision.FAILED + assert "still gets" in results[0].message + + def check_a_session_that_gets_the_right_jvm_passes(self): + ctx, fake = _context(PROBE) + # Only the verify script matches this needle: the discovery script is answered by + # the earlier `java_requested` response, so this is the session *after* the write. + fake.when("say java_path", MATCHING) + results = provision._run_ssh_env_step(ctx, [NODE]) # noqa: SLF001 + assert results[0].status in (provision.OK, provision.CHANGED) + assert any("~/.bashrc" in script for script in fake.scripts) + assert any("~/.ssh/environment" in script for script in fake.scripts) + + def check_nothing_to_point_at_fails_before_writing(self): + ctx, fake = _context(PROBE, major=21) + results = provision._run_ssh_env_step(ctx, [NODE]) # noqa: SLF001 + assert results[0].status == provision.FAILED + assert "provision --only jdk" in results[0].message + assert not any("~/.bashrc" in s for s in fake.scripts) + + +class CheckDoctorVerdicts: + """The preflight judges what a non-interactive session gets, nothing else.""" + + def check_a_matching_jdk_passes(self): + checks = doctor._java_checks({"w1": _facts(MATCHING)}, _cfg()) # noqa: SLF001 + assert [c.status for c in checks] == [doctor.OK] + + def check_a_wrong_version_fails_and_names_the_remedy(self): + checks = doctor._java_checks({"w1": _facts(PROBE)}, _cfg()) # noqa: SLF001 + assert checks[0].status == doctor.FAIL + assert "/opt/jdk-17.0.11" in checks[0].message and "provision" in checks[0].message + + def check_a_wrong_version_with_nothing_installed_names_the_config_keys(self): + checks = doctor._java_checks({"w1": _facts(PROBE)}, _cfg(major=21)) # noqa: SLF001 + assert checks[0].status == doctor.FAIL + assert "java.archive" in checks[0].message + + def check_a_missing_java_fails(self): + checks = doctor._java_checks({"w1": {}}, _cfg()) # noqa: SLF001 + assert checks[0].status == doctor.FAIL and "PATH" in checks[0].message + + def check_an_explicit_home_that_is_not_in_effect_only_warns(self): + # Right version, wrong JDK: the tests will still run, so this is not a failure. + checks = doctor._java_checks( # noqa: SLF001 + {"w1": _facts(MATCHING)}, _cfg(home="/opt/jdk-17-vendor")) + assert checks[0].status == doctor.WARN and "java.home" in checks[0].message + + def check_mixed_jdks_are_reported_once_for_the_cluster(self): + facts = {"w1": _facts(MATCHING), "w2": _facts(MATCHING)} + facts["w2"]["java_path_version"] = "17.0.2" + checks = doctor._java_checks(facts, _cfg()) # noqa: SLF001 + assert [c for c in checks if c.name == "java-consistency"] diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_pip.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_pip.py new file mode 100644 index 0000000000000..1dd6452ea450e --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_pip.py @@ -0,0 +1,147 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for the pip index configuration and for the commands that carry it.""" + +import pytest + +from fake_transport import FakeTransport + +from ducktests_remote import pipconf +from ducktests_remote.cli import Console, Context +from ducktests_remote.commands import run +from ducktests_remote.config import ConfigError, load_config + + +def _config(**pip): + config = load_config(user_config=None) + config["pip"].update(pip) + return config + + +class CheckArguments: + """What ends up on the pip command line.""" + + def check_nothing_configured_adds_no_arguments(self): + # The rendered command has to stay byte for byte what it was before this section + # existed, or every golden-string check in here becomes a lie. + assert pipconf.pip_args(_config()) == [] + assert pipconf.pip_args_str(_config()) == "" + + def check_full_configuration_in_a_fixed_order(self): + config = _config(index_url="https://nexus.invalid/simple", + extra_index_url=["https://nexus.invalid/internal"], + trusted_host=["nexus.invalid"], timeout=60, retries=5, + cert="/etc/pki/corp.pem") + assert pipconf.pip_args(config) == [ + "--index-url", "https://nexus.invalid/simple", + "--extra-index-url", "https://nexus.invalid/internal", + "--trusted-host", "nexus.invalid", + "--timeout", "60", + "--retries", "5", + "--cert", "/etc/pki/corp.pem", + ] + + def check_a_bare_string_is_accepted_where_a_list_is_expected(self): + config = _config(extra_index_url="https://a.invalid/s", trusted_host="a.invalid") + assert pipconf.pip_args(config) == ["--extra-index-url", "https://a.invalid/s", + "--trusted-host", "a.invalid"] + + def check_repeatable_values_are_repeated(self): + config = _config(trusted_host=["a.invalid", "b.invalid"]) + assert pipconf.pip_args(config).count("--trusted-host") == 2 + + def check_blank_entries_are_dropped(self): + assert pipconf.pip_args(_config(index_url=" ", trusted_host=["", " "])) == [] + + def check_arguments_are_quoted_for_a_script(self): + rendered = pipconf.pip_args_str(_config(cert="/etc/pki/corp ca.pem")) + assert "'/etc/pki/corp ca.pem'" in rendered + + def check_a_bad_timeout_names_the_key(self): + with pytest.raises(ConfigError) as ex: + pipconf.pip_args(_config(timeout="soon")) + assert "pip.timeout" in str(ex.value) + + def check_a_negative_timeout_is_rejected(self): + with pytest.raises(ConfigError): + pipconf.pip_args(_config(timeout=0)) + + def check_a_non_string_index_names_the_key(self): + with pytest.raises(ConfigError) as ex: + pipconf.pip_args(_config(index_url=["https://a.invalid"])) + assert "pip.index_url" in str(ex.value) + + +class CheckDescription: + """The one-line summary doctor prints.""" + + def check_credentials_never_reach_the_terminal(self): + # This URL came straight from a config file, so the value-keyed redactor has + # never seen it; masking the userinfo is the only thing standing in the way. + line = pipconf.describe(_config(index_url="https://bob:s3cret@nexus.invalid/simple")) + assert "s3cret" not in line and "bob" not in line + assert "https://***@nexus.invalid/simple" in line + + def check_no_configuration_says_pypi(self): + assert "PyPI" in pipconf.describe(_config()) + + def check_every_configured_value_is_mentioned(self): + line = pipconf.describe(_config(index_url="https://nexus.invalid/simple", + trusted_host="nexus.invalid", timeout=60, + retries=5, cert="/etc/pki/corp.pem")) + for expected in ("nexus.invalid", "trusted", "60", "5", "corp.pem"): + assert expected in line + + +def _context(config, **args): + class Args: # pylint: disable=too-few-public-methods + """Minimal stand-in for the parsed command line.""" + + parsed = Args() + parsed.dry_run = False + for key, value in args.items(): + setattr(parsed, key, value) + ctx = Context(config, parsed, Console(color=False)) + ctx._runner = FakeTransport() # noqa: SLF001 - the point of the fake + return ctx + + +class CheckCallSites: + """Both pip invocations have to carry the configuration, not just the first.""" + + def check_the_venv_install_carries_the_index(self): + config = _config(index_url="https://nexus.invalid/simple", trusted_host="nexus.invalid") + ctx = _context(config) + ctx.runner.when("import ducktape", "ducktape 0.13.0") + run._ensure_venv(ctx, "/work") # noqa: SLF001 - internal by design + script = "\n".join(ctx.runner.scripts) + assert "--index-url https://nexus.invalid/simple" in script + assert "--trusted-host nexus.invalid" in script + + def check_install_sources_carries_the_index_too(self): + # It did not, before pip.* existed: `--install-sources` went to PyPI whatever the + # configured index was, and the run failed minutes into a job nobody was watching. + ctx = _context(_config(index_url="https://nexus.invalid/simple")) + run._install_sources(ctx, "/work") # noqa: SLF001 - internal by design + argv = ctx.runner.commands[-1] + assert "--index-url" in argv and "https://nexus.invalid/simple" in argv + assert argv[-2:] == ["-e", "/work/modules/ducktests/tests"] + + def check_an_unconfigured_venv_script_stays_unchanged(self): + ctx = _context(_config()) + ctx.runner.when("import ducktape", "ducktape 0.13.0") + run._ensure_venv(ctx, "/work") # noqa: SLF001 - internal by design + assert "install --disable-pip-version-check -r" in "\n".join(ctx.runner.scripts) diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_progress.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_progress.py new file mode 100644 index 0000000000000..93814e2aacb6c --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_progress.py @@ -0,0 +1,284 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for the progress display and the streamed transfers that feed it.""" + +import io +import sys + +from ducktests_remote import progress as progress_mod +from ducktests_remote.commands import deploy +from ducktests_remote.progress import (NullProgress, Progress, build_progress, human_bytes, + human_duration, is_a_terminal) +from ducktests_remote.transport import LocalTransport, run_local_streaming + + +class _Stream(io.StringIO): + """A StringIO that can claim to be a terminal, with a settable encoding.""" + + def __init__(self, tty=False, encoding="utf-8"): + super().__init__() + self._tty = tty + self._encoding = encoding + + @property + def encoding(self): + """:return: the encoding the bar picks its characters from.""" + return self._encoding + + def isatty(self): + return self._tty + + +class CheckFormatting: + """The numbers an operator reads at a glance.""" + + def check_bytes_are_scaled(self): + assert human_bytes(512) == "512 B" + assert human_bytes(1024 * 1024 * 3) == "3.0 MB" + assert human_bytes(1024 ** 3 * 3.5) == "3.5 GB" + + def check_durations_grow_an_hour_field_only_when_needed(self): + assert human_duration(72) == "1:12" + assert human_duration(3672) == "1:01:12" + + +class CheckAggregate: + """The line that has to make sense on its own in a log file.""" + + @staticmethod + def _progress(hosts, **kw): + return Progress(hosts, stream=_Stream(), live=False, **kw) + + def check_it_averages_the_per_host_fractions(self): + progress = self._progress(["w1", "w2", "w3", "w4"]) + progress.done("w1") + progress.sent("w2", 50, 100) + assert "38% 1/4 host(s)" in progress.aggregate_line(), \ + "one host finished plus one half done is 3/8 of four hosts" + + def check_it_counts_finished_hosts_and_bytes_moved(self): + progress = self._progress(["w1", "w2"]) + progress.sent("w1", 1024 * 1024, 1024 * 1024 * 2) + progress.done("w2") + line = progress.aggregate_line() + assert "1/2 host(s)" in line and "1.0 MB sent" in line + + def check_a_rsync_fraction_beats_bytes_over_total(self): + progress = self._progress(["w1"]) + progress.sent("w1", 10, 1000, fraction=0.9) + assert "90% 0/1 host(s)" in progress.aggregate_line(), \ + "rsync knows what it decided not to send; sent/total does not" + + def check_an_unknown_host_is_ignored(self): + progress = self._progress(["w1"]) + progress.sent("nobody", 10, 100) + progress.done("nobody") + assert "0% 0/1 host(s)" in progress.aggregate_line() + + +class CheckRows: + """One line per host, and never more lines than a terminal can hold.""" + + @staticmethod + def _progress(hosts, **kw): + return Progress(hosts, stream=_Stream(tty=True), live=True, **kw) + + def check_a_sending_host_shows_bytes_and_a_bar(self): + progress = self._progress(["w1"]) + progress.sent("w1", 1024 * 1024 * 30, 1024 * 1024 * 100) + row = progress.rows()[0] + assert "w1" in row and " 30%" in row and "30.0 MB / 100.0 MB" in row + + def check_phases_without_bytes_still_say_what_is_happening(self): + progress = self._progress(["w1"]) + progress.phase("w1", "swapping") + assert "swapping" in progress.rows()[0] + + def check_finished_hosts_make_room_for_active_ones(self): + progress = self._progress(["w1", "w2"]) + progress.done("w1") + progress.sent("w2", 1, 2) + rows = progress.rows() + assert len(rows) == 1 and "w2" in rows[0] + + def check_a_large_cluster_is_capped_and_says_so(self): + progress = self._progress(["w%02d" % i for i in range(30)], max_rows=5) + for index in range(30): + progress.sent("w%02d" % index, 1, 10) + rows = progress.rows() + assert len(rows) == 6 and "25 more host(s)" in rows[-1] + + def check_ascii_bars_when_the_stream_cannot_encode_blocks(self): + progress = Progress(["w1"], stream=_Stream(tty=True, encoding="cp1251"), live=True) + progress.sent("w1", 1, 2) + assert "#" in progress.rows()[0] and "█" not in progress.rows()[0] + + +class CheckRendering: + """What actually reaches the terminal.""" + + def check_live_mode_redraws_in_place(self): + stream = _Stream(tty=True) + progress = Progress(["w1"], stream=stream, live=True, interval=0.01) + progress.start() + progress.sent("w1", 5, 10) + progress.close() + written = stream.getvalue() + assert "\033[2K" in written and "\033[" in written + assert written.endswith("\033[?25h"), "the cursor must be handed back" + + def check_plain_mode_prints_whole_lines_only(self): + stream = _Stream(tty=False) + progress = Progress(["w1"], stream=stream, live=False, interval=0.01, + plain_interval=0.0) + progress.start() + progress.sent("w1", 5, 10) + progress.close() + written = stream.getvalue() + assert "\033[" not in written, "a log file must not collect cursor commands" + assert "total" in written + + def check_a_broken_stream_does_not_break_the_deploy(self): + stream = _Stream(tty=True) + progress = Progress(["w1"], stream=stream, live=True, interval=0.01) + progress.start() + stream.close() + progress.sent("w1", 5, 10) + progress.close() + + def check_secrets_are_redacted_on_the_way_out(self): + class _Redactor: # pylint: disable=too-few-public-methods + @staticmethod + def redact(text): + return text.replace("w1", "***") + + stream = _Stream(tty=False) + progress = Progress(["w1"], stream=stream, live=False, interval=0.01, + plain_interval=0.0, redactor=_Redactor()) + progress.start() + progress.sent("w1", 1, 2) + progress.close() + assert "w1" not in stream.getvalue() + + +class CheckNullProgress: + """The disabled display has to accept every call the live one does.""" + + def check_every_method_is_a_no_op(self): + with NullProgress() as progress: + progress.phase("w1", "sending") + progress.sent("w1", 1, 2, fraction=0.5) + progress.done("w1", "done") + assert NullProgress().live is False + + def check_a_non_terminal_is_never_live(self): + assert is_a_terminal(_Stream(tty=False)) is False + assert is_a_terminal(io.StringIO()) is False + + +class CheckWhenItIsUsed: + """The command decides; the display only draws.""" + + @staticmethod + def _ctx(*, dry_run=False, quiet=False, verbose=False, no_progress=False): + class _Console: # pylint: disable=too-few-public-methods + pass + + class _Args: # pylint: disable=too-few-public-methods + pass + + class _Ctx: # pylint: disable=too-few-public-methods + pass + + ctx = _Ctx() + ctx.console = _Console() + ctx.console.quiet = quiet + ctx.console.verbose = verbose + ctx.console.redactor = None + ctx.args = _Args() + ctx.args.no_progress = no_progress + ctx.dry_run = dry_run + return ctx + + @staticmethod + def _nodes(): + class _Node: # pylint: disable=too-few-public-methods + host = "w1" + + return [_Node()] + + def check_a_dry_run_reports_nothing(self): + assert isinstance(build_progress(self._ctx(dry_run=True), self._nodes()), + NullProgress) + + def check_quiet_and_no_progress_report_nothing(self): + assert isinstance(build_progress(self._ctx(quiet=True), self._nodes()), + NullProgress) + assert isinstance(build_progress(self._ctx(no_progress=True), self._nodes()), + NullProgress) + + def check_verbose_reports_without_redrawing(self, monkeypatch): + monkeypatch.setattr(progress_mod, "is_a_terminal", lambda _: True) + progress = build_progress(self._ctx(verbose=True), self._nodes()) + assert isinstance(progress, Progress) and progress.live is False, \ + "a redrawn block would fight with the traced command lines" + + def check_a_terminal_gets_the_live_display(self, monkeypatch): + monkeypatch.setattr(progress_mod, "is_a_terminal", lambda _: True) + assert build_progress(self._ctx(), self._nodes()).live is True + + +class CheckStreamedTransfers: + """The two ways bytes are counted while they move.""" + + def check_output_is_delivered_line_by_line_as_it_arrives(self): + lines = [] + script = ("import sys\n" + "sys.stdout.write(' 1,024 10% 1MB/s\\r')\n" + "sys.stdout.write(' 2,048 20% 1MB/s\\n')\n" + "sys.stdout.write('Number of regular files transferred: 2\\n')\n") + result = run_local_streaming([sys.executable, "-c", script], on_output=lines.append) + assert result.ok + assert deploy.parse_rsync_progress(lines[0]) == (1024, 0.1) + assert deploy.parse_rsync_progress(lines[1]) == (2048, 0.2) + assert deploy.parse_rsync_stats(result.stdout) is None or True + assert len(lines) == 3, "a carriage return ends a progress line just as a newline does" + + def check_a_failing_command_keeps_its_output_and_status(self): + result = run_local_streaming( + [sys.executable, "-c", "import sys; sys.stderr.write('boom'); sys.exit(3)"], + on_output=lambda _: None) + assert result.returncode == 3 and "boom" in result.stderr + + def check_progress_lines_that_are_not_progress_are_ignored(self): + assert deploy.parse_rsync_progress("sending incremental file list") is None + assert deploy.parse_rsync_progress("") is None + + def check_a_local_copy_reports_every_chunk(self, tmp_path): + source = tmp_path / "payload.tar.gz" + source.write_bytes(b"x" * (3 * 1024 * 1024 + 7)) + seen = [] + LocalTransport().upload_watched(source, str(tmp_path / "out.tar.gz"), + on_bytes=seen.append) + assert seen[-1] == source.stat().st_size + assert seen == sorted(seen) and len(seen) == 4 + assert (tmp_path / "out.tar.gz").read_bytes() == source.read_bytes() + + def check_an_unwatched_upload_is_the_plain_one(self, tmp_path): + source = tmp_path / "a" + source.write_text("body", encoding="utf-8") + LocalTransport().upload_watched(source, str(tmp_path / "b")) + assert (tmp_path / "b").read_text(encoding="utf-8") == "body" diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_runs.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_runs.py new file mode 100644 index 0000000000000..07cea8c26dce4 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_runs.py @@ -0,0 +1,195 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for run ids, run directory layout, state derivation, and run.sh rendering.""" + +from datetime import datetime + +from fake_transport import FakeTransport + +from ducktests_remote import runs + +GOLDEN_RUN_SH = """set -euo pipefail + +cd '/opt/my sources/ignite/modules/ducktests/tests' +# activate the runner venv; `set +u` because older activate +# scripts read unset variables +set +u +. '/opt/venvs/dt env/bin/activate' +set -u + +exec ducktape \\ + --results-root /state/runs/r/results \\ + --cluster-file /state/runs/r/cluster.json \\ + --globals /state/runs/r/globals.json \\ + --parameters /state/runs/r/parameters.json \\ + --repeat 3 \\ + --max-parallel 4 \\ + --test-runner-timeout 900000 \\ + --debug \\ + './tests/a b.py::Cls.test' './tests/{braces}.py' +""" + + +class CheckRunId: + """Format and uniqueness.""" + + def check_format(self): + run_id = runs.new_run_id("max", now=datetime(2026, 7, 27, 14, 12, 33), entropy="9f2a") + assert run_id == "max-20260727-141233-9f2a" + assert runs.is_run_id(run_id) + + def check_unsafe_characters_in_the_user_are_replaced(self): + run_id = runs.new_run_id("DOMAIN\\user", now=datetime(2026, 1, 1), entropy="0000") + assert runs.is_run_id(run_id) + + def check_uniqueness_within_the_same_second(self): + now = datetime(2026, 7, 27, 14, 12, 33) + ids = {runs.new_run_id("max", now=now) for _ in range(200)} + assert len(ids) > 190, "the hex suffix must keep same-second runs apart" + + def check_rejects_arbitrary_directory_names(self): + assert not runs.is_run_id("latest") + assert not runs.is_run_id("results") + + +class CheckPaths: + """Run directory layout.""" + + def check_layout(self): + paths = runs.RunPaths("/state", "max-20260727-141233-9f2a") + assert paths.run_dir == "/state/runs/max-20260727-141233-9f2a" + assert paths.globals_file.endswith("/globals.json") + assert paths.results_dir.endswith("/results") + assert paths.latest_link == "/state/runs/latest" + assert paths.src_dir == "/state/src/max-20260727-141233-9f2a" + + +class CheckStateDerivation: + """The three observable facts map onto one state.""" + + def check_running(self): + assert runs.derive_state(pid_alive=True, exit_code=None, stopped=False) == runs.RUNNING + + def check_finished(self): + assert runs.derive_state(pid_alive=False, exit_code=0, stopped=False) == runs.FINISHED + + def check_failed(self): + assert runs.derive_state(pid_alive=False, exit_code=1, stopped=False) == runs.FAILED + + def check_stopped(self): + assert runs.derive_state(pid_alive=False, exit_code=143, stopped=True) == runs.STOPPED + + def check_exit_code_beats_a_reused_pid(self): + assert runs.derive_state(pid_alive=True, exit_code=0, stopped=False) == runs.FINISHED + + def check_unknown(self): + assert runs.derive_state(pid_alive=False, exit_code=None, stopped=False) == runs.UNKNOWN + + def check_read_state_parses_the_probe_output(self): + transport = FakeTransport() + transport.when("bash", stdout=("pid=4242\npgid=4242\nalive=1\n" + "meta={\"test_paths\": [\"a.py\"]}\n")) + state = runs.read_state(transport, runs.RunPaths("/state", "r")) + assert state.pid == 4242 and state.state == runs.RUNNING + assert state.meta["test_paths"] == ["a.py"] + + +class CheckListing: + """Only real run directories are listed.""" + + def check_latest_symlink_is_not_a_run(self): + transport = FakeTransport() + transport.when("ls", stdout="latest\nmax-20260727-141233-9f2a\n" + "max-20260726-090000-aaaa\n") + ids = runs.list_run_ids(transport, "/state") + assert ids == ["max-20260727-141233-9f2a", "max-20260726-090000-aaaa"] + + +class CheckRunScript: + """Golden-file rendering, including paths with spaces and shell metacharacters.""" + + def _render(self): + return runs.render_run_script( + version="0.1.0", timestamp="2026-07-27T00:00:00+00:00", author="max@laptop", + cwd="/opt/my sources/ignite/modules/ducktests/tests", + results_root="/state/runs/r/results", + cluster_file="/state/runs/r/cluster.json", + globals_file="/state/runs/r/globals.json", + parameters_file="/state/runs/r/parameters.json", + test_paths=["./tests/a b.py::Cls.test", "./tests/{braces}.py"], + venv="/opt/venvs/dt env", + repeat=3, max_parallel=4, test_runner_timeout=900000, + extra_args=["--debug"]) + + def check_golden_body(self): + rendered = self._render() + body = rendered[rendered.index("set -euo pipefail"):] + assert body == GOLDEN_RUN_SH + + def check_header_records_provenance(self): + rendered = self._render() + assert "Generated by ducktests-remote 0.1.0" in rendered + assert "max@laptop" in rendered + + def check_globals_is_passed_as_a_file_path(self): + # ducktape 0.13 checks os.path.isfile before parsing --globals as JSON + # (command_line/main.py::get_user_defined_globals), so the blob never has to + # cross a shell command line. + rendered = self._render() + assert "--globals /state/runs/r/globals.json" in rendered + assert "{" not in rendered.split("exec ducktape")[1].replace("{braces}", "") + + def check_optional_flags_are_omitted_when_unset(self): + rendered = runs.render_run_script( + version="0.1.0", timestamp="t", author="a", cwd="/w", + results_root="/r", cluster_file="/c", globals_file="/g", + test_paths=["./t.py"], venv=None) + assert "--parameters" not in rendered + assert "--repeat" not in rendered + assert "--max-parallel" not in rendered + assert "no venv configured" in rendered + + def check_a_json_like_test_path_is_quoted(self): + rendered = runs.render_run_script( + version="0.1.0", timestamp="t", author="a", cwd="/w", + results_root="/r", cluster_file="/c", globals_file="/g", + test_paths=['./t.py::C.m@{"x": 1}'], venv=None) + assert """'./t.py::C.m@{"x": 1}'""" in rendered + + +class CheckLaunchScripts: + """Detachment mechanics.""" + + def check_launch_records_the_exit_code(self): + script = runs.render_launch_script(runs.RunPaths("/state", "r")) + assert 'echo $? > "$rd/exit_code"' in script + assert 'pgid' in script + + def check_detach_prefers_setsid_and_falls_back(self): + script = runs.render_detach_script(runs.RunPaths("/state", "r")) + assert "setsid nohup bash" in script + assert "disown" in script, "a runner without setsid must still detach" + assert 'echo $! > "$rd/pid"' in script + + +class CheckFormatting: + """Duration rendering.""" + + def check_durations(self): + assert runs.format_duration(None) == "-" + assert runs.format_duration(42) == "42s" + assert runs.format_duration(125) == "2m 05s" + assert runs.format_duration(3852) == "1h 04m 12s" diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_sources.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_sources.py new file mode 100644 index 0000000000000..7a046a4914564 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_sources.py @@ -0,0 +1,200 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for finding the Ignite checkout and for the test paths handed to ducktape.""" + +import os + +import pytest + +from fake_transport import FakeTransport + +from ducktests_remote.cli import Console, Context +from ducktests_remote.commands import run +from ducktests_remote.config import ConfigError, load_config + +TESTS = ("modules", "ducktests", "tests") + + +def _checkout(tmp_path, name="ignite"): + """Build the smallest tree that counts as an Ignite checkout.""" + root = tmp_path / name + tests = root.joinpath(*TESTS) + (tests / "docker").mkdir(parents=True, exist_ok=True) + (tests / "docker" / "requirements.txt").write_text("ducktape==0.13.0\n", encoding="utf-8") + suite = tests / "ignitetest" / "tests" + suite.mkdir(parents=True, exist_ok=True) + (suite / "smoke_test.py").write_text("# tests\n", encoding="utf-8") + return root.resolve() + + +def _context(**args): + class Args: # pylint: disable=too-few-public-methods + """Minimal stand-in for the parsed command line.""" + + parsed = Args() + parsed.dry_run = False + parsed.source_root = None + parsed.no_sync = False + parsed.work_dir = None + for key, value in args.items(): + setattr(parsed, key, value) + ctx = Context(load_config(user_config=None), parsed, Console(color=False)) + ctx._runner = FakeTransport() # noqa: SLF001 - the point of the fake + return ctx + + +@pytest.fixture(name="in_dir") +def _in_dir(): + """Run a check from a chosen directory and restore the old one afterwards.""" + previous = os.getcwd() + yield os.chdir + os.chdir(previous) + + +class CheckSourceRoot: + """Where the sources are taken from when nothing says so explicitly.""" + + def check_the_checkout_root_is_found_from_the_tests_directory(self, tmp_path, in_dir): + # The Docker flow is run from here, so this is where operators stand. + root = _checkout(tmp_path) + in_dir(str(root.joinpath(*TESTS))) + assert run._source_root(_context()) == root.resolve() # noqa: SLF001 + + def check_the_checkout_root_is_found_from_deeper_still(self, tmp_path, in_dir): + root = _checkout(tmp_path) + in_dir(str(root.joinpath(*TESTS) / "ignitetest" / "tests")) + assert run._source_root(_context()) == root.resolve() # noqa: SLF001 + + def check_the_root_itself_is_recognised(self, tmp_path, in_dir): + root = _checkout(tmp_path) + in_dir(str(root)) + assert run._source_root(_context()) == root.resolve() # noqa: SLF001 + + def check_an_explicit_root_is_taken_as_given(self, tmp_path, in_dir): + root = _checkout(tmp_path) + in_dir(str(tmp_path)) + ctx = _context(source_root=str(root)) + assert run._source_root(ctx) == root.resolve() # noqa: SLF001 + + def check_a_directory_outside_any_checkout_falls_back_to_itself(self, tmp_path, in_dir): + # Kept as the old behaviour so the failure comes from the validation below, with + # a message, rather than from an unrelated parent directory being picked up. + elsewhere = tmp_path / "state" + elsewhere.mkdir() + in_dir(str(elsewhere)) + assert run._source_root(_context()) == elsewhere.resolve() # noqa: SLF001 + + +class CheckSourceRootValidation: + """The mistake this exists for: syncing something that is not the checkout.""" + + def check_a_non_checkout_is_refused_before_anything_is_uploaded(self, tmp_path): + state = tmp_path / ".ducktests-remote" + state.mkdir() + with pytest.raises(ConfigError) as ex: + run._check_source_root(_context(), state) # noqa: SLF001 + message = str(ex.value) + assert str(state) in message, "the message names the directory that was wrong" + assert "modules/ducktests/tests/docker/requirements.txt" in message + assert "--source-root" in message, "and how to fix it" + + def check_a_real_checkout_passes(self, tmp_path): + run._check_source_root(_context(), _checkout(tmp_path)) # noqa: SLF001 + + def check_no_sync_against_an_unseen_tree_is_allowed(self, tmp_path): + # With --no-sync the path describes the runner, which the coordinator cannot check. + run._check_source_root(_context(no_sync=True), # noqa: SLF001 + tmp_path / "on-the-runner-only") + + def check_no_sync_still_refuses_a_local_directory_that_is_wrong(self, tmp_path): + state = tmp_path / ".ducktests-remote" + state.mkdir() + with pytest.raises(ConfigError): + run._check_source_root(_context(no_sync=True), state) # noqa: SLF001 + + +class CheckTestPaths: + """ducktape runs from the tests directory, so the paths it gets are relative to it.""" + + @staticmethod + def _paths(root, given, cwd=None): + os.chdir(str(cwd or root.joinpath(*TESTS))) + return run._test_paths(_context(), given, root, # noqa: SLF001 + "/state/runs/r/src") + + def check_a_docker_style_path_survives_unchanged(self, tmp_path, in_dir): + in_dir(str(tmp_path)) + assert self._paths(_checkout(tmp_path), ["./ignitetest/tests/smoke_test.py"]) == \ + ["./ignitetest/tests/smoke_test.py"] + + def check_the_class_and_method_suffix_is_preserved(self, tmp_path, in_dir): + in_dir(str(tmp_path)) + given = "./ignitetest/tests/smoke_test.py::SmokeServicesTest.test_ignite_start_stop" + assert self._paths(_checkout(tmp_path), [given]) == [given] + + def check_a_repository_relative_path_is_shortened(self, tmp_path, in_dir): + # What README used to require, typed from the checkout root. + in_dir(str(tmp_path)) + root = _checkout(tmp_path) + given = "./modules/ducktests/tests/ignitetest/tests/smoke_test.py" + assert self._paths(root, [given], cwd=root) == ["./ignitetest/tests/smoke_test.py"] + + def check_an_absolute_path_is_shortened_too(self, tmp_path, in_dir): + in_dir(str(tmp_path)) + root = _checkout(tmp_path) + absolute = str(root.joinpath(*TESTS) / "ignitetest" / "tests" / "smoke_test.py") + assert self._paths(root, [absolute]) == ["./ignitetest/tests/smoke_test.py"] + + def check_a_path_inside_the_checkout_but_outside_tests_becomes_absolute( + self, tmp_path, in_dir): + # Relative would be resolved against the tests directory and miss; the runner-side + # absolute path is the only form that cannot be misread. + in_dir(str(tmp_path)) + root = _checkout(tmp_path) + (root / "extra").mkdir(exist_ok=True) + (root / "extra" / "other_test.py").write_text("# t\n", encoding="utf-8") + assert self._paths(root, ["./extra/other_test.py"], cwd=root) == \ + ["/state/runs/r/src/extra/other_test.py"] + + def check_an_unresolvable_path_is_passed_through(self, tmp_path, in_dir): + # --no-sync against a checkout this machine does not have. + in_dir(str(tmp_path)) + assert self._paths(_checkout(tmp_path), ["./ignitetest/tests/absent.py"]) == \ + ["./ignitetest/tests/absent.py"] + + +class CheckSyncProgress: + """The 171 MB that goes to the runner before a run starts.""" + + class _Paths: # pylint: disable=too-few-public-methods + src_dir = "/state/runs/r/src" + + def _sync(self, tmp_path, **args): + root = _checkout(tmp_path) + args.setdefault("exclude", []) + ctx = _context(**args) + run._sync_sources(ctx, root, self._Paths()) # noqa: SLF001 + return ctx._runner # noqa: SLF001 + + def check_the_sync_is_watched(self, tmp_path): + runner = self._sync(tmp_path) + assert runner.uploads and runner.uploads[0][2] == "dir" + assert runner.reported, "a sync that takes minutes has to show movement" + + def check_no_progress_still_syncs(self, tmp_path): + runner = self._sync(tmp_path, no_progress=True) + assert runner.uploads, "the transfer happens either way" + assert not runner.reported, "--no-progress means nothing is asked to report" diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_sshdiag.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_sshdiag.py new file mode 100644 index 0000000000000..3d989b4497529 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_sshdiag.py @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Table-driven checks over recorded OpenSSH stderr samples.""" + +import pytest + +from ducktests_remote import sshdiag + +SAMPLES = [ + (sshdiag.UNRESOLVED, + "ssh: Could not resolve hostname node07.dc.local: Name or service not known"), + (sshdiag.UNRESOLVED, + "ssh: Could not resolve hostname host: nodename nor servname provided, or not known"), + (sshdiag.UNRESOLVED, + "ssh: Could not resolve hostname x: Temporary failure in name resolution"), + (sshdiag.NO_SSHD, + "ssh: connect to host node03.dc.local port 22: Connection refused"), + (sshdiag.UNREACHABLE, + "ssh: connect to host node04.dc.local port 22: Connection timed out"), + (sshdiag.UNREACHABLE, + "ssh: connect to host node05 port 22: No route to host"), + (sshdiag.UNREACHABLE, + "ssh: connect to host node06 port 22: Network is unreachable"), + (sshdiag.NO_ACCESS, + "max@node02.dc.local: Permission denied (publickey,gssapi-keyex,password)."), + (sshdiag.NO_ACCESS, + "Received disconnect from 10.0.0.5 port 22:2: Too many authentication failures"), + (sshdiag.NO_USER, + "Invalid user max from 10.0.0.9 port 51234"), + (sshdiag.NO_USER, + "Please login as the user \"ec2-user\" rather than the user \"root\"."), + (sshdiag.HOSTKEY, + "@@@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @@@"), + (sshdiag.HOSTKEY, + "Host key verification failed."), + (sshdiag.NO_SUDO, + "sudo: a password is required"), + (sshdiag.NO_SUDO, + "sudo: no tty present and no askpass program specified"), + (sshdiag.NO_SUDO, + "max is not in the sudoers file. This incident will be reported."), + (sshdiag.UNKNOWN, + "some entirely novel failure nobody has seen before"), +] + + +class CheckClassification: + """Every recorded sample maps to exactly one class.""" + + @pytest.mark.parametrize("expected,stderr", SAMPLES) + def check_sample(self, expected, stderr): + assert sshdiag.classify(255, stderr) == expected + + def check_success_is_ok_regardless_of_stderr(self): + assert sshdiag.classify(0, "Warning: Permanently added 'x' to known hosts.") \ + == sshdiag.OK + + def check_advice_is_host_and_user_specific(self): + diagnosis = sshdiag.SshDiagnosis("node02", sshdiag.NO_ACCESS, user="max", port=2222) + assert "max" in diagnosis.advice + + def check_unreachable_advice_names_the_port(self): + diagnosis = sshdiag.SshDiagnosis("node02", sshdiag.UNREACHABLE, user="max", port=2222) + assert "2222" in diagnosis.advice + + +class CheckMixedCluster: + """A partially working cluster is the normal case, not an edge case.""" + + def _diagnoses(self): + return ([sshdiag.SshDiagnosis("node0%d" % i, sshdiag.OK, user="max") + for i in range(1, 10)] + + [sshdiag.SshDiagnosis("node10", sshdiag.NO_ACCESS, user="max"), + sshdiag.SshDiagnosis("node11", sshdiag.NO_ACCESS, user="max"), + sshdiag.SshDiagnosis("node12", sshdiag.NO_USER, user="max")]) + + def check_summary_counts_every_class(self): + assert sshdiag.summarise(self._diagnoses()) == "9 ok, 2 no-access, 1 no-user" + + def check_admin_block_names_the_right_hosts_per_class(self): + block = sshdiag.admin_request_block(self._diagnoses(), user="max") + no_access = block.split("does not exist")[0] + assert "node10" in no_access and "node11" in no_access + assert "node12" not in no_access, "a no-user host must not be listed as no-access" + assert "node12" in block + no_user = block.split("does not exist")[1] + assert "node10" not in no_user.split("it does exist on")[0], \ + "a no-access host must not be listed as no-user" + + def check_admin_block_mentions_the_account_and_the_authorized_keys_line(self): + block = sshdiag.admin_request_block(self._diagnoses(), user="max") + assert "'max'" in block + assert "authorized_keys" in block + + def check_admin_block_lists_hosts_that_do_have_the_account(self): + block = sshdiag.admin_request_block(self._diagnoses(), user="max") + assert "it does exist on" in block + + def check_no_block_when_everything_works(self): + healthy = [sshdiag.SshDiagnosis("node01", sshdiag.OK, user="max")] + assert sshdiag.admin_request_block(healthy, user="max") == "" + + def check_sudo_block_names_the_tests_that_need_it(self): + block = sshdiag.admin_request_block( + [sshdiag.SshDiagnosis("node01", sshdiag.NO_SUDO, user="max")], user="max") + assert "discovery_test.py" in block and "cellular_affinity_test.py" in block + + def check_hostkey_block_never_removes_anything_itself(self): + block = sshdiag.admin_request_block( + [sshdiag.SshDiagnosis("node01", sshdiag.HOSTKEY, user="max")], user="max") + assert "ssh-keygen -R node01" in block + assert "verify before removing" in block diff --git a/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py b/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py new file mode 100644 index 0000000000000..88d8dcb6c3f69 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/check_remote_transport.py @@ -0,0 +1,297 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for the transport boundary, the import guard, and exit-code mapping.""" + +import os +import subprocess +import sys +import threading +from pathlib import Path + +import pytest +from fake_transport import FakeTransport + +from ducktests_remote import cli, runs +from ducktests_remote import transport as transport_mod +from ducktests_remote.transport import (LocalTransport, ProxiedTransport, Result, + SshTransport, TransportError, is_excluded) + +PACKAGE_ROOT = Path(__file__).resolve().parent.parent + + +class CheckImportGuard: + """``ducktests_remote`` must be usable on a coordinator with no ducktape at all.""" + + def check_ducktape_is_not_in_the_import_graph(self): + script = ( + "import sys\n" + "import ducktests_remote.cli as c\n" + "c.build_parser()\n" + "leaked = [m for m in sys.modules if m == 'ducktape' " + "or m.startswith('ducktape.')]\n" + "assert not leaked, leaked\n" + "print('clean')\n") + result = subprocess.run([sys.executable, "-c", script], capture_output=True, + text=True, check=False, cwd=str(PACKAGE_ROOT.parent)) + assert result.returncode == 0, result.stderr + assert "clean" in result.stdout + + def check_no_source_file_imports_ducktape(self): + offenders = [] + for path in PACKAGE_ROOT.rglob("*.py"): + if "checks" in path.parts: + continue + text = path.read_text(encoding="utf-8") + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith(("import ducktape", "from ducktape")): + offenders.append("%s: %s" % (path.name, stripped)) + assert not offenders, offenders + + +class CheckTransportEquivalence: + """Above the transport boundary, local and ssh runs are indistinguishable.""" + + def check_identical_command_sequences(self): + sequences = [] + for name in ("local", "build-vm-01"): + transport = FakeTransport(name=name) + transport.mkdirs("/state/runs/r") + transport.run(["mkdir", "-p", "/state/runs/r/results"]) + transport.write_file("{}", "/state/runs/r/globals.json", mode=0o600) + runs.read_state(transport, runs.RunPaths("/state", "r")) + sequences.append((transport.commands, sorted(transport.files))) + assert sequences[0] == sequences[1] + + +class CheckSshOptions: + """The ssh client is the system one, driven with deliberate options.""" + + def check_batch_mode_is_always_on(self): + # An interactive password prompt inside a fan-out across N hosts hangs the whole + # run, so a missing key is a diagnosable failure rather than a prompt. + opts = SshTransport(name="h").ssh_options() + assert "BatchMode=yes" in opts + + def check_identity_is_used_exclusively_when_given(self): + opts = SshTransport(name="h", identity_file="/k").ssh_options() + assert "IdentitiesOnly=yes" in opts and "/k" in opts + + def check_non_default_port_uses_the_right_flag(self): + assert "-p" in SshTransport(name="h", port=2222).ssh_options() + assert "-P" in SshTransport(name="h", port=2222).ssh_options(for_scp=True) + + def check_default_port_is_not_passed(self): + assert "-p" not in SshTransport(name="h", port=22).ssh_options() + + def check_target_includes_the_user(self): + assert SshTransport(name="h", user="max").target == "max@h" + assert SshTransport(name="h").target == "h" + + +class CheckProxiedTransport: + """``deploy --via`` and runner-side probing hop through another transport.""" + + def check_command_is_wrapped_in_ssh_on_the_intermediate_host(self): + via = FakeTransport(name="jump") + proxied = ProxiedTransport(name="node01", via=via, user="max") + proxied.run(["true"], check=False) + argv = via.commands[-1] + assert argv[0] == "ssh" and "max@node01" in argv + + +class CheckLocalTransport: + """The local transport really does run things.""" + + def check_run_captures_output(self): + transport = LocalTransport() + result = transport.run([sys.executable, "-c", "print('hi')"]) + assert result.out == "hi" + + def check_failure_raises_with_the_command_in_the_message(self): + transport = LocalTransport() + with pytest.raises(TransportError) as ex: + transport.run([sys.executable, "-c", "import sys;sys.exit(3)"]) + assert "exit 3" in str(ex.value) + + def check_missing_binary_is_a_transport_error(self): + with pytest.raises(TransportError): + LocalTransport().run(["definitely-not-a-real-binary-xyz"]) + + def check_dry_run_executes_nothing(self): + printed = [] + transport = LocalTransport(dry_run=True, printer=printed.append) + result = transport.run([sys.executable, "-c", "raise SystemExit(9)"]) + assert result.ok and printed and printed[0].startswith("[dry-run]") + + def check_tilde_expansion_uses_the_remote_home(self): + transport = FakeTransport(home="/home/tester") + assert transport.expand("~/.ducktests-remote") == "/home/tester/.ducktests-remote" + assert transport.expand("/absolute") == "/absolute" + + +class CheckExcludes: + """Sync exclusions behave like rsync patterns.""" + + @pytest.mark.parametrize("path", [".git/config", "target/classes/A.class", + "mod/target/x.jar", "a/b/__pycache__/c.pyc", + "x/y.pyc", "ignitetest.egg-info/PKG-INFO"]) + def check_excluded(self, path): + assert is_excluded(path, [".git", "target", "__pycache__", "*.pyc", "*.egg-info"]) + + @pytest.mark.parametrize("path", ["ignitetest/tests/smoke_test.py", "README.md", + "targeting/notes.txt"]) + def check_kept(self, path): + assert not is_excluded(path, [".git", "target", "__pycache__", "*.pyc"]) + + +class CheckExitCodes: + """Jenkins needs 'tests failed' to be distinguishable from 'the cluster is broken'.""" + + def check_values(self): + assert (cli.EXIT_OK, cli.EXIT_USAGE, cli.EXIT_PREFLIGHT, cli.EXIT_BUSY, + cli.EXIT_TESTS_FAILED, cli.EXIT_TRANSPORT, cli.EXIT_INTERRUPTED) \ + == (0, 1, 2, 3, 4, 5, 130) + + def check_tests_failed_is_not_a_transport_error(self): + assert cli.EXIT_TESTS_FAILED != cli.EXIT_TRANSPORT + + def check_config_error_maps_to_usage(self): + assert cli.main(["--config", "/nope/missing.yaml", "doctor"]) == cli.EXIT_USAGE + + def check_no_command_prints_help_and_reports_usage(self, capsys): + assert cli.main([]) == cli.EXIT_USAGE + assert "ducktests-remote" in capsys.readouterr().out + + def check_passthrough_split(self): + head, tail = cli.split_passthrough(["run", "-t", "a.py", "--", "--debug", "--sample", "3"]) + assert head == ["run", "-t", "a.py"] + assert tail == ["--debug", "--sample", "3"] + + def check_passthrough_absent(self): + assert cli.split_passthrough(["status"]) == (["status"], []) + + +class CheckResult: + """Result helpers.""" + + def check_ok_and_out(self): + result = Result(["true"], 0, " value \n") + assert result.ok and result.out == "value" + + +class CheckWatchedTransfers: + """A transfer only pays for progress when something is drawing it.""" + + @staticmethod + def _rsync_transport(monkeypatch, recorded): + transport = SshTransport(name="w1", user="tester") + monkeypatch.setattr(transport, "has_rsync", lambda: True) + monkeypatch.setattr(transport, "mkdirs", lambda *a, **kw: None) + + def record(argv, **kw): + recorded.append(list(argv)) + return Result(list(argv), 0, "", "", "w1") + + monkeypatch.setattr(transport_mod, "_spawn", record) + monkeypatch.setattr(transport_mod, "run_local_streaming", record) + return transport + + def check_an_unwatched_sync_keeps_the_quiet_rsync(self, tmp_path, monkeypatch): + recorded = [] + self._rsync_transport(monkeypatch, recorded).upload_dir(tmp_path, "/remote") + assert "--info=progress2" not in recorded[0] + + def check_a_watched_sync_asks_rsync_for_its_meter(self, tmp_path, monkeypatch): + recorded = [] + transport = self._rsync_transport(monkeypatch, recorded) + transport.upload_dir(tmp_path, "/remote", on_progress=lambda sent, fraction: None) + assert "--info=progress2" in recorded[0] + + def check_the_meter_is_turned_into_calls(self): + seen = [] + watch = transport_mod.rsync_reporter(lambda sent, fraction: seen.append((sent, fraction))) + watch(" 1,048,576 25% 12.34MB/s 0:00:12") + watch("sending incremental file list") + assert seen == [(1048576, 0.25)], "only meter lines count" + + +class CheckStreamedUpload: + """ + A watched upload really runs a child process, so these checks really run one too. + + The path they cover broke in a way no mock could have shown: the payload reached the + far end intact and the transfer was then reported as failed, because ``communicate`` + flushed a stdin handle the caller had already closed. Nothing short of a live + subprocess sees that, and it is worth the second these take. + """ + + @staticmethod + def _child(body, *args): + return [sys.executable, "-c", body] + list(args) + + def check_the_whole_payload_arrives_and_every_byte_is_reported(self, tmp_path): + source = tmp_path / "payload.bin" + source.write_bytes(os.urandom(3 * 1024 * 1024 + 17)) + landed = tmp_path / "landed.bin" + seen = [] + argv = self._child("import sys\n" + "open(sys.argv[1], 'wb').write(sys.stdin.buffer.read())\n", + str(landed)) + + result = transport_mod._stream_to_stdin(argv, source, seen.append, host="w1") + + assert result.ok + assert landed.read_bytes() == source.read_bytes() + assert seen[-1] == source.stat().st_size, "the last report is the whole file" + + def check_a_chatty_command_does_not_wedge_the_transfer(self, tmp_path): + # ssh talks on the way in - a login banner, a host-key notice - and its output + # pipe holds 64 KB. With nobody draining it the child blocks on its own write + # while we block on ours, and the transfer stops for good at some arbitrary + # percentage. Run it on a thread so a regression fails the check instead of + # hanging the suite. + source = tmp_path / "payload.bin" + source.write_bytes(b"x" * (2 * 1024 * 1024)) + landed = tmp_path / "landed.bin" + argv = self._child("import sys\n" + "sys.stderr.write('noise\\n' * 40000)\n" + "sys.stderr.flush()\n" + "open(sys.argv[1], 'wb').write(sys.stdin.buffer.read())\n", + str(landed)) + + done = [] + worker = threading.Thread( + target=lambda: done.append( + transport_mod._stream_to_stdin(argv, source, lambda sent: None, host="w1")), + daemon=True) + worker.start() + worker.join(timeout=60) + + assert not worker.is_alive(), "the transfer deadlocked on an undrained output pipe" + assert done[0].ok and landed.stat().st_size == source.stat().st_size + + def check_a_failing_command_reports_its_stderr(self, tmp_path): + source = tmp_path / "payload.bin" + source.write_bytes(b"payload") + argv = self._child("import sys\n" + "sys.stderr.write('no such file or directory\\n')\n" + "sys.exit(1)\n") + + with pytest.raises(TransportError) as raised: + transport_mod._stream_to_stdin(argv, source, lambda sent: None, host="w1") + + assert "no such file or directory" in str(raised.value) diff --git a/modules/ducktests/tests/ducktests_remote/checks/fake_transport.py b/modules/ducktests/tests/ducktests_remote/checks/fake_transport.py new file mode 100644 index 0000000000000..524d388a5c36e --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/checks/fake_transport.py @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A recording Transport used by the unit checks. No processes, no network.""" + +import posixpath + +from ducktests_remote.transport import Result, Transport + + +class FakeTransport(Transport): + """ + Records every command and simulates a small filesystem. + + Responses are registered as ``(predicate, Result)`` pairs; anything unmatched + succeeds with empty output, which keeps the checks focused on what they assert. + """ + + def __init__(self, name="fake", home="/home/tester", **kw): + super().__init__(name=name, **kw) + self.commands = [] + self.scripts = [] + self.uploads = [] + self.downloads = [] + self.dirs = [] + self.reported = [] + self.files = {} + self.responses = [] + self._home = home + + # -- registration ------------------------------------------------------------ + + def when(self, needle, stdout="", returncode=0, stderr=""): + """Reply to any command whose joined argv contains ``needle``.""" + self.responses.append((needle, Result([], returncode, stdout, stderr, self.name))) + return self + + # -- Transport --------------------------------------------------------------- + + def run(self, argv, *, check=True, timeout=None, input=None): # noqa: A002 + argv = [str(a) for a in argv] + self.commands.append(argv) + if input is not None: + self.scripts.append(input if isinstance(input, str) else input.decode("utf-8")) + joined = " ".join(argv) + for needle, canned in self.responses: + if needle in joined or (input and needle in str(input)): + result = Result(argv, canned.returncode, canned.stdout, canned.stderr, self.name) + return result.check() if check and canned.returncode else result + return Result(argv, 0, "", "", self.name) + + def upload(self, local_path, remote_path, *, mode=None): + self.uploads.append((str(local_path), remote_path, mode)) + + def download(self, remote_path, local_path): + self.downloads.append((remote_path, str(local_path))) + + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False, + on_progress=None): + self.uploads.append((str(local_dir), remote_dir, "dir")) + if on_progress is not None: + # One synthetic report, so a caller's wiring is exercised rather than assumed. + self.reported.append((1024, 1.0)) + on_progress(1024, 1.0) + + def upload_watched(self, local_path, remote_path, *, mode=None, on_bytes=None): + self.uploads.append((str(local_path), remote_path, mode)) + if on_bytes is not None: + self.reported.append((1024, None)) + on_bytes(1024) + + def write_file(self, content, remote_path, *, mode=None): + self.files[remote_path] = (content, mode) + + def exists(self, remote_path): + return remote_path in self.files + + def mkdirs(self, remote_path, *, mode=None): + self.dirs.append(remote_path) + + def home(self): + return self._home + + def expand(self, path): + text = str(path) + if text.startswith("~/"): + return posixpath.join(self._home, text[2:]) + return text diff --git a/modules/ducktests/tests/ducktests_remote/cli.py b/modules/ducktests/tests/ducktests_remote/cli.py new file mode 100644 index 0000000000000..57443e1ce70e4 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/cli.py @@ -0,0 +1,297 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Argument parsing, the shared command context, and exit-code mapping.""" + +import argparse +import os +import sys + +from ducktests_remote import __version__ +from ducktests_remote.cluster import load_extra_hosts, load_nodes, select_nodes +from ducktests_remote.config import ConfigError, expand_path, load_config, set_dotted +from ducktests_remote.globals_builder import Redactor +from ducktests_remote.transport import TransportError, build_transport + +EXIT_OK = 0 +EXIT_USAGE = 1 +EXIT_PREFLIGHT = 2 +EXIT_BUSY = 3 # reserved: this deployment has a single runner and no leases +EXIT_TESTS_FAILED = 4 +EXIT_TRANSPORT = 5 +EXIT_INTERRUPTED = 130 + +_COLORS = {"red": "\033[31m", "green": "\033[32m", "yellow": "\033[33m", + "blue": "\033[34m", "bold": "\033[1m", "dim": "\033[2m"} +_RESET = "\033[0m" + + +class Console: + """Everything the CLI prints goes through here, so redaction cannot be bypassed.""" + + def __init__(self, *, verbose=False, quiet=False, color=True, redactor=None): + self.verbose = verbose + self.quiet = quiet + self.color = color and sys.stdout.isatty() and os.environ.get("TERM") != "dumb" + self.redactor = redactor or Redactor() + + def paint(self, text, style): + """:return: ``text`` wrapped in an ANSI style when colour is enabled.""" + if not self.color or style not in _COLORS: + return text + return "%s%s%s" % (_COLORS[style], text, _RESET) + + def out(self, message=""): + """Print a line of normal output.""" + print(self.redactor.redact(message)) + + def info(self, message): + """Print a line suppressed by ``--quiet``.""" + if not self.quiet: + self.out(message) + + def detail(self, message): + """Print a line shown only with ``--verbose``.""" + if self.verbose and not self.quiet: + self.out(self.paint(message, "dim")) + + def warn(self, message): + """Print a warning to stderr.""" + print(self.redactor.redact(self.paint("WARN " + message, "yellow")), file=sys.stderr) + + def error(self, message): + """Print an error to stderr.""" + print(self.redactor.redact(self.paint("ERROR " + message, "red")), file=sys.stderr) + + def heading(self, message): + """Print a section heading.""" + if not self.quiet: + self.out("") + self.out(self.paint(message, "bold")) + + +class Context: # pylint: disable=too-many-instance-attributes + """Config, transports and console, shared by every command.""" + + def __init__(self, config, args, console): + self.config = config + self.args = args + self.console = console + self.dry_run = bool(getattr(args, "dry_run", False)) + self.jobs = int(config.get("jobs", 16)) + self._runner = None + self._workers = {} + + # -- inventory --------------------------------------------------------------- + + @property + def cluster_cfg(self): + """:return: the ``cluster`` config section.""" + return self.config["cluster"] + + @property + def nodes(self): + """:return: worker nodes, truncated by ``--num-nodes`` when given.""" + nodes = load_nodes(self.cluster_cfg) + return select_nodes(nodes, getattr(self.args, "num_nodes", None)) + + @property + def all_nodes(self): + """:return: worker nodes plus ``extra_hosts`` (provision/deploy/clean/doctor).""" + return self.nodes + load_extra_hosts(self.cluster_cfg) + + @property + def runner_host(self): + """:return: the host ducktape will run on.""" + return self.cluster_cfg.get("runner") or "local" + + @property + def state_root(self): + """:return: the runner-side state root, ``~`` unexpanded.""" + return self.cluster_cfg.get("state_root") or "~/.ducktests-remote" + + @property + def identity_file(self): + """:return: the ssh identity path *as the runner sees it*.""" + return self.cluster_cfg.get("identity_file") + + # -- transports -------------------------------------------------------------- + + @property + def runner(self): + """:return: a transport to the runner, created once.""" + if self._runner is None: + self._runner = build_transport( + self.runner_host, + user=self.cluster_cfg.get("user") if self.runner_host != "local" else None, + port=self.cluster_cfg.get("port", 22), + identity_file=self._coordinator_identity(), + connect_timeout=self.config["ssh"]["connect_timeout"], + dry_run=self.dry_run, verbose=self.console.verbose, + printer=self.console.out) + return self._runner + + def worker(self, node): + """:return: a transport to one worker, created once per host.""" + if node.host not in self._workers: + self._workers[node.host] = build_transport( + node.host, user=node.user, port=node.port, + identity_file=self._coordinator_identity(node.identity_file), + connect_timeout=self.config["ssh"]["connect_timeout"], + dry_run=self.dry_run, verbose=self.console.verbose, + printer=self.console.out) + return self._workers[node.host] + + def _coordinator_identity(self, identity=None): + """ + The configured ``identity_file`` is a runner-side path; it may or may not name a + file that exists here. Use it when it does, otherwise let the system ssh client + fall back to ``~/.ssh/config`` and the agent. + """ + candidate = expand_path(identity or self.identity_file) + if candidate and os.path.isfile(candidate): + return candidate + return None + + def state_root_resolved(self): + """:return: the state root with ``~`` expanded against the runner's home.""" + return self.runner.expand(self.state_root) + + +def _common_parser(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--config", action="append", default=argparse.SUPPRESS, metavar="FILE", + help="configuration file (YAML or JSON); repeatable, applied in order") + parser.add_argument("--profile", action="append", default=argparse.SUPPRESS, metavar="NAME", + help="named profile from profiles_dir; repeatable, applied in order") + parser.add_argument("--runner", default=argparse.SUPPRESS, metavar="HOST", + help="host running ducktape, or 'local'; overrides the config") + parser.add_argument("--jobs", type=int, default=argparse.SUPPRESS, metavar="N", + help="fan-out parallelism (default 16)") + parser.add_argument("-v", "--verbose", action="store_true", default=argparse.SUPPRESS, + help="show every command and per-host output") + parser.add_argument("-q", "--quiet", action="store_true", default=argparse.SUPPRESS, + help="only print results and errors") + parser.add_argument("--dry-run", action="store_true", default=argparse.SUPPRESS, + help="print every command and generated file, execute nothing") + parser.add_argument("--no-color", action="store_true", default=argparse.SUPPRESS, + help="disable ANSI colour") + parser.add_argument("--fail-fast", action="store_true", default=argparse.SUPPRESS, + help="abort a fan-out after the first host failure") + parser.add_argument("--no-progress", action="store_true", default=argparse.SUPPRESS, + help="do not report per-host progress during long transfers") + return parser + + +def build_parser(): + """:return: the fully wired argument parser.""" + common = _common_parser() + parser = argparse.ArgumentParser( + prog="ducktests-remote", parents=[common], + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Run Apache Ignite ducktests against a real VM cluster.", + epilog="Start with `ducktests-remote doctor` and `--dry-run`.") + parser.add_argument("--version", action="version", version="ducktests-remote " + __version__) + subparsers = parser.add_subparsers(dest="command", metavar="") + + # Imported here so `ducktests_remote.cli` stays importable without the command + # modules having been loaded, which keeps the import-guard test cheap. + from ducktests_remote.commands import ( # pylint: disable=import-outside-toplevel + clean, deploy, doctor, fetch, keys, logs, provision, run, status, stop) + + for module in (run, status, logs, fetch, stop, provision, deploy, clean, doctor, keys): + module.register(subparsers, common) + + return parser + + +def _flag_overrides(args): + """:return: a config overlay built from the explicit command-line flags.""" + overlay = {} + if getattr(args, "runner", None): + set_dotted(overlay, "cluster.runner", args.runner) + if getattr(args, "jobs", None): + set_dotted(overlay, "jobs", int(args.jobs)) + for flag, dotted in (("install_root", "cluster.install_root"), + ("state_root", "cluster.state_root"), + ("dist_dir", "deploy.dist_dir"), + ("source_root", "run.source_root"), + ("work_dir", "run.work_dir"), + # Repeatable pip flags replace the configured list rather than + # extending it, matching how deep_merge treats every other list. + ("pip_index_url", "pip.index_url"), + ("pip_extra_index_url", "pip.extra_index_url"), + ("pip_trusted_host", "pip.trusted_host"), + ("pip_timeout", "pip.timeout"), + ("pip_cert", "pip.cert"), + ("java_home", "java.home"), + ("java_major", "java.major"), + ("java_archive", "java.archive")): + value = getattr(args, flag, None) + if value: + set_dotted(overlay, dotted, value) + return overlay + + +def split_passthrough(argv): + """ + Split ``-- `` off the end of the command line. + + argparse cannot express "everything after the first bare ``--`` belongs to another + program" without swallowing legitimate arguments, so it is done up front. + """ + if "--" not in argv: + return argv, [] + index = argv.index("--") + return argv[:index], argv[index + 1:] + + +def main(argv=None): + """CLI entry point. :return: the process exit code.""" + argv = list(sys.argv[1:] if argv is None else argv) + argv, passthrough = split_passthrough(argv) + + parser = build_parser() + args = parser.parse_args(argv) + args.passthrough = passthrough + + if not getattr(args, "command", None): + parser.print_help() + return EXIT_USAGE + + console = Console(verbose=getattr(args, "verbose", False), + quiet=getattr(args, "quiet", False), + color=not getattr(args, "no_color", False)) + try: + config = load_config(config_files=getattr(args, "config", None) or [], + profiles=getattr(args, "profile", None) or [], + overrides=_flag_overrides(args), + redactor=console.redactor) + ctx = Context(config, args, console) + return args.handler(ctx) + except ConfigError as ex: + console.error(str(ex)) + return EXIT_USAGE + except TransportError as ex: + console.error(str(ex)) + if console.verbose: + raise + return EXIT_TRANSPORT + except KeyboardInterrupt: + console.out("") + console.error("interrupted") + return EXIT_INTERRUPTED + except BrokenPipeError: + return EXIT_OK diff --git a/modules/ducktests/tests/ducktests_remote/cluster.py b/modules/ducktests/tests/ducktests_remote/cluster.py new file mode 100644 index 0000000000000..b5d56ab22b83e --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/cluster.py @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Cluster inventory: from the YAML ``cluster.nodes`` list to ducktape's ``cluster.json``. + +Nothing here assumes a cluster size. Two nodes and forty-nine nodes go down the same +code path and produce the same output shape. +""" + +import json +import re +from dataclasses import dataclass +from typing import List, Optional + +from ducktests_remote.config import ConfigError + +_RANGE = re.compile(r"\[(\d+)-(\d+)\]") + + +@dataclass(frozen=True) +class Node: + """One worker host as ducktape will see it.""" + + host: str + ip: Optional[str] = None + user: Optional[str] = None + port: int = 22 + identity_file: Optional[str] = None + + @property + def target(self): + """:return: ``user@host`` for ssh, or the bare host when no user is set.""" + return "%s@%s" % (self.user, self.host) if self.user else self.host + + @property + def externally_routable_ip(self): + """:return: the ip when the inventory gave one, else the hostname.""" + return self.ip or self.host + + def to_ssh_config(self): + """:return: the ``ssh_config`` block ducktape's RemoteAccountSSHConfig accepts.""" + return { + "host": self.host, + "hostname": self.host, + "user": self.user, + "port": int(self.port), + "identityfile": self.identity_file, + "password": "", + } + + +def expand_hosts(pattern): + """ + Expand ``node[01-12].dc.local`` into twelve hostnames. + + Zero padding follows the width of the lower bound, so ``[01-12]`` yields ``node01`` + through ``node12`` and ``[1-12]`` yields ``node1`` through ``node12``. Several + ranges in one pattern expand as a cartesian product, left to right. + """ + match = _RANGE.search(pattern) + if not match: + return [pattern] + start_raw, end_raw = match.group(1), match.group(2) + start, end = int(start_raw), int(end_raw) + if end < start: + raise ConfigError("invalid host range %r: %d > %d" % (pattern, start, end)) + width = len(start_raw) + expanded = [] + for value in range(start, end + 1): + replaced = pattern[:match.start()] + str(value).zfill(width) + pattern[match.end():] + expanded.extend(expand_hosts(replaced)) + return expanded + + +def load_nodes(cluster_cfg) -> List[Node]: + """ + Build the node list from the ``cluster`` config section. + + Entries may be a bare string (``10.0.0.13`` or ``node[01-12].dc.local``) or a mapping + with ``host`` and optional ``ip`` / ``user`` / ``port`` / ``identity_file``. Per-host + overrides exist because mixed clusters, where an account exists on some machines and + not others, are the normal case rather than the exception. + """ + default_user = cluster_cfg.get("user") + default_port = cluster_cfg.get("port", 22) + default_identity = cluster_cfg.get("identity_file") + + nodes = [] + for index, entry in enumerate(cluster_cfg.get("nodes") or []): + if isinstance(entry, str): + entry = {"host": entry} + if not isinstance(entry, dict): + raise ConfigError("cluster.nodes[%d]: expected a string or a mapping, found %s" + % (index, type(entry).__name__)) + host = entry.get("host") + if not host: + raise ConfigError("cluster.nodes[%d]: missing 'host'" % index) + unknown = set(entry) - {"host", "ip", "user", "port", "identity_file"} + if unknown: + raise ConfigError("cluster.nodes[%d]: unknown keys %s" + % (index, ", ".join(sorted(unknown)))) + expanded = expand_hosts(host) + if len(expanded) > 1 and entry.get("ip"): + raise ConfigError("cluster.nodes[%d]: 'ip' cannot be combined with a host range" + % index) + for name in expanded: + nodes.append(Node(host=name, + ip=entry.get("ip"), + user=entry.get("user", default_user), + port=int(entry.get("port", default_port)), + identity_file=entry.get("identity_file", default_identity))) + + seen = set() + for node in nodes: + if node.host in seen: + raise ConfigError("cluster.nodes: duplicate host %r" % node.host) + seen.add(node.host) + return nodes + + +def load_extra_hosts(cluster_cfg) -> List[Node]: + """:return: hosts targeted by provision/deploy/clean/doctor but kept out of cluster.json.""" + sub = dict(cluster_cfg) + sub["nodes"] = cluster_cfg.get("extra_hosts") or [] + return load_nodes(sub) + + +def select_nodes(nodes, num_nodes=None): + """ + Take the first ``num_nodes`` inventory entries. + + This is the analogue of ``IGNITE_NUM_CONTAINERS`` in ``docker/run_tests.sh``. + """ + if num_nodes is None: + return list(nodes) + if num_nodes < 1: + raise ConfigError("--num-nodes must be at least 1") + if num_nodes > len(nodes): + raise ConfigError( + "--num-nodes %d exceeds the inventory, which lists %d host%s. " + "Add hosts to cluster.nodes or lower --num-nodes." + % (num_nodes, len(nodes), "" if len(nodes) == 1 else "s")) + return list(nodes[:num_nodes]) + + +def cluster_json(nodes, identity_file=None): + """ + :param nodes: the selected :class:`Node` list. + :param identity_file: runner-side fallback identity for nodes without their own. + :return: the ducktape cluster file as a dict. + + The schema is the one ``ducktape.cluster.json.JsonCluster`` reads: a ``nodes`` list of + ``{externally_routable_ip, ssh_config}``, where ``ssh_config`` is passed straight into + ``RemoteAccountSSHConfig(host, hostname, user, port, password, identityfile)``. + """ + if not nodes: + raise ConfigError("cluster.nodes is empty; there is nothing for ducktape to run on") + payload = {"nodes": []} + for node in nodes: + ssh_config = node.to_ssh_config() + if not ssh_config.get("identityfile"): + ssh_config["identityfile"] = identity_file + payload["nodes"].append({ + "externally_routable_ip": node.externally_routable_ip, + "ssh_config": ssh_config, + }) + return payload + + +def dumps(payload): + """:return: the cluster file rendered as JSON text.""" + return json.dumps(payload, indent=2, sort_keys=True) + "\n" diff --git a/modules/ducktests/tests/ducktests_remote/commands/__init__.py b/modules/ducktests/tests/ducktests_remote/commands/__init__.py new file mode 100644 index 0000000000000..ddddae46b451c --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI subcommands. Each module exposes ``register(subparsers, common)``.""" diff --git a/modules/ducktests/tests/ducktests_remote/commands/clean.py b/modules/ducktests/tests/ducktests_remote/commands/clean.py new file mode 100644 index 0000000000000..a215c37d4d959 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/clean.py @@ -0,0 +1,138 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``clean`` - kill leftover Ignite JVMs and remove work directories. + +Stale JVMs from an aborted run are the single most common source of baffling failures on +a shared VM cluster, and the second most common is a work directory that still holds a +previous run's persistence. Both are removed here, and only here. +""" + +import posixpath +import shlex + +from ducktests_remote.cli import EXIT_OK, EXIT_TRANSPORT +from ducktests_remote.config import IGNITE_MAIN_CLASSES, ConfigError +from ducktests_remote.fanout import (CHANGED, FAILED, HostResult, OK, any_failed, fanout, + render_table, summarise) + + +def register(subparsers, common): + """Wire up the ``clean`` subcommand.""" + parser = subparsers.add_parser( + "clean", parents=[common], help="kill stale Ignite processes and remove work dirs", + description="Fan out across the workers, terminate anything matching " + "clean.process_pattern, and remove clean.paths. Run it with --dry-run " + "first: it prints exactly what it would kill and delete.") + parser.add_argument("-n", "--num-nodes", type=int, default=None, + help="only clean the first N inventory hosts") + parser.add_argument("--keep-paths", action="store_true", + help="kill processes but leave the work directories in place") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Clean the workers. :return: the process exit code.""" + nodes = ctx.all_nodes + if not nodes: + raise ConfigError("cluster.nodes is empty; there is nothing to clean") + results = clean_hosts(ctx, nodes, dry_run=ctx.dry_run, + remove_paths=not ctx.args.keep_paths) + return EXIT_TRANSPORT if any_failed(results) else EXIT_OK + + +def clean_hosts(ctx, nodes, *, dry_run=False, remove_paths=True): + """Run the clean fan-out. :return: the per-host results.""" + pattern = ctx.config["clean"]["process_pattern"] + paths = validated_paths(ctx.config["clean"]) if remove_paths else [] + script = _script(pattern, paths, dry_run=dry_run) + + ctx.console.info("pattern: %s" % pattern) + ctx.console.info("paths : %s" % (", ".join(paths) if paths else "(none)")) + if dry_run: + ctx.console.info("--dry-run: nothing will be killed or removed") + + def operation(node): + result = ctx.worker(node).run_script(script, check=False) + if not result.ok: + return HostResult(node.host, FAILED, "clean failed", detail=result.stderr.strip()) + lines = [ln for ln in result.stdout.splitlines() if ln.strip()] + killed = [ln for ln in lines if ln.startswith("proc ")] + removed = [ln for ln in lines if ln.startswith("path ")] + status = CHANGED if (killed or removed) else OK + message = "%d process(es), %d path(s)" % (len(killed), len(removed)) + return HostResult(node.host, status, message, detail="\n".join(lines)) + + results = fanout(nodes, operation, jobs=ctx.jobs, + fail_fast=getattr(ctx.args, "fail_fast", False)) + ctx.console.out(render_table(results, verbose=ctx.console.verbose)) + ctx.console.out("") + ctx.console.out(summarise(results)) + return results + + +def validated_paths(clean_cfg): + """ + Check every configured path against the allow-list before it is ever sent to a host. + + A bug here deletes distributions across every machine in the cluster at once, so the + rule is deliberately blunt: a path must sit under one of ``clean.allowed_roots``, and + the roots themselves are not removable. + """ + allowed = [posixpath.normpath(p) for p in clean_cfg.get("allowed_roots") or []] + checked = [] + for raw in clean_cfg.get("paths") or []: + path = posixpath.normpath(str(raw)) + if not path.startswith("/"): + raise ConfigError("clean.paths: %r must be absolute" % raw) + if path in ("/", "") or path in allowed: + raise ConfigError("clean.paths: refusing to remove %r" % raw) + if not any(path == root or path.startswith(root.rstrip("/") + "/") for root in allowed): + raise ConfigError( + "clean.paths: %r is outside clean.allowed_roots (%s). Add the root " + "explicitly if you really mean it." % (raw, ", ".join(allowed))) + checked.append(path) + return checked + + +def _script(pattern, paths, *, dry_run): + quoted_paths = " ".join(shlex.quote(p) for p in paths) + return """set -u +pattern=%(pattern)s +dry=%(dry)d +pids=$(pgrep -f "$pattern" 2>/dev/null || true) +for p in $pids; do + cmd=$(ps -o args= -p "$p" 2>/dev/null | cut -c1-100) + echo "proc $p $cmd" +done +if [ "$dry" -eq 0 ] && [ -n "$pids" ]; then + kill -TERM $pids 2>/dev/null || true + sleep 5 + left=$(pgrep -f "$pattern" 2>/dev/null || true) + [ -n "$left" ] && kill -KILL $left 2>/dev/null || true +fi +for d in %(paths)s; do + [ -e "$d" ] || continue + echo "path $d ($(du -sh "$d" 2>/dev/null | cut -f1))" + if [ "$dry" -eq 0 ]; then rm -rf -- "$d"; fi +done +exit 0 +""" % {"pattern": shlex.quote(pattern), "dry": 1 if dry_run else 0, "paths": quoted_paths} + + +def known_main_classes(): + """:return: the Ignite main classes ignitetest launches, for documentation and help.""" + return IGNITE_MAIN_CLASSES diff --git a/modules/ducktests/tests/ducktests_remote/commands/deploy.py b/modules/ducktests/tests/ducktests_remote/commands/deploy.py new file mode 100644 index 0000000000000..20ad5c135c815 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/deploy.py @@ -0,0 +1,624 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``deploy`` - push distributions from a coordinator-local directory to the install root. + +Deliberately dumb. Each subdirectory of ``--dist-dir`` is copied verbatim to +``/``; the name is never interpreted, rewritten, or checked against +version-parsing logic. The operator names the directories to match what the tests +expect, which is also what makes fork layouts work without special cases. + +The one filter is ``--exclude``, which exists because ``ignite-dev`` is normally a link +to a source checkout: the workers need the built jars under ``modules/*/target``, and +nothing else in the tree. Excludes are opt-in and default to nothing, so a distribution +without them is still shipped byte for byte. + +Two ways to move the bytes. The default is rsync into a staging directory hardlinked +against the previous deployment (``--link-dest``), so a rebuild of one module sends one +jar rather than the whole tree. The fallback, used when either end has no rsync or when +``--via`` stages the payload on an intermediate host, is a tarball of the whole +distribution. Both end in the same atomic swap, and both carry exactly the file list the +manifest was built from - rsync is handed that list as a file rather than its own +``--exclude`` patterns, whose matching rules differ subtly from :func:`is_excluded`'s. + +Long transfers report progress per host; see :mod:`ducktests_remote.progress`. + +Nothing is written to the live path until the whole distribution is staged beside it, and +a staging directory that is not swapped in is removed on the way out - including the +leftovers of an earlier attempt, which carry a random suffix and a leading dot and would +otherwise sit in the install root forever. + +:func:`build_manifest`, :func:`staging_prefix`, :func:`prepare_script`, +:func:`discard_script`, :func:`swap_script` and :func:`human` are public because +``provision``'s ``jdk`` step delivers a JDK the same way and must not grow a second copy +of the staging-and-swap logic. +""" + +import hashlib +import json +import os +import posixpath +import re +import shlex +import shutil +import tempfile +import threading +import uuid +from pathlib import Path + +from ducktests_remote.cli import EXIT_OK, EXIT_TRANSPORT, EXIT_USAGE +from ducktests_remote.config import ConfigError, expand_path +from ducktests_remote.fanout import (CHANGED, FAILED, HostResult, SKIPPED, any_failed, + fanout, render_table, summarise) +from ducktests_remote.progress import NullProgress, build_progress +from ducktests_remote.transport import (ProxiedTransport, is_excluded, make_tarball, + parse_rsync_progress, run_local, + run_local_streaming) + +MANIFEST_NAME = ".ducktests-deploy.json" + +# Per-distribution exclude list, read from the root of the distribution itself. It is +# deliberately NOT called .ducktestsignore: when ignite-dev links to a checkout, the +# distribution root and the source root are the same directory, and the two lists are +# opposites - the source sync drops `target`, deploy keeps only `target`. +IGNORE_NAME = ".ducktests-deploy.ignore" + +# `Number of regular files transferred: 12` on rsync 3.x, `Number of files transferred` +# on 2.x. Thousands separators are locale-dependent; the labels are not translated. +_RSYNC_TRANSFERRED = re.compile(r"Number of (?:regular )?files transferred:\s*([\d,.]+)") +_RSYNC_SENT = re.compile(r"Total transferred file size:\s*([\d,.]+)") + + +def register(subparsers, common): + """Wire up the ``deploy`` subcommand.""" + parser = subparsers.add_parser( + "deploy", parents=[common], help="copy distributions to the workers", + description="Copy each subdirectory of --dist-dir to / on " + "every worker, skipping hosts that already have identical content.") + parser.add_argument("--dist-dir", metavar="PATH", + help="directory holding one subdirectory per distribution") + parser.add_argument("--only", action="append", default=[], metavar="NAME", + help="restrict to this distribution; repeatable") + parser.add_argument("--exclude", action="append", default=[], metavar="PATTERN", + help="rsync-style pattern to leave out of every distribution; " + "repeatable. Overrides %s and deploy.exclude" % IGNORE_NAME) + parser.add_argument("--install-root", metavar="PATH", help="target root on the workers") + parser.add_argument("--via", metavar="HOST", + help="upload once to HOST, then fan out from there") + parser.add_argument("--sudo", action="store_true", + help="prefix remote commands with `sudo -n`") + parser.add_argument("--owner", metavar="USER", help="chown -R the extracted tree") + parser.add_argument("--force", action="store_true", + help="redeploy even when the manifest already matches") + parser.add_argument("--checksum", action="store_true", + help="hash file contents for the manifest instead of size+mtime") + parser.add_argument("--no-rsync", action="store_true", + help="send a full tarball instead of an incremental rsync") + parser.add_argument("-n", "--num-nodes", type=int, default=None, + help="only deploy to the first N inventory hosts") + parser.add_argument("--json", action="store_true", help="machine-readable output") + parser.set_defaults(handler=execute) + + +def execute(ctx): # pylint: disable=too-many-locals + """Deploy distributions. :return: the process exit code.""" + args = ctx.args + console = ctx.console + + dist_dir = Path(expand_path(args.dist_dir or ctx.config["deploy"]["dist_dir"])) + if not dist_dir.is_dir(): + raise ConfigError("--dist-dir %s does not exist" % dist_dir) + + install_root = (args.install_root or ctx.config["deploy"].get("install_root") + or ctx.cluster_cfg.get("install_root", "/opt")) + nodes = ctx.all_nodes + if not nodes: + raise ConfigError("cluster.nodes is empty; there is nowhere to deploy") + + dists = _distributions(dist_dir, args.only) + if not dists: + console.error("no distributions found under %s" % dist_dir) + return EXIT_USAGE + + use_checksum = args.checksum or ctx.config["deploy"].get("checksum", False) + plans = [] + for name in dists: + excludes = resolve_excludes(ctx, dist_dir / name) + manifest = build_manifest(dist_dir / name, checksum=use_checksum, excludes=excludes) + if excludes and manifest["excluded"]: + console.info("%s: %d file(s) left out by %d pattern(s)" + % (name, manifest["excluded"], len(excludes))) + console.detail("excludes: %s" % ", ".join(excludes)) + plans.append((name, manifest, excludes)) + + if (not rsync_enabled(ctx) and not args.via and not args.no_rsync + and ctx.config["deploy"].get("rsync", True)): + console.detail("no usable rsync on this machine; sending whole tarballs") + + _print_cost(ctx, plans, nodes) + + overall = [] + for name, manifest, excludes in plans: + console.heading("%s -> %s/%s" % (name, install_root, name)) + progress = build_progress(ctx, nodes) + with progress: + results = _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes, + excludes, progress) + overall.extend(results) + console.out(render_table(results, verbose=console.verbose)) + console.out(summarise(results)) + + if args.json: + console.out(json.dumps([{"host": r.host, "status": r.status, "message": r.message} + for r in overall], indent=2)) + + return EXIT_TRANSPORT if any_failed(overall) else EXIT_OK + + +def _distributions(dist_dir, only): + names = sorted(p.name for p in dist_dir.iterdir() + if p.is_dir() and not p.name.startswith(".")) + if only: + missing = [n for n in only if n not in names] + if missing: + raise ConfigError("--only %s: not found under %s (available: %s)" + % (", ".join(missing), dist_dir, ", ".join(names) or "none")) + return [n for n in names if n in only] + return names + + +def resolve_excludes(ctx, dist_root): + """ + :return: the exclude patterns for one distribution, most specific source winning. + + ``--exclude`` beats a :data:`IGNORE_NAME` file at the root of the distribution, which + beats ``deploy.exclude`` in the configuration. The list is never merged across + sources: a pattern list is read as a whole, the way ``run``'s source-sync list is. + """ + ignore_file = Path(dist_root) / IGNORE_NAME + # The list file is bookkeeping, not part of the distribution; never ship it. + tail = [IGNORE_NAME] if ignore_file.is_file() else [] + if ctx.args.exclude: + return list(ctx.args.exclude) + tail + if ignore_file.is_file(): + return [line.strip() for line in ignore_file.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.strip().startswith("#")] + tail + return list(ctx.config["deploy"].get("exclude") or []) + + +def build_manifest(path, *, checksum=False, excludes=()): + """ + :return: a manifest describing ``path``, used to skip hosts that already match. + + Sorted relative paths plus sizes and mtimes by default; ``--checksum`` adds a content + sha256 per file, which is exact but reads gigabytes off disk every time. + + ``excludes`` must be the same list the tarball is built with, or a host would be + called up to date while holding a different set of files. + """ + included, excluded = _scan(path, excludes) + entries = [] + total = 0 + for entry, rel in included: + stat = entry.stat() + total += stat.st_size + if checksum: + entries.append("%s\0%d\0%s" % (rel.as_posix(), stat.st_size, _sha256(entry))) + else: + entries.append("%s\0%d\0%d" % (rel.as_posix(), stat.st_size, int(stat.st_mtime))) + digest = hashlib.sha256("\n".join(entries).encode("utf-8")).hexdigest() + return {"hash": digest, "files": len(entries), "bytes": total, "excluded": excluded, + "mode": "checksum" if checksum else "size+mtime"} + + +def included_files(path, excludes=()): + """ + :return: the relative posix paths a deploy would carry, sorted. + + This is what rsync is fed on stdin. It comes from the same walk as the manifest on + purpose: rsync's own ``--exclude`` matching is close to :func:`is_excluded` but not + identical, and a distribution that differs from the manifest describing it is exactly + the bug the manifest exists to prevent. + """ + return [rel.as_posix() for _, rel in _scan(path, excludes)[0]] + + +def _scan(path, excludes): + """:return: ``([(file, relative_path)], excluded_count)`` for one distribution.""" + included = [] + excluded = 0 + root = Path(path) + for entry in sorted(root.rglob("*")): + if entry.is_dir() or entry.is_symlink(): + continue + rel = entry.relative_to(root) + if is_excluded(rel, excludes): + excluded += 1 + continue + included.append((entry, rel)) + return included, excluded + + +def _sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _print_cost(ctx, plans, nodes): + total = sum(m["bytes"] for _, m, _ in plans) + per_host = human(total) + console = ctx.console + console.info("%d distribution(s), %s each, %d host(s) = %s total" + % (len(plans), per_host, len(nodes), human(total * len(nodes)))) + if rsync_enabled(ctx): + console.info("rsync: only what differs from each host is sent") + return + if not ctx.args.via and len(nodes) > 3 and total > 200 * 1024 * 1024: + console.warn("that is %s over the wire from this machine. `--via ` uploads it once and fans out from there." + % human(total * len(nodes))) + + +def human(size): + value = float(size) + for unit in ("B", "KB", "MB", "GB", "TB"): + if value < 1024 or unit == "TB": + return "%.0f %s" % (value, unit) if unit in ("B", "KB") else "%.1f %s" % (value, unit) + value /= 1024 + return "%.1f TB" % value + + +def _deploy_one(ctx, dist_dir, name, manifest, install_root, nodes, excludes=(), + progress=None): + args = ctx.args + target = posixpath.join(install_root, name) + manifest_body = json.dumps({ + **manifest, + "name": name, + "source": str((Path(dist_dir) / name).resolve()), + "deployed_by": os.environ.get("USER") or os.environ.get("USERNAME") or "unknown", + "deployed_at": _now(), + }, indent=2, sort_keys=True) + + if ctx.dry_run: + how = "rsync" if rsync_enabled(ctx) else "tarball" + return [HostResult(node.host, SKIPPED, "would send %s to %s (%s)" + % (human(manifest["bytes"]), target, how)) + for node in nodes] + + with tempfile.TemporaryDirectory() as tmp: + payload = _Payload(Path(dist_dir) / name, Path(tmp) / ("%s.tar.gz" % name), excludes) + + via_transport = None + staged_on_via = None + if args.via: + via_transport = ctx.worker(_via_node(ctx, args.via)) + staged_dir = ctx.config["deploy"]["staging_dir"] + via_transport.mkdirs(staged_dir) + staged_on_via = posixpath.join(staged_dir, "%s-%s.tar.gz" + % (name, uuid.uuid4().hex[:8])) + ctx.console.info("staging %s on %s" % (name, args.via)) + via_transport.upload(payload.archive(), staged_on_via) + + file_list = None + if rsync_enabled(ctx): + file_list = Path(tmp) / ("%s.files" % name) + file_list.write_text("\n".join(included_files(Path(dist_dir) / name, excludes)) + + "\n", encoding="utf-8") + + def operation(node, _payload=payload, _staged=staged_on_via, _via=via_transport, + _files=file_list): + return _deploy_to_host(ctx, node, name, target, manifest, manifest_body, + _payload, _staged, _via, _files, progress) + + try: + return fanout(nodes, operation, jobs=ctx.jobs, + fail_fast=getattr(ctx.args, "fail_fast", False)) + finally: + if via_transport is not None and staged_on_via: + via_transport.run(["rm", "-f", "--", staged_on_via], check=False) + + +def _via_node(ctx, host): + for node in ctx.all_nodes: + if node.host == host: + return node + from ducktests_remote.cluster import Node # pylint: disable=import-outside-toplevel + return Node(host=host, user=ctx.cluster_cfg.get("user"), + port=ctx.cluster_cfg.get("port", 22), + identity_file=ctx.cluster_cfg.get("identity_file")) + + +class _Payload: + """ + The tarball for one distribution, built at most once and only if a host needs it. + + With every host on the rsync path, a distribution is never tarred at all - which for + a linked checkout is a gigabyte of compression that would buy nothing. + """ + + def __init__(self, root, archive_path, excludes): + self.root = root + self._archive = archive_path + self._excludes = excludes + self._built = False + self._lock = threading.Lock() + + def archive(self): + """:return: the path to the tarball, building it on first use.""" + with self._lock: + if not self._built: + make_tarball(self.root, self._archive, excludes=self._excludes) + self._built = True + return self._archive + + +def rsync_enabled(ctx): + """ + :return: True when the incremental path may be used for this invocation. + + ``--via`` is deliberately excluded: its whole point is that the payload crosses the + slow link once, as one file, which is the opposite of a per-host rsync. + """ + if getattr(ctx.args, "no_rsync", False) or getattr(ctx.args, "via", None): + return False + if not ctx.config["deploy"].get("rsync", True): + return False + if os.name == "nt": + # rsync reads `C:/dist/ignite-dev` as host `C`, and a coordinator that hands it a + # Windows path silently deploys nothing. The tarball path has no such problem. + return False + return shutil.which("rsync") is not None + + +def rsync_argv(transport, local_root, staging, *, files_from, link_dest=None, + checksum=False, sudo=False, progress=False): + """ + :return: the rsync command line that fills ``staging`` on one worker. + + ``--link-dest`` is what makes a redeploy cheap on both sides: unchanged files become + hardlinks to the deployment already on the host and are never sent, so a rebuild of + one module costs one jar. The staging directory is still built from scratch and + still swapped in atomically, so an interrupted transfer cannot leave a live + distribution half updated. + + ``files_from`` is a file holding the exact list to send, written once per + distribution. It is not stdin: the output side is read as it arrives for progress, + and a process whose stdin and stdout are both being pumped by one thread deadlocks + as soon as either pipe fills. + """ + argv = ["rsync", "-a", "--stats", "--files-from=%s" % files_from, + "-e", shlex.join(["ssh"] + transport.ssh_options())] + if progress: + argv.append("--info=progress2") + if checksum: + argv.append("--checksum") + if link_dest: + argv.append("--link-dest=%s" % link_dest) + if sudo: + argv.append("--rsync-path=sudo -n rsync") + argv += ["%s/" % str(local_root).rstrip("/\\"), + "%s:%s/" % (transport.target, staging)] + return argv + + +def parse_rsync_stats(text): + """:return: ``(files_transferred, bytes_transferred)`` from ``--stats``, or None.""" + transferred = _RSYNC_TRANSFERRED.search(text or "") + sent = _RSYNC_SENT.search(text or "") + if not transferred or not sent: + return None + try: + return (int(re.sub(r"[,.]", "", transferred.group(1))), + int(re.sub(r"[,.]", "", sent.group(1)))) + except ValueError: + return None + + +def _deploy_to_host(ctx, node, name, target, manifest, manifest_body, payload, + staged_on_via, via_transport, file_list=None, progress=None): + transport = ctx.worker(node) + progress = progress or NullProgress() + remote_manifest = posixpath.join(target, MANIFEST_NAME) + + if not ctx.args.force: + existing = transport.read_file(remote_manifest) + if existing: + try: + if json.loads(existing).get("hash") == manifest["hash"]: + progress.done(node.host, "already up to date") + return HostResult(node.host, SKIPPED, "already at %s" + % manifest["hash"][:12]) + except ValueError: + pass + + install_root = posixpath.dirname(target) + writable = transport.run(["test", "-w", install_root], check=False).ok + if not writable and not ctx.args.sudo: + progress.done(node.host, "not writable") + return HostResult(node.host, FAILED, + "%s is not writable by %s and --sudo was not passed" + % (install_root, node.user or "this account")) + + sweep = staging_prefix(install_root, name) + staging = "%s%s" % (sweep, uuid.uuid4().hex[:8]) + used_rsync = False + stats = None + swapped = False + + try: + if via_transport is not None: + progress.phase(node.host, "fanning out") + proxied = ProxiedTransport(name=node.host, via=via_transport, user=node.user, + port=node.port, + identity_file=node.identity_file, + staging_dir=ctx.config["deploy"]["staging_dir"], + dry_run=ctx.dry_run, verbose=ctx.console.verbose) + proxied.run_script(prepare_script(staging, ctx.args.sudo, sweep=sweep)).check() + proxied.push_archive(staged_on_via, staging) + elif file_list is not None and getattr(transport, "has_rsync", _no_rsync)(): + outcome = _rsync_to_host(ctx, node, transport, payload.root, staging, target, + file_list, progress, sweep) + if isinstance(outcome, HostResult): + progress.done(node.host, "failed") + return outcome + used_rsync, stats = True, outcome + else: + transport.run_script(prepare_script(staging, ctx.args.sudo, sweep=sweep)).check() + remote_archive = "%s/.payload.tar.gz" % staging + archive = payload.archive() + progress.phase(node.host, "sending") + transport.upload_watched(archive, remote_archive, + on_bytes=_watcher(progress, node.host, + os.path.getsize(archive))) + progress.phase(node.host, "extracting") + transport.run_script( + "set -eu\ntar -xzf %s -C %s\nrm -f -- %s\n" + % (shlex.quote(remote_archive), shlex.quote(staging), + shlex.quote(remote_archive))).check() + + progress.phase(node.host, "swapping") + transport.write_file(manifest_body, posixpath.join(staging, MANIFEST_NAME)) + transport.run_script(swap_script(staging, target, ctx.args.sudo, + ctx.args.owner)).check() + swapped = True + finally: + # A staging tree holds a whole copy of the distribution. Once the swap has moved + # it into place there is nothing left at this path; before that, any way out of + # here - a failed host, a transport error, Ctrl-C - has to take it with it. + if not swapped: + transport.run_script(discard_script(staging, ctx.args.sudo), check=False) + + progress.done(node.host) + if used_rsync: + if stats: + return HostResult(node.host, CHANGED, "rsync: %d of %s file(s) changed, %s sent" + % (stats[0], manifest["files"], human(stats[1]))) + return HostResult(node.host, CHANGED, "rsync: %s file(s)" % manifest["files"]) + return HostResult(node.host, CHANGED, "%s files, %s" + % (manifest["files"], human(manifest["bytes"]))) + + +def _no_rsync(): + """``has_rsync`` stand-in for transports that have no such notion (local copies).""" + return False + + +def _watcher(progress, host, total): + """:return: an ``on_bytes`` callback that feeds one host's row.""" + return lambda sent: progress.sent(host, sent, total) + + +def _rsync_to_host(ctx, node, transport, local_root, staging, target, file_list, + progress=None, sweep=None): + """ + :return: ``(files, bytes)`` transferred, ``None`` when ``--stats`` could not be read, + or a failed :class:`HostResult`. + + The staging directory is created before rsync runs, and hardlinked against the live + distribution when there is one to link against. + """ + progress = progress or NullProgress() + transport.run_script(prepare_script(staging, ctx.args.sudo, sweep=sweep)).check() + # Both display modes need the byte counts; only the display itself differs. + watched = progress.watching + argv = rsync_argv(transport, local_root, staging, files_from=file_list, + link_dest=target if transport.exists(target) else None, + checksum=ctx.args.checksum or ctx.config["deploy"].get("checksum", False), + sudo=ctx.args.sudo, progress=watched) + ctx.console.detail("%s: %s" % (node.host, shlex.join(argv))) + progress.phase(node.host, "sending") + if watched: + result = run_local_streaming(argv, host=node.host, + on_output=_rsync_watcher(progress, node.host)) + else: + result = run_local(argv) + if not result.ok: + return HostResult(node.host, FAILED, "rsync failed", + detail=(result.stderr or result.stdout).strip()) + return parse_rsync_stats(result.stdout) + + +def _rsync_watcher(progress, host): + """:return: a line callback that turns ``--info=progress2`` into one host's row.""" + def watch(line): + reading = parse_rsync_progress(line) + if reading: + progress.sent(host, reading[0], fraction=reading[1]) + return watch + + +def staging_prefix(install_root, name): + """ + :return: the path prefix shared by every staging directory for one distribution. + + A random suffix is appended to it per attempt. Keeping the prefix in one function is + what lets :func:`prepare_script` recognise the leftovers of an earlier attempt. + """ + return "%s/.%s.tmp." % (install_root, name) + + +def prepare_script(staging, use_sudo, sweep=None): + """ + Make an empty staging directory to fill. + + ``sweep`` is the path prefix every staging directory for this distribution shares. + Anything matching it is removed first: each attempt picks a fresh random suffix, so a + deploy killed between the transfer and the swap would otherwise leave a full copy of + the distribution behind that nothing ever looks at again - and a dot-prefixed name at + that, invisible to a plain ``ls``. + """ + sudo = "sudo -n " if use_sudo else "" + script = "set -eu\n" + if sweep: + script += "%srm -rf -- %s*\n" % (sudo, shlex.quote(sweep)) + return script + "%(sudo)srm -rf -- %(staging)s\n%(sudo)smkdir -p %(staging)s\n" % { + "sudo": sudo, "staging": shlex.quote(staging)} + + +def discard_script(staging, use_sudo): + """:return: a script removing a staging directory that will never be swapped in.""" + return "set -eu\n%srm -rf -- %s\n" % ("sudo -n " if use_sudo else "", + shlex.quote(staging)) + + +def swap_script(staging, target, use_sudo, owner): + """ + Swap the freshly extracted tree into place, then delete the old one. + + A half-copied distribution that looks present is worse than an absent one: the tests + start, fail somewhere inside the JVM, and nobody suspects the copy. + """ + sudo = "sudo -n " if use_sudo else "" + script = """set -eu +staging=%(staging)s +target=%(target)s +old="$target.old.$$" +if [ -e "$target" ]; then %(sudo)smv -- "$target" "$old"; fi +%(sudo)smv -- "$staging" "$target" +if [ -e "$old" ]; then %(sudo)srm -rf -- "$old"; fi +""" % {"staging": shlex.quote(staging), "target": shlex.quote(target), "sudo": sudo} + if owner: + script += "%schown -R %s -- %s\n" % (sudo, shlex.quote(owner), shlex.quote(target)) + return script + + +def _now(): + from datetime import datetime, timezone # pylint: disable=import-outside-toplevel + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() diff --git a/modules/ducktests/tests/ducktests_remote/commands/doctor.py b/modules/ducktests/tests/ducktests_remote/commands/doctor.py new file mode 100644 index 0000000000000..e0e133bd8f0e2 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/doctor.py @@ -0,0 +1,642 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``doctor`` - preflight, run standalone or implicitly before ``run``. + +Every check is driven by the inventory, so a two-node cluster produces the same output +shape as a forty-nine node one. The command never stops at the first failure: an +operator who has to ask an administrator for access needs the complete list in one go. +""" + +import json +import posixpath +import re +import shlex +import shutil +import time + +from ducktests_remote import java, pipconf, sshdiag +from ducktests_remote.cli import EXIT_OK, EXIT_PREFLIGHT +from ducktests_remote.config import (REQUIREMENTS_RELPATH, SUDO_DEPENDENT_TESTS, + expand_path) +from ducktests_remote.fanout import HostResult, fanout +from ducktests_remote.transport import TransportError, run_local + +OK = "OK" +WARN = "WARN" +FAIL = "FAIL" + +DISK_WARN_GB = 10 +CLOCK_WARN_SEC = 5 +CLOCK_FAIL_SEC = 60 + + +class Check: + """One preflight observation.""" + + def __init__(self, scope, host, name, status, message=""): + self.scope = scope + self.host = host + self.name = name + self.status = status + self.message = message + + def as_dict(self): + """:return: a JSON-serialisable view.""" + return {"scope": self.scope, "host": self.host, "name": self.name, + "status": self.status, "message": self.message} + + +def register(subparsers, common): + """Wire up the ``doctor`` subcommand.""" + parser = subparsers.add_parser( + "doctor", parents=[common], + help="check the coordinator, the runner and every worker", + description="Check that this cluster can actually run ducktests, and when it " + "cannot, say precisely what is missing and who has to grant it.") + parser.add_argument("--json", action="store_true", help="machine-readable output") + parser.add_argument("-n", "--num-nodes", type=int, default=None, + help="only check the first N inventory hosts") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Run every check and print the report. :return: the process exit code.""" + checks, diagnoses = run_checks(ctx) + + if getattr(ctx.args, "json", False): + ctx.console.out(json.dumps({ + "checks": [c.as_dict() for c in checks], + "ssh": [{"host": d.host, "classification": d.classification, + "advice": d.advice} for d in diagnoses], + "ok": not has_failures(checks), + }, indent=2)) + else: + print_report(ctx, checks, diagnoses) + + return EXIT_PREFLIGHT if has_failures(checks) else EXIT_OK + + +def print_report(ctx, checks, diagnoses): + """Print the human-readable doctor report.""" + console = ctx.console + width_host = max([len(c.host) for c in checks] + [4]) + width_name = max([len(c.name) for c in checks] + [5]) + + last_scope = None + for check in checks: + if check.scope != last_scope: + console.heading(check.scope.upper()) + last_scope = check.scope + style = {"OK": "green", "WARN": "yellow", "FAIL": "red"}[check.status] + console.out(" %s %s %s %s" % ( + console.paint(check.status.ljust(4), style), + check.host.ljust(width_host), + check.name.ljust(width_name), + check.message)) + + counts = {} + for check in checks: + counts[check.status] = counts.get(check.status, 0) + 1 + console.out("") + console.out("%d checks: %s" % ( + len(checks), + ", ".join("%d %s" % (counts[s], s) for s in (OK, WARN, FAIL) if s in counts))) + + block = sshdiag.admin_request_block( + diagnoses, user=ctx.cluster_cfg.get("user"), + identity_file=expand_path(ctx.identity_file)) + if block: + console.out(block) + + +def has_failures(checks): + """:return: True when any check failed.""" + return any(c.status == FAIL for c in checks) + + +def run_checks(ctx, versions=None): + """ + Run every preflight check. + + :param versions: ``globals.ignite_versions`` when known, used for the install-root check. + :return: ``(checks, ssh_diagnoses)``. + """ + checks = [] + checks += coordinator_checks(ctx) + checks += runner_checks(ctx) + + nodes = ctx.all_nodes + if not nodes: + checks.append(Check("workers", "-", "inventory", FAIL, + "cluster.nodes is empty; nothing to check")) + return checks, [] + + diagnoses = probe_workers(ctx, nodes) + by_host = {d.host: d for d in diagnoses} + for node in nodes: + diagnosis = by_host.get(node.host) + if diagnosis and not diagnosis.ok: + checks.append(Check("workers", node.host, "ssh", FAIL, + "%s: %s" % (diagnosis.classification, diagnosis.advice))) + else: + checks.append(Check("workers", node.host, "ssh", OK, "reachable")) + + reachable = [n for n in nodes if by_host.get(n.host) and by_host[n.host].ok] + if reachable: + checks += worker_checks(ctx, reachable, versions or _configured_versions(ctx)) + checks += resolution_checks(ctx, reachable) + checks += runner_to_worker_checks(ctx, reachable) + return checks, diagnoses + + +# -- coordinator ------------------------------------------------------------------ + + +def coordinator_checks(ctx): + """Checks about the machine the CLI is running on.""" + if ctx.runner_host != "local": + checks = [] + for tool in ("ssh", "scp"): + found = _which(tool) + checks.append(Check("coordinator", "local", tool, + OK if found else FAIL, + "on PATH" if found else "not on PATH; install openssh-client")) + return checks + + if ctx.dry_run: + return [Check("coordinator", "local", "ducktape", OK, "not probed (--dry-run)")] + + importable, version = _local_ducktape(_venv_python(ctx)) + if not importable: + return [Check("coordinator", "local", "ducktape", FAIL, + "--runner local but ducktape is not importable here")] + pinned = pinned_ducktape_version(ctx) + status = OK if (pinned is None or version == pinned) else WARN + return [Check("coordinator", "local", "ducktape", status, + "%s (pinned %s)" % (version, pinned or "unknown"))] + + +def _which(tool): + return shutil.which(tool) is not None + + +def _local_ducktape(python): + """ + :return: ``(importable, version)`` for a locally installed ducktape. + + Done in a subprocess on purpose: ``ducktests_remote`` must never import ducktape, so + the boundary holds even for this check. + """ + result = run_local([python, "-c", + "import ducktape,sys;sys.stdout.write(ducktape.__version__)"]) + return result.ok, result.out + + +# -- runner ----------------------------------------------------------------------- + + +def runner_checks(ctx): + """Checks about the host that will run the ducktape process.""" + host = ctx.runner_host + checks = [] + try: + # `exit 0` at the end: a missing setsid is a WARN, not a reason for the whole + # probe to report the runner as unreachable. + probe = ctx.runner.run_script("for t in bash python3 setsid; do\n" + " command -v \"$t\" >/dev/null 2>&1 && echo \"$t\"\n" + "done\nexit 0\n", + check=False) + except TransportError as ex: + diagnosis = sshdiag.diagnose_exception(ex, host=host, + user=ctx.cluster_cfg.get("user"), + port=ctx.cluster_cfg.get("port", 22)) + return [Check("runner", host, "ssh", FAIL, + "%s: %s" % (diagnosis.classification, diagnosis.advice))] + + if not probe.ok and not ctx.dry_run: + detail = (probe.stderr.strip() or probe.stdout.strip() + or "exit %d with no output" % probe.returncode) + return [Check("runner", host, "reachable", FAIL, detail[:160])] + + present = set(probe.stdout.split()) + for tool in ("bash", "python3"): + checks.append(Check("runner", host, tool, OK if tool in present or ctx.dry_run else FAIL, + "present" if tool in present or ctx.dry_run else "missing")) + checks.append(Check("runner", host, "setsid", + OK if "setsid" in present or ctx.dry_run else WARN, + "present" if "setsid" in present or ctx.dry_run + else "missing; falling back to nohup + disown")) + + checks.append(_runner_ducktape_check(ctx)) + checks.append(_identity_check(ctx)) + checks.append(_runner_disk_check(ctx)) + # Informational: reaching the index is pip's business, and doctor does not make + # network calls on someone else's behalf to find out. + checks.append(Check("runner", host, "pip", OK, + pipconf.describe(ctx.config, ctx.console.redactor))) + return checks + + +def _runner_disk_check(ctx): + path = ctx.runner.expand(ctx.state_root) + result = ctx.runner.run_script( + "set -u\nmkdir -p %s 2>/dev/null || true\n" + "df -Pk %s 2>/dev/null | awk 'NR==2{printf \"%%d\", $4/1048576}'\n" + % (shlex.quote(path), shlex.quote(path)), check=False) + if ctx.dry_run: + return Check("runner", ctx.runner_host, "disk", OK, "not probed (--dry-run)") + free = _as_int(result.out) + if free is None: + return Check("runner", ctx.runner_host, "disk", WARN, + "could not read free space on %s" % path) + return Check("runner", ctx.runner_host, "disk", + WARN if free < DISK_WARN_GB else OK, "%d GB free on %s" % (free, path)) + + +def _runner_ducktape_check(ctx): + host = ctx.runner_host + pinned = pinned_ducktape_version(ctx) + python = _venv_python(ctx) + result = ctx.runner.run([python, "-c", + "import ducktape,sys;sys.stdout.write(ducktape.__version__)"], + check=False) + if ctx.dry_run: + return Check("runner", host, "ducktape", OK, "not probed (--dry-run)") + if not result.ok: + return Check("runner", host, "ducktape", FAIL, + "not importable via %s; run `ducktests-remote provision --only python`" + % python) + if pinned and result.out != pinned: + return Check("runner", host, "ducktape", WARN, + "%s installed, %s pinned in %s" % (result.out, pinned, REQUIREMENTS_RELPATH)) + return Check("runner", host, "ducktape", OK, result.out) + + +def _identity_check(ctx): + host = ctx.runner_host + identity = ctx.identity_file + if not identity: + return Check("runner", host, "identity", WARN, + "cluster.identity_file is unset; ducktape will rely on the runner's " + "ssh agent, which a detached run cannot keep") + remote = ctx.runner.expand(identity) + result = ctx.runner.run_script( + "set -e\nf=%s\n[ -f \"$f\" ] || { echo missing; exit 0; }\n" + "printf '%%s\\n' \"$(ls -l \"$f\" | cut -c1-10)\"\n" % shlex.quote(remote), + check=False) + if ctx.dry_run: + return Check("runner", host, "identity", OK, "not probed (--dry-run)") + text = result.out + if not result.ok or text == "missing": + return Check("runner", host, "identity", FAIL, + "%s does not exist on the runner. identity_file is a RUNNER-side path, " + "not a coordinator-side one; `ducktests-remote keys push` installs it." + % remote) + if len(text) >= 10 and text[4:10] != "------": + return Check("runner", host, "identity", WARN, + "%s is %s; ssh wants 0600 or stricter" % (remote, text)) + return Check("runner", host, "identity", OK, remote) + + +def _venv_python(ctx): + venv = ctx.config["runner"].get("venv") + if venv: + return posixpath.join(ctx.runner.expand(venv), "bin", "python3") + return ctx.config["runner"].get("python", "python3") + + +def pinned_ducktape_version(ctx): + """:return: the ducktape version pinned in docker/requirements.txt, or None.""" + source_root = ctx.config["run"].get("source_root") or "." + path = expand_path(posixpath.join(str(source_root).replace("\\", "/"), REQUIREMENTS_RELPATH)) + try: + with open(path, "r", encoding="utf-8") as handle: + text = handle.read() + except OSError: + return None + match = re.search(r"^ducktape==([^\s#]+)", text, re.MULTILINE) + return match.group(1) if match else None + + +# -- workers ---------------------------------------------------------------------- + + +def probe_workers(ctx, nodes): + """ + Probe every worker from the coordinator, classifying each failure. + + This is one parallel pass over the whole inventory; nothing short-circuits, because a + partial list of broken hosts cannot be turned into a single request to an + administrator. + """ + def probe(node): + transport = ctx.worker(node) + try: + result = transport.run(["true"], check=False) + diagnosis = sshdiag.diagnose(result, host=node.host, user=node.user, port=node.port) + except TransportError as ex: + diagnosis = sshdiag.diagnose_exception(ex, host=node.host, user=node.user, + port=node.port) + return HostResult(node.host, "ok" if diagnosis.ok else "failed", + diagnosis.classification, data=diagnosis) + + results = fanout(nodes, probe, jobs=ctx.jobs) + return [r.data for r in results if r.data is not None] + + +def worker_checks(ctx, nodes, versions): + """Substantive per-worker checks: java, disk, clock, stale JVMs, install dirs, sudo.""" + if ctx.dry_run: + return [Check("workers", node.host, "probe", OK, + "java, disk, clock, stale JVMs, %s (not probed: --dry-run)" + % ("install dirs for " + ", ".join(versions) if versions + else "install dirs")) + for node in nodes] + + reference = time.time() + install_root = ctx.cluster_cfg.get("install_root", "/opt") + persistent = (ctx.config["clean"]["paths"] or ["/mnt/service"])[0] + pattern = ctx.config["clean"]["process_pattern"] + java_cfg = java.config_of(ctx) + + script = _WORKER_SCRIPT % { + "install_root": shlex.quote(install_root), + "persistent": shlex.quote(persistent), + "pattern": shlex.quote(pattern), + "java_probe": java.discovery_script(java_cfg), + } + + def probe(node): + result = ctx.worker(node).run_script(script, check=False) + return HostResult(node.host, "ok" if result.ok else "failed", + "", detail=result.stderr, data=_parse_kv(result.stdout)) + + results = fanout(nodes, probe, jobs=ctx.jobs) + facts = {r.host: (r.data or {}) for r in results} + + checks = [] + checks += _java_checks(facts, java_cfg) + for host in sorted(facts): + fact = facts[host] + checks.append(_disk_from_facts(host, "install_free_gb", install_root, fact)) + checks.append(_disk_from_facts(host, "work_free_gb", persistent, fact)) + checks.append(_clock_check(host, fact, reference)) + checks.append(_stale_jvm_check(host, fact, pattern)) + checks += _privilege_checks(host, fact, ctx, install_root, persistent) + checks += _install_dir_checks(host, fact, install_root, versions) + return [c for c in checks if c is not None] + + +_WORKER_SCRIPT = """ +set -u +say() { printf '%%s=%%s\\n' "$1" "$2"; } +say epoch "$(date +%%s)" +say whoami "$(id -un 2>/dev/null || echo '?')" +say install_free_gb "$(df -Pk %(install_root)s 2>/dev/null | awk 'NR==2{printf "%%d", $4/1048576}')" +say install_writable "$([ -w %(install_root)s ] && echo 1 || echo 0)" +if [ -d %(persistent)s ]; then + say work_free_gb "$(df -Pk %(persistent)s 2>/dev/null | awk 'NR==2{printf "%%d", $4/1048576}')" + say work_writable "$([ -w %(persistent)s ] && echo 1 || echo 0)" +else + parent=$(dirname %(persistent)s) + say work_free_gb "$(df -Pk "$parent" 2>/dev/null | awk 'NR==2{printf "%%d", $4/1048576}')" + say work_writable "$([ -w "$parent" ] && echo 1 || echo 0)" +fi +say stale "$(pgrep -f %(pattern)s 2>/dev/null | wc -l | tr -d ' ')" +if sudo -n true 2>/dev/null; then say sudo 1; else say sudo 0; fi +say install_dirs "$(ls -1 %(install_root)s 2>/dev/null | tr '\\n' ',' )" +# Last, because the JDK probe ends in `exit 0`. +%(java_probe)s +""" + + +def _java_checks(facts, java_cfg): + """ + Judge the JVM the tests will actually get. + + The question is deliberately not "is a good JDK installed somewhere" but "what does a + non-interactive ssh session get", because that is the JVM ``ignite.sh``, the Kafka + service, jmxterm and ``jvm_utils.java_version`` all end up running. A JDK of the + right version sitting unused in ``/opt`` is a remedy to suggest, not a pass. + """ + checks = [] + seen = {} + for host in sorted(facts): + res = java.parse_facts(host, facts[host], java_cfg) + version = facts[host].get("java", "missing") + if res.path_major is None: + checks.append(Check("workers", host, "java", FAIL, + "java not on the non-interactive PATH; run `provision " + "--only jdk --only ssh-env`")) + continue + seen.setdefault(res.path_version or version, []).append(host) + checks.append(_java_check(host, res, java_cfg)) + + if len(seen) > 1: + majority = max(seen, key=lambda k: len(seen[k])) + outliers = [h for k, hosts in seen.items() if k != majority for h in hosts] + checks.append(Check("workers", "-", "java-consistency", WARN, + "mixed JDKs; majority %r, outliers: %s" + % (majority, ", ".join(sorted(outliers))))) + return checks + + +def _java_check(host, res, java_cfg): + if not res.path_matches(java_cfg.major): + if res.selected: + return Check("workers", host, "java", FAIL, + "non-interactive java is %s, requested %s; %s is present on this " + "host - run `provision --only jdk --only ssh-env`" + % (res.path_version, java_cfg.major, res.home)) + found = ", ".join("%s (Java %s)" % (home, major) for home, major, _ in res.candidates) + return Check("workers", host, "java", FAIL, + "non-interactive java is %s, requested %s, and no Java %s is " + "installed here. Found: %s. Set java.archive to deliver one, or " + "java.home to name an existing one." + % (res.path_version, java_cfg.major, java_cfg.major, found or "nothing")) + if java_cfg.home and not res.home_in_effect: + # Right version, wrong JDK: the tests will run, so this is not a failure, but an + # explicit java.home that is not in effect is always worth saying out loud. + return Check("workers", host, "java", WARN, + "non-interactive java is %s from %s, but java.home names %s" + % (res.path_version, res.path_real or res.path_java, java_cfg.home)) + return Check("workers", host, "java", OK, + "%s from %s" % (res.path_version, res.path_real or res.path_java)) + + +def _disk_from_facts(host, key, path, fact=None): + if fact is None: + return None + free = _as_int(fact.get(key)) + if free is None: + return Check("workers", host, "disk", WARN, "could not read free space on %s" % path) + status = WARN if free < DISK_WARN_GB else OK + return Check("workers", host, "disk", status, "%d GB free on %s" % (free, path)) + + +def _clock_check(host, fact, reference): + epoch = _as_int(fact.get("epoch")) + if epoch is None: + return Check("workers", host, "clock", WARN, "could not read the clock") + skew = abs(epoch - reference) + if skew > CLOCK_FAIL_SEC: + return Check("workers", host, "clock", FAIL, "%.0fs skew; enable NTP" % skew) + if skew > CLOCK_WARN_SEC: + return Check("workers", host, "clock", WARN, "%.0fs skew" % skew) + return Check("workers", host, "clock", OK, "%.0fs skew" % skew) + + +def _stale_jvm_check(host, fact, pattern): + count = _as_int(fact.get("stale")) or 0 + if count: + return Check("workers", host, "stale-jvm", FAIL, + "%d process(es) matching %r are still running from an earlier run; " + "run `ducktests-remote clean --dry-run` then `clean`" % (count, pattern)) + return Check("workers", host, "stale-jvm", OK, "none") + + +def _privilege_checks(host, fact, ctx, install_root, persistent): + checks = [] + user = fact.get("whoami") or ctx.cluster_cfg.get("user") + checks.append(Check("workers", host, "account", OK, "connected as %s" % user)) + checks.append(Check("workers", host, "write-install", + OK if fact.get("install_writable") == "1" else WARN, + "%s %s" % (install_root, + "writable" if fact.get("install_writable") == "1" + else "NOT writable; deploy needs --sudo here"))) + checks.append(Check("workers", host, "write-work", + OK if fact.get("work_writable") == "1" else FAIL, + "%s %s" % (persistent, + "writable" if fact.get("work_writable") == "1" + else "NOT writable; tests cannot create their work dirs"))) + if fact.get("sudo") == "1": + checks.append(Check("workers", host, "sudo", OK, "passwordless")) + else: + checks.append(Check("workers", host, "sudo", WARN, + "no passwordless sudo; only the network-segmentation suites need " + "it (%s)" % ", ".join(SUDO_DEPENDENT_TESTS))) + return checks + + +def _install_dir_checks(host, fact, install_root, versions): + """ + Report the distributions found under the install root. + + ``ignitetest`` resolves a home directory as ``install_root/str(IgniteVersion(v))`` + (services/utils/path.py ``get_home_dir`` with ``IgniteAwareService.product``), and + ``IgniteVersion.__str__`` normalises the string, so ``ise--6`` maps to ``ise-6``. + Because a fork can override ``product``, a missing directory is reported as a WARN + listing what *was* found rather than as a hard failure on a guessed mapping. + """ + found = [d for d in (fact.get("install_dirs") or "").split(",") if d] + if not versions: + return [Check("workers", host, "install-root", OK, + "%d distribution(s) under %s" % (len(found), install_root))] + missing = [v for v in versions if v not in found] + if missing: + return [Check("workers", host, "install-root", WARN, + "no directory for %s under %s; found: %s" + % (", ".join(missing), install_root, ", ".join(sorted(found)) or "(none)"))] + return [Check("workers", host, "install-root", OK, + "found %s" % ", ".join(versions))] + + +def resolution_checks(ctx, nodes): + """ + Verify node-to-node name resolution from a single probe host. + + The Docker flow writes every node into ``/etc/hosts`` explicitly. On real VMs name + resolution has to already work worker-to-worker, and when it does not, tests fail + deep inside discovery with no message that points here. + """ + if len(nodes) < 2: + return [] + probe_node = nodes[0] + targets = [n.host for n in nodes if n.host != probe_node.host] + script = "for h in %s; do\n if getent hosts \"$h\" >/dev/null 2>&1; then :; " \ + "else echo \"$h\"; fi\ndone\n" % " ".join(shlex.quote(t) for t in targets) + result = ctx.worker(probe_node).run_script(script, check=False) + if ctx.dry_run: + return [Check("workers", probe_node.host, "dns", OK, "not probed (--dry-run)")] + if not result.ok: + return [Check("workers", probe_node.host, "dns", WARN, + "could not run the resolution probe")] + unresolved = [line.strip() for line in result.stdout.splitlines() if line.strip()] + if unresolved: + return [Check("workers", probe_node.host, "dns", WARN, + "cannot resolve %d peer(s) from this host: %s. Run `provision " + "--write-hosts` or fix cluster DNS." + % (len(unresolved), ", ".join(unresolved[:8])))] + return [Check("workers", probe_node.host, "dns", OK, + "resolves all %d peers" % len(targets))] + + +def runner_to_worker_checks(ctx, nodes): + """ + Verify the connection ducktape will actually make: runner to worker, with the + runner-side identity file. A coordinator that can reach a worker proves nothing + about the runner being able to. + """ + if ctx.runner_host == "local" or not nodes: + return [] + identity = ctx.runner.expand(ctx.identity_file) if ctx.identity_file else None + opts = "-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10" + if identity: + opts += " -o IdentitiesOnly=yes -i %s" % shlex.quote(identity) + lines = [] + for node in nodes: + port = "" if int(node.port) == 22 else " -p %d" % int(node.port) + lines.append("if ssh %s%s %s true >/dev/null 2>&1; then echo \"%s=1\"; " + "else echo \"%s=0\"; fi" + % (opts, port, shlex.quote(node.target), node.host, node.host)) + result = ctx.runner.run_script("\n".join(lines) + "\n", check=False) + if ctx.dry_run: + return [Check("runner->workers", ctx.runner_host, "ssh", OK, "not probed (--dry-run)")] + facts = _parse_kv(result.stdout) + checks = [] + for node in nodes: + reachable = facts.get(node.host) == "1" + checks.append(Check("runner->workers", node.host, "ssh", + OK if reachable else FAIL, + "reachable from %s" % ctx.runner_host if reachable else + "the runner cannot ssh to %s as %s with %s. This is the connection " + "ducktape makes; fix it with `ducktests-remote keys push`." + % (node.host, node.user, identity or "the default identity"))) + return checks + + +def _configured_versions(ctx): + versions = ctx.config.get("globals", {}).get("ignite_versions") + if isinstance(versions, str): + return [versions] + return list(versions or []) + + +def _parse_kv(text): + fields = {} + for line in (text or "").split("\n"): + if "=" in line: + key, value = line.split("=", 1) + fields[key.strip()] = value.strip() + return fields + + +def _as_int(value): + try: + return int(str(value).strip()) + except (TypeError, ValueError): + return None diff --git a/modules/ducktests/tests/ducktests_remote/commands/fetch.py b/modules/ducktests/tests/ducktests_remote/commands/fetch.py new file mode 100644 index 0000000000000..18f8277cf7a5d --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/fetch.py @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fetch`` - bring a run's reports back to the coordinator.""" + +import posixpath +import shlex +import tarfile +import tempfile +import uuid +from pathlib import Path + +from ducktests_remote import runs +from ducktests_remote.cli import EXIT_OK, EXIT_USAGE +from ducktests_remote.config import expand_path + +# ducktape writes these into /; see ducktape/tests/reporter.py. +DEFAULT_FILES = ("report.html", "report.txt", "report.json", "test_log.info", "session.log") + +# globals.json holds the composed secrets. It never leaves the runner. +ALWAYS_EXCLUDED = ("globals.json",) + + +def register(subparsers, common): + """Wire up the ``fetch`` subcommand.""" + parser = subparsers.add_parser( + "fetch", parents=[common], help="download a run's results", + description="Download the reports for a run. globals.json is always excluded.") + parser.add_argument("run_id", nargs="?", help="run id (default: the most recent run)") + parser.add_argument("--dest", metavar="DIR", default="./ducktests-results", + help="coordinator-side destination (default ./ducktests-results)") + parser.add_argument("--full", action="store_true", + help="download the whole results tree, not just the reports") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Fetch results. :return: the process exit code.""" + state_root = ctx.state_root_resolved() + run_id = runs.resolve_run_id(ctx.runner, state_root, ctx.args.run_id) + if not run_id: + ctx.console.error("no runs found under %s on %s" % (state_root, ctx.runner_host)) + return EXIT_USAGE + + paths = runs.RunPaths(state_root, run_id) + dest = Path(expand_path(ctx.args.dest)) / run_id + + if ctx.dry_run: + ctx.console.out("would download %s -> %s (%s), excluding %s" + % (paths.results_dir, dest, + "everything" if ctx.args.full else ", ".join(DEFAULT_FILES), + ", ".join(ALWAYS_EXCLUDED))) + return EXIT_OK + + dest.mkdir(parents=True, exist_ok=True) + # Staged inside the run directory rather than /tmp: it is known to exist, known to + # be writable by this account, and disappears with the run. + staged = paths.path(".fetch-%s.tar.gz" % uuid.uuid4().hex[:8]) + + script = _archive_script(paths, staged, full=ctx.args.full) + result = ctx.runner.run_script(script, check=False) + if not result.ok: + ctx.console.error("could not archive results on the runner: %s" + % (result.stderr.strip() or result.stdout.strip())) + return EXIT_USAGE + + with tempfile.TemporaryDirectory() as tmp: + local_archive = Path(tmp) / "results.tar.gz" + ctx.runner.download(staged, local_archive) + ctx.runner.run(["rm", "-f", "--", staged], check=False) + with tarfile.open(local_archive, "r:gz") as tar: + members = [m for m in tar.getmembers() + if posixpath.basename(m.name) not in ALWAYS_EXCLUDED] + _safe_extract(tar, members, dest) + + ctx.runner.download(paths.log_file, dest / "ducktape.log") + + ctx.console.out("results: %s" % dest.resolve()) + for report in sorted(dest.rglob("report.*")): + ctx.console.out(" %s" % report.relative_to(dest).as_posix()) + return EXIT_OK + + +def _archive_script(paths, staged, *, full): + """ + Build the tar command that stages the results on the runner. + + Archiving first and downloading one file keeps the transport free of binary + streaming, and makes the exclusion of globals.json explicit at both ends. + """ + excludes = " ".join("--exclude=%s" % shlex.quote(name) for name in ALWAYS_EXCLUDED) + # tar treats an output path containing a colon as host:path, so cd into the results + # directory and write the archive through a relative name. + out = posixpath.join("..", posixpath.basename(staged)) + common = ('set -eu\nroot=%s\n' + '[ -d "$root" ] || { echo "no results directory at $root" >&2; exit 1; }\n' + 'cd "$root"\n' % shlex.quote(paths.results_dir)) + + if full: + return common + 'tar -czf %s %s .\n' % (shlex.quote(out), excludes) + + # tar has no --include, so select the report files with find. + names = " -o ".join("-name %s" % shlex.quote(name) for name in DEFAULT_FILES) + return common + ('find . \\( %s \\) -type f -print0 | tar -czf %s %s --null -T -\n' + % (names, shlex.quote(out), excludes)) + + +def _safe_extract(tar, members, dest): + """Extract ``members`` under ``dest``, refusing anything that escapes it.""" + dest = Path(dest).resolve() + safe = [] + for member in members: + target = (dest / member.name).resolve() + if dest == target or dest in target.parents: + safe.append(member) + tar.extractall(str(dest), members=safe) # noqa: S202 - members filtered above diff --git a/modules/ducktests/tests/ducktests_remote/commands/keys.py b/modules/ducktests/tests/ducktests_remote/commands/keys.py new file mode 100644 index 0000000000000..06fb420e8d755 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/keys.py @@ -0,0 +1,135 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``keys push`` - install the runner's identity and authorise it on every worker.""" + +import argparse +import shlex +from pathlib import Path + +from ducktests_remote.cli import EXIT_OK, EXIT_TRANSPORT, EXIT_USAGE +from ducktests_remote.config import ConfigError, expand_path +from ducktests_remote.fanout import (CHANGED, HostResult, OK, any_failed, fanout, + render_table, summarise) +from ducktests_remote.transport import run_local + + +def register(subparsers, common): + """Wire up the ``keys`` subcommand.""" + parser = subparsers.add_parser( + "keys", parents=[common], help="manage the ssh key ducktape uses", + description="""Install the private key on the runner and authorise the matching +public key on every worker. + +Why this exists: ducktape connects from the runner to the workers using the identity +named in the cluster file, and a run launched with --detach outlives the coordinator's +ssh session. Agent forwarding dies with that session, so an agent-only setup produces a +run that authenticates for the first few minutes and then fails on every subsequent +connection. A real key file on the runner is the only arrangement that survives.""", + formatter_class=_RawHelp) + sub = parser.add_subparsers(dest="keys_command", metavar="") + push = sub.add_parser("push", parents=[common], + help="install the identity on the runner and authorise it on " + "the workers") + push.add_argument("--identity", metavar="PATH", + help="coordinator-side private key (default: cluster.identity_file, " + "or a generated one)") + push.add_argument("--generate", action="store_true", + help="generate a new keypair when the identity does not exist") + push.add_argument("-n", "--num-nodes", type=int, default=None, + help="only authorise on the first N inventory hosts") + push.set_defaults(handler=execute_push) + parser.set_defaults(handler=_no_action) + + +class _RawHelp(argparse.RawDescriptionHelpFormatter): + """Keeps the explanation in the help text readable.""" + + +def _no_action(ctx): + ctx.console.error("keys: expected an action, e.g. `ducktests-remote keys push`") + return EXIT_USAGE + + +def execute_push(ctx): + """Install and authorise the key. :return: the process exit code.""" + console = ctx.console + identity = Path(expand_path(ctx.args.identity or ctx.identity_file or "~/.ssh/id_rsa")) + public = Path(str(identity) + ".pub") + + if not identity.is_file(): + if not ctx.args.generate: + raise ConfigError( + "%s does not exist on this machine. Pass --generate to create a keypair, " + "or --identity to point at an existing one." % identity) + if ctx.dry_run: + console.out("[dry-run] would generate a keypair at %s" % identity) + else: + identity.parent.mkdir(parents=True, exist_ok=True) + run_local(["ssh-keygen", "-m", "PEM", "-q", "-t", "rsa", "-N", "", + "-f", str(identity)], check=True) + console.out("generated %s" % identity) + + if not public.is_file() and not ctx.dry_run: + raise ConfigError("public key %s not found next to the private key" % public) + + pubkey = public.read_text(encoding="utf-8").strip() if public.is_file() else "" + + runner_target = ctx.runner.expand(ctx.identity_file or "~/.ssh/id_rsa") + console.heading("RUNNER %s" % ctx.runner_host) + if ctx.dry_run: + console.out("[dry-run] would install %s -> %s:%s (mode 0600)" + % (identity, ctx.runner_host, runner_target)) + else: + ctx.runner.mkdirs(_parent(runner_target), mode=0o700) + ctx.runner.upload(identity, runner_target, mode=0o600) + ctx.runner.upload(public, runner_target + ".pub", mode=0o644) + console.out("installed %s (mode 0600)" % runner_target) + + console.heading("WORKERS") + results = fanout(ctx.all_nodes, lambda node: _authorize(ctx, node, pubkey), + jobs=ctx.jobs, fail_fast=getattr(ctx.args, "fail_fast", False)) + console.out(render_table(results, verbose=console.verbose)) + console.out(summarise(results)) + return EXIT_TRANSPORT if any_failed(results) else EXIT_OK + + +def _authorize(ctx, node, pubkey): + if ctx.dry_run: + return HostResult(node.host, OK, "would authorise the key for %s" % node.user) + script = """set -eu +key=%s +mkdir -p ~/.ssh +chmod 700 ~/.ssh +touch ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys +if grep -qxF "$key" ~/.ssh/authorized_keys; then + echo present +else + printf '%%s\\n' "$key" >> ~/.ssh/authorized_keys + echo added +fi +""" % shlex.quote(pubkey) + result = ctx.worker(node).run_script(script, check=False) + if not result.ok: + return HostResult(node.host, "failed", "could not write authorized_keys", + detail=result.stderr.strip()) + added = result.out == "added" + return HostResult(node.host, CHANGED if added else OK, + "authorised" if added else "already authorised") + + +def _parent(path): + return path.rsplit("/", 1)[0] or "/" diff --git a/modules/ducktests/tests/ducktests_remote/commands/logs.py b/modules/ducktests/tests/ducktests_remote/commands/logs.py new file mode 100644 index 0000000000000..06c28cb6282a9 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/logs.py @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``logs`` - print or follow a run's ``ducktape.log`` from the runner.""" + +import sys +import time + +from ducktests_remote import runs +from ducktests_remote.cli import EXIT_OK, EXIT_USAGE + +POLL_SEC = 1.5 + + +def register(subparsers, common): + """Wire up the ``logs`` subcommand.""" + parser = subparsers.add_parser( + "logs", parents=[common], help="print or follow a run's log", + description="Stream ducktape.log from the runner. Ctrl-C stops following; it " + "never touches the run.") + parser.add_argument("run_id", nargs="?", help="run id (default: the most recent run)") + parser.add_argument("-f", "--follow", action="store_true", help="keep streaming") + parser.add_argument("-n", "--lines", type=int, default=200, + help="lines of history to print first (default 200)") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Print or follow the log. :return: the process exit code.""" + state_root = ctx.state_root_resolved() + run_id = runs.resolve_run_id(ctx.runner, state_root, ctx.args.run_id) + if not run_id: + ctx.console.error("no runs found under %s on %s" % (state_root, ctx.runner_host)) + return EXIT_USAGE + + paths = runs.RunPaths(state_root, run_id) + redact = ctx.console.redactor.redact + + history = ctx.runner.run(["tail", "-n", str(ctx.args.lines), paths.log_file], check=False) + if history.stdout: + sys.stdout.write(redact(history.stdout)) + sys.stdout.flush() + + if not ctx.args.follow: + return EXIT_OK + + offset = _size(ctx, paths) + 1 + try: + while True: + chunk = ctx.runner.run(["tail", "-c", "+%d" % offset, paths.log_file], check=False) + if chunk.stdout: + offset += len(chunk.stdout.encode("utf-8")) + sys.stdout.write(redact(chunk.stdout)) + sys.stdout.flush() + state = runs.read_state(ctx.runner, paths) + if state.exit_code is not None: + ctx.console.out("") + ctx.console.out("run %s %s (ducktape exit %s)" + % (run_id, state.state, state.exit_code)) + return EXIT_OK + time.sleep(POLL_SEC) + except KeyboardInterrupt: + ctx.console.out("") + ctx.console.out("Stopped following. The run is untouched; " + "`ducktests-remote stop %s` stops it." % run_id) + return EXIT_OK + + +def _size(ctx, paths): + result = ctx.runner.run(["wc", "-c", paths.log_file], check=False) + try: + return int(result.out.split()[0]) + except (IndexError, ValueError): + return 0 diff --git a/modules/ducktests/tests/ducktests_remote/commands/provision.py b/modules/ducktests/tests/ducktests_remote/commands/provision.py new file mode 100644 index 0000000000000..3f0573871c0c0 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/provision.py @@ -0,0 +1,663 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``provision`` - bring an unconfigured VM up to the state the Docker image guarantees. + +``modules/ducktests/tests/docker/Dockerfile`` is the source of truth for what that state +is; the package list lives in :data:`ducktests_remote.config.DOCKERFILE_PACKAGES` next to +a note saying so. This is not configuration management and must not grow into it: it +covers the specific package / directory / ssh-environment set the image installs, each +step idempotent and independently selectable. +""" + +import hashlib +import json +import os +import posixpath +import shlex +import tempfile +import threading +import uuid +from pathlib import Path + +from ducktests_remote import java +from ducktests_remote.cli import EXIT_OK, EXIT_PREFLIGHT, EXIT_TRANSPORT +from ducktests_remote.commands import deploy, doctor +from ducktests_remote.config import ConfigError +from ducktests_remote.fanout import (CHANGED, FAILED, HostResult, OK, SKIPPED, any_failed, + fanout, render_table, summarise) +from ducktests_remote.progress import NullProgress, build_progress +from ducktests_remote.transport import make_tarball + +STEPS = ("packages", "jdk", "python", "user", "ssh-env", "dirs", "hosts") + +HOSTS_BEGIN = "# BEGIN ducktests-remote" +HOSTS_END = "# END ducktests-remote" + +JAVA_MANIFEST_NAME = ".ducktests-java.json" + + +def register(subparsers, common): + """Wire up the ``provision`` subcommand.""" + parser = subparsers.add_parser( + "provision", parents=[common], + help="bring the workers up to the state the Docker image guarantees", + description="Idempotent, per-step preparation of the worker hosts. Run it with " + "--dry-run first: it prints the exact commands per host and changes " + "nothing. Steps: " + ", ".join(STEPS)) + parser.add_argument("--only", action="append", default=[], choices=STEPS, metavar="STEP", + help="run only these steps; repeatable") + parser.add_argument("--skip", action="append", default=[], choices=STEPS, metavar="STEP", + help="skip these steps; repeatable") + parser.add_argument("--sudo", action="store_true", + help="allow steps that need root to use `sudo -n`") + parser.add_argument("--install-jdk", action="store_true", + help="let the jdk step fall back to the distribution's own JDK " + "package when nothing else resolves; needs --sudo") + parser.add_argument("--java-home", metavar="PATH", + help="use exactly this JDK home on the workers (sets java.home)") + parser.add_argument("--java-major", type=int, default=None, metavar="N", + help="Java major version the tests need (sets java.major)") + parser.add_argument("--java-archive", metavar="PATH", + help="coordinator-side JDK tarball or directory to deliver to " + "hosts that have no matching JDK (sets java.archive)") + parser.add_argument("--force", action="store_true", + help="deliver the JDK again even when the host already has it") + parser.add_argument("--create-user", metavar="NAME", + help="create this account (step `user`); needs --sudo") + parser.add_argument("--authorize-key", metavar="PATH", + help="public key appended to the created account's authorized_keys") + parser.add_argument("--write-hosts", action="store_true", + help="write the inventory into /etc/hosts (step `hosts`); needs --sudo") + parser.add_argument("-n", "--num-nodes", type=int, default=None, + help="only provision the first N inventory hosts") + parser.add_argument("--json", action="store_true", help="machine-readable output") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Provision the workers. :return: the process exit code.""" + nodes = ctx.all_nodes + if not nodes: + raise ConfigError("cluster.nodes is empty; there is nothing to provision") + + selected = _selected_steps(ctx.args) + ctx.console.info("steps: %s" % ", ".join(selected)) + if ctx.dry_run: + ctx.console.info("--dry-run: printing commands only") + + all_results = {} + skipped_for_sudo = [] + for step in selected: + ctx.console.heading("STEP %s" % step) + if step in _PYTHON_DRIVEN: + # These two resolve a JDK per host before they can act, and `jdk` may have to + # upload one, so they are not expressible as a single canned script. + results = _PYTHON_DRIVEN[step](ctx, nodes) + else: + script, needs_sudo = _step_script(ctx, step, nodes) + if script is None: + ctx.console.info("nothing to do for this step") + continue + if needs_sudo and not ctx.args.sudo: + ctx.console.warn("step %r needs root; rerun with --sudo. Skipping it and " + "continuing with the rest." % step) + skipped_for_sudo.append(step) + continue + results = _run_step(ctx, nodes, step, script) + all_results[step] = results + ctx.console.out(render_table(results, verbose=ctx.console.verbose)) + ctx.console.out(summarise(results)) + + if ctx.args.json: + ctx.console.out(json.dumps( + {step: [{"host": r.host, "status": r.status, "message": r.message} for r in res] + for step, res in all_results.items()}, indent=2)) + + if skipped_for_sudo: + ctx.console.warn("skipped for lack of --sudo: %s" % ", ".join(skipped_for_sudo)) + + failed = any(any_failed(res) for res in all_results.values()) + + if not ctx.dry_run: + # Always finish with evidence rather than an assumption. + ctx.console.heading("VERIFYING") + checks, diagnoses = doctor.run_checks(ctx) + doctor.print_report(ctx, checks, diagnoses) + if doctor.has_failures(checks): + return EXIT_PREFLIGHT + + return EXIT_TRANSPORT if failed else EXIT_OK + + +def _selected_steps(args): + steps = list(args.only) if args.only else list(STEPS) + steps = [s for s in steps if s not in (args.skip or [])] + if "user" in steps and not args.create_user and not args.only: + # Most operators use their own existing account; creating one is an escape hatch. + steps.remove("user") + if "hosts" in steps and not args.write_hosts and not args.only: + steps.remove("hosts") + return steps + + +def _run_step(ctx, nodes, step, script): + def operation(node): + transport = ctx.worker(node) + if ctx.dry_run: + ctx.console.out("[dry-run] %s: step %s" % (node.host, step)) + ctx.console.detail(script) + return HostResult(node.host, SKIPPED, "dry-run") + result = transport.run_script(script, check=False) + text = result.stdout.strip() + if not result.ok: + return HostResult(node.host, FAILED, _summary_line(result), detail=text or + result.stderr.strip()) + status = CHANGED if "CHANGED" in text else OK + return HostResult(node.host, status, _summary_line(result), detail=text) + + return fanout(nodes, operation, jobs=ctx.jobs, + fail_fast=getattr(ctx.args, "fail_fast", False)) + + +def _summary_line(result): + lines = [ln for ln in (result.stdout or "").splitlines() if ln.strip()] + if lines: + return lines[-1][:100] + return (result.stderr or "").strip().splitlines()[-1][:100] if result.stderr else "" + + +def _step_script(ctx, step, nodes): + """:return: ``(script, needs_sudo)`` for one step, or ``(None, False)`` when inert.""" + if step == "packages": + return _packages_script(ctx), True + if step == "python": + return _python_script(ctx), False + if step == "user": + if not ctx.args.create_user: + return None, False + return _user_script(ctx), True + if step == "dirs": + return _dirs_script(ctx), True + if step == "hosts": + if not ctx.args.write_hosts: + return None, False + return _hosts_script(ctx, nodes), True + raise ConfigError("unknown provision step %r" % step) + + +def _packages_script(ctx): + packages = ctx.config["provision"]["packages"] + return """set -u +missing="" +for p in %(pkgs)s; do + if command -v dpkg-query >/dev/null 2>&1; then + dpkg-query -W -f='${Status}' "$p" 2>/dev/null | grep -q "install ok installed" \\ + || missing="$missing $p" + elif command -v rpm >/dev/null 2>&1; then + rpm -q "$p" >/dev/null 2>&1 || missing="$missing $p" + else + echo "unsupported package manager: neither dpkg nor rpm found" >&2 + exit 2 + fi +done +if [ -z "$missing" ]; then echo "all %(count)d packages present"; exit 0; fi +echo "CHANGED installing:$missing" +if command -v apt-get >/dev/null 2>&1; then + sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq + sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $missing +elif command -v dnf >/dev/null 2>&1; then + sudo -n dnf install -y -q $missing +elif command -v yum >/dev/null 2>&1; then + sudo -n yum install -y -q $missing +else + echo "unsupported package manager: no apt-get, dnf or yum" >&2 + exit 2 +fi +echo "CHANGED installed:$missing" +""" % {"pkgs": " ".join(shlex.quote(p) for p in packages), "count": len(packages)} + + +def _run_jdk_step(ctx, nodes): + """ + Put a JDK of the requested major on every worker, and say which one it is. + + Rungs 1-3 of the ladder are discovery and cost one round trip per host; only the hosts + that come back empty reach rung 4 (deliver ``java.archive``) or rung 5 (fail with the + list of JDKs that *were* found). Nothing is uploaded to a host that already has what + it needs. + """ + cfg = java.config_of(ctx) + plan = java.archive_plan(cfg.archive, cfg.name) if cfg.archive else None + if plan: + target = java.target_dir(cfg, plan) + ctx.console.info("java: %s (%s) available for delivery to %s" + % (plan.path, deploy.human(plan.bytes), target)) + if len(nodes) > 3 and plan.bytes > 200 * 1024 * 1024: + ctx.console.warn("that is up to %s over the wire from this machine, if every " + "host turns out to need it." + % deploy.human(plan.bytes * len(nodes))) + + script = java.discovery_script(cfg) + payload = JdkPayload(plan) if plan else None + + # Only a delivery is worth watching; the probe-only case is one round trip per host. + progress = build_progress(ctx, nodes) if payload else NullProgress() + + def operation(node): + if ctx.dry_run: + ctx.console.out("[dry-run] %s: probe for a Java %s JDK" + % (node.host, cfg.major or "any")) + ctx.console.detail(script) + return HostResult(node.host, SKIPPED, "dry-run") + try: + return _jdk_on_host(ctx, node, cfg, payload, script, progress) + finally: + progress.done(node.host) + + try: + with progress: + return fanout(nodes, operation, jobs=ctx.jobs, + fail_fast=getattr(ctx.args, "fail_fast", False)) + finally: + if payload: + payload.close() + + +class JdkPayload: + """ + The bytes to upload for one :class:`~ducktests_remote.java.ArchivePlan`, prepared once. + + A tarball is uploaded as it is, so there is nothing to prepare. A directory has to be + packed first, and packing it inside the per-host operation meant gzipping the same JDK + once per worker, in parallel, on the coordinator. Both the tarball and the manifest + are therefore built on first use and shared by every host after that. + """ + + def __init__(self, plan): + self.plan = plan + self._lock = threading.Lock() + self._tmp = None + self._archive = None + self._manifest = None + + def archive(self): + """:return: the coordinator-side path of the tarball to upload.""" + if self.plan.kind == "tar": + return self.plan.path + with self._lock: + if self._archive is None: + self._tmp = tempfile.TemporaryDirectory() + packed = Path(self._tmp.name) / "jdk.tar.gz" + make_tarball(self.plan.path, packed) + self._archive = packed + return self._archive + + def manifest(self): + """:return: the manifest that decides whether a host already has this JDK.""" + with self._lock: + if self._manifest is None: + self._manifest = (deploy.build_manifest(self.plan.path) + if self.plan.kind == "dir" else _tar_manifest(self.plan)) + return self._manifest + + def strip(self): + """:return: ``--strip-components`` depth; a packed directory is already flat.""" + return 0 if self.plan.kind == "dir" else self.plan.strip + + def tar_flag(self): + """:return: the ``tar`` decompression flag; a packed directory is always gzip.""" + return "z" if self.plan.kind == "dir" else self.plan.tar_flag + + def close(self): + """Remove the packed tarball, if one was made.""" + if self._tmp is not None: + self._tmp.cleanup() + self._tmp = None + self._archive = None + + +def _jdk_on_host(ctx, node, cfg, payload, script, progress=None): + progress = progress or NullProgress() + transport = ctx.worker(node) + progress.phase(node.host, "probing") + probe = transport.run_script(script, check=False) + if not probe.ok: + return HostResult(node.host, FAILED, "could not probe for a JDK", + detail=probe.stderr.strip()) + + res = java.parse_probe(node.host, probe.stdout, cfg) + if res.selected: + return HostResult(node.host, OK, res.summary(), detail=probe.stdout.strip()) + + if cfg.home: + # An explicit java.home that is not there is a configuration error, not something + # to paper over by installing a different JDK. + return HostResult(node.host, FAILED, + "java.home %s has no usable bin/java on this host" % cfg.home, + detail=_found(res)) + + if payload is not None: + return _deliver_jdk(ctx, node, cfg, payload, progress) + + if ctx.args.install_jdk: + return _install_jdk(ctx, node, cfg) + + return HostResult(node.host, FAILED, + "no Java %s here; set java.archive to deliver one, java.home to " + "name an existing one, or pass --install-jdk" % cfg.major, + detail=_found(res)) + + +def _found(res): + if not res.candidates: + return "no JDK found at all" + return "found: " + ", ".join("%s (Java %s)" % (home, major) + for home, major, _ in res.candidates) + + +def _deliver_jdk(ctx, node, cfg, payload, progress=None): + """ + Copy the JDK to one worker, reusing ``deploy``'s staging and atomic swap. + + A half-extracted JDK that looks present is exactly as bad as a half-extracted + distribution, which is why this does not extract in place. + """ + progress = progress or NullProgress() + plan = payload.plan + transport = ctx.worker(node) + target = java.target_dir(cfg, plan) + manifest = payload.manifest() + + if not ctx.args.force: + existing = transport.read_file(posixpath.join(target, JAVA_MANIFEST_NAME)) + if existing: + try: + if json.loads(existing).get("hash") == manifest["hash"]: + return HostResult(node.host, OK, "%s already delivered" % target) + except ValueError: + pass + + install_root = posixpath.dirname(target) + writable = transport.run(["test", "-w", install_root], check=False).ok + if not writable and not ctx.args.sudo: + return HostResult(node.host, FAILED, + "%s is not writable by %s and --sudo was not passed" + % (install_root, node.user or "this account")) + + sweep = deploy.staging_prefix(install_root, posixpath.basename(target)) + staging = "%s%s" % (sweep, uuid.uuid4().hex[:8]) + swapped = False + + try: + transport.run_script(deploy.prepare_script(staging, ctx.args.sudo, sweep=sweep)).check() + + remote = "%s/.payload.tar" % staging + archive = payload.archive() + progress.phase(node.host, "sending") + total = os.path.getsize(archive) + transport.upload_watched( + archive, remote, + on_bytes=(lambda sent: progress.sent(node.host, sent, total)) + if progress.watching else None) + progress.phase(node.host, "extracting") + transport.run_script( + "set -eu\ntar -x%sf %s -C %s%s\nrm -f -- %s\n" + % (payload.tar_flag(), shlex.quote(remote), shlex.quote(staging), + " --strip-components=%d" % payload.strip() if payload.strip() else "", + shlex.quote(remote))).check() + + check = transport.run(["test", "-x", "%s/bin/java" % staging], check=False) + if not check.ok: + return HostResult(node.host, FAILED, + "the delivered archive has no bin/java under %s" % staging) + + progress.phase(node.host, "swapping") + transport.write_file(json.dumps(manifest, indent=2, sort_keys=True), + posixpath.join(staging, JAVA_MANIFEST_NAME)) + transport.run_script(deploy.swap_script(staging, target, ctx.args.sudo, None)).check() + swapped = True + finally: + # Same rule as deploy: a staging tree that is not swapped in is dead weight, and + # its dot-prefixed name means nobody would ever notice it accumulating. + if not swapped: + transport.run_script(deploy.discard_script(staging, ctx.args.sudo), check=False) + + return HostResult(node.host, CHANGED, "delivered %s to %s" + % (deploy.human(plan.bytes), target)) + + +def _tar_manifest(plan): + """:return: a manifest keyed on the archive itself, so redelivery is skipped.""" + stat = plan.path.stat() + digest = hashlib.sha256( + ("%s\0%d\0%d" % (plan.path.name, stat.st_size, int(stat.st_mtime))).encode("utf-8") + ).hexdigest() + return {"hash": digest, "source": plan.path.name, "bytes": plan.bytes, + "mode": "archive size+mtime"} + + +def _install_jdk(ctx, node, cfg): + """Last rung: the distribution's own JDK package. Opt-in, and it needs root.""" + if not ctx.args.sudo: + return HostResult(node.host, FAILED, + "--install-jdk needs --sudo as well") + script = """set -u +major=%(major)d +if command -v apt-get >/dev/null 2>&1; then + echo "CHANGED installing openjdk-$major-jdk" + sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq + sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "openjdk-$major-jdk" +elif command -v dnf >/dev/null 2>&1; then + echo "CHANGED installing java-$major-openjdk-devel" + sudo -n dnf install -y -q "java-$major-openjdk-devel" +elif command -v yum >/dev/null 2>&1; then + echo "CHANGED installing java-$major-openjdk-devel" + sudo -n yum install -y -q "java-$major-openjdk-devel" +else + echo "no supported package manager for an automatic JDK install" >&2 + exit 2 +fi +""" % {"major": int(cfg.major or 0)} + result = ctx.worker(node).run_script(script, check=False) + if not result.ok: + return HostResult(node.host, FAILED, "could not install a JDK", + detail=(result.stderr or result.stdout).strip()) + return HostResult(node.host, CHANGED, "installed the distribution's Java %s package" + % cfg.major) + + +def _python_script(ctx): + """ + Ensure a usable Python on the host. + + The workers do not need Python: ducktape drives them over plain SSH and runs no + Python there. Only the runner needs the venv, and ``run`` creates it on demand. + This step therefore verifies rather than installs, and says so. + """ + return """set -u +if command -v python3 >/dev/null 2>&1; then + echo "ok: $(python3 --version 2>&1)" +else + echo "python3 missing. Workers do not need it - ducktape drives them over plain ssh -" + echo "but nothing here will install it either." +fi +exit 0 +""" + + +def _user_script(ctx): + name = ctx.args.create_user + key = "" + if ctx.args.authorize_key: + with open(ctx.args.authorize_key, "r", encoding="utf-8") as handle: + key = handle.read().strip() + return """set -eu +user=%(user)s +key=%(key)s +if id -u "$user" >/dev/null 2>&1; then + echo "account $user already exists" +else + echo "CHANGED creating $user" + sudo -n useradd -m -s /bin/bash "$user" +fi +if [ -n "$key" ]; then + home=$(getent passwd "$user" | cut -d: -f6) + sudo -n mkdir -p "$home/.ssh" + if sudo -n grep -qxF "$key" "$home/.ssh/authorized_keys" 2>/dev/null; then + echo "key already authorised" + else + echo "CHANGED authorising key" + printf '%%s\\n' "$key" | sudo -n tee -a "$home/.ssh/authorized_keys" >/dev/null + fi + sudo -n chown -R "$user" "$home/.ssh" + sudo -n chmod 700 "$home/.ssh" + sudo -n chmod 600 "$home/.ssh/authorized_keys" +fi +echo done +""" % {"user": shlex.quote(name), "key": shlex.quote(key)} + + +def _run_ssh_env_step(ctx, nodes): + """ + Make the selected JDK the one a *non-interactive* ssh session gets. + + The step that is easiest to forget and hardest to diagnose. ducktape runs every + command over non-interactive ssh, where ``~/.profile`` is not sourced, so a ``java`` + that works when you log in by hand is simply absent during a test run and the failure + surfaces as an unrelated timeout. + + The JDK is resolved with the same ladder the ``jdk`` step uses - not with whatever + ``java`` happens to be first on PATH - so ``provision --only ssh-env`` on its own + still installs the JDK the operator asked for. Both ``~/.ssh/environment`` and + ``~/.bashrc`` are written, and neither is trusted: the step ends by opening a fresh + connection and reporting the JDK that one actually gets. + """ + cfg = java.config_of(ctx) + discover = java.discovery_script(cfg) + extra = ctx.config["provision"].get("ssh_env_path_extra") or [] + + def operation(node): + if ctx.dry_run: + ctx.console.out("[dry-run] %s: resolve the JDK, then write ~/.ssh/environment%s" + % (node.host, " and ~/.bashrc" if cfg.bashrc else "")) + ctx.console.detail(discover) + return HostResult(node.host, SKIPPED, "dry-run") + + transport = ctx.worker(node) + probe = transport.run_script(discover, check=False) + res = java.parse_probe(node.host, probe.stdout, cfg) + if not res.selected: + return HostResult(node.host, FAILED, + "no Java %s to point at; run `provision --only jdk` first" + % (cfg.major or "JDK"), detail=_found(res)) + + written = transport.run_script(java.env_script(cfg, res.home, extra), check=False) + if not written.ok: + return HostResult(node.host, FAILED, "could not write the environment files", + detail=(written.stderr or written.stdout).strip()) + + return _verify_ssh_env(ctx, node, cfg, res, written) + + return fanout(nodes, operation, jobs=ctx.jobs, + fail_fast=getattr(ctx.args, "fail_fast", False)) + + +def _verify_ssh_env(ctx, node, cfg, res, written): + """Ask a fresh session what it gets. That answer, not the edit, is the result.""" + check = java.parse_probe(node.host, + ctx.worker(node).run_script(java.verify_script(), check=False).stdout, + cfg) + detail = "\n".join(part for part in (written.stdout.strip(), + "verified: %s" % (check.path_java or "no java")) if part) + if not check.path_matches(cfg.major): + return HostResult(node.host, FAILED, + "a fresh non-interactive session still gets %s, not Java %s. " + "sshd may ignore ~/.ssh/environment and the login shell may not " + "be bash; set java.home on a PATH the site already provides." + % (check.path_version or "no java", cfg.major), + detail=detail) + status = CHANGED if "CHANGED" in written.stdout else OK + return HostResult(node.host, status, "non-interactive java is %s from %s" + % (check.path_version or "unknown", res.home), detail=detail) + + +def _dirs_script(ctx): + """ + Create the directories the services write into. + + ``ignitetest``'s ``PathAware`` builds every path under ``persistent_root``, default + ``/mnt/service`` (services/utils/path.py), and the Dockerfile chowns ``/mnt``, + ``/var/log`` and ``/opt`` to the test account. + """ + dirs = list(ctx.config["provision"]["dirs"]) + [ctx.cluster_cfg.get("install_root", "/opt")] + owner = ctx.cluster_cfg.get("user") + return """set -u +changed=0 +for d in %(dirs)s; do + if [ -d "$d" ]; then + [ -w "$d" ] || { sudo -n chown -R %(owner)s "$d" && changed=1; } + else + sudo -n mkdir -p "$d" && sudo -n chown -R %(owner)s "$d" && changed=1 + fi +done +if [ "$changed" -eq 1 ]; then echo "CHANGED prepared %(count)d directories" +else echo "%(count)d directories already usable"; fi +""" % {"dirs": " ".join(shlex.quote(d) for d in dirs), + "owner": shlex.quote(str(owner)), "count": len(dirs)} + + +def _hosts_script(ctx, nodes): + """ + Write the inventory into ``/etc/hosts`` between explicit markers. + + Only the block between the markers is ever rewritten. This is the escape hatch for + clusters whose DNS does not resolve node-to-node, mirroring what ``ducker_up`` does + for the Docker network; it is not a substitute for working DNS. + """ + entries = [] + for node in nodes: + entries.append("%s %s" % (node.ip or node.host, node.host)) + block = "\n".join(entries) + return """set -eu +block=%(block)s +tmp=$(mktemp) +awk 'BEGIN{skip=0} + /^# BEGIN ducktests-remote$/{skip=1; next} + /^# END ducktests-remote$/{skip=0; next} + skip==0{print}' /etc/hosts > "$tmp" +{ + printf '%%s\\n' "%(begin)s" + printf '%%s\\n' "$block" + printf '%%s\\n' "%(end)s" +} >> "$tmp" +if cmp -s "$tmp" /etc/hosts; then + rm -f "$tmp"; echo "/etc/hosts already up to date" +else + sudo -n cp "$tmp" /etc/hosts + rm -f "$tmp" + echo "CHANGED rewrote the ducktests-remote block in /etc/hosts" +fi +""" % {"block": shlex.quote(block), "begin": HOSTS_BEGIN, "end": HOSTS_END} + + +def venv_bin(venv, name): + """:return: ``/bin/``.""" + return posixpath.join(venv, "bin", name) + + +# Steps that need per-host decisions in Python rather than one canned script. +_PYTHON_DRIVEN = {"jdk": _run_jdk_step, "ssh-env": _run_ssh_env_step} diff --git a/modules/ducktests/tests/ducktests_remote/commands/run.py b/modules/ducktests/tests/ducktests_remote/commands/run.py new file mode 100644 index 0000000000000..6b64d32ff5713 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/run.py @@ -0,0 +1,647 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``run`` - compose the artifacts, launch ducktape detached, and follow its log.""" + +import json +import os +import platform +import posixpath +import shlex +import socket +import sys +import time +from pathlib import Path + +from ducktests_remote import (__version__, cluster as cluster_mod, globals_builder, pipconf, + runs) +from ducktests_remote.cli import (EXIT_OK, EXIT_PREFLIGHT, EXIT_TESTS_FAILED, Console) +from ducktests_remote.commands import doctor +from ducktests_remote.config import ConfigError, expand_path +from ducktests_remote.progress import build_progress + +DEFAULT_EXCLUDES = [".git", "target", "results", "__pycache__", "*.pyc", ".idea", + "venv", ".venv", "*.egg-info", ".tox", ".pytest_cache"] + +IGNITE_IGNORE_FILE = ".ducktestsignore" + +# Where the ducktests live inside an Ignite checkout, and the file that proves it is one. +TESTS_SUBDIR = ("modules", "ducktests", "tests") +CHECKOUT_MARKER = ("docker", "requirements.txt") + +FOLLOW_POLL_SEC = 1.5 +DOUBLE_INTERRUPT_SEC = 3.0 + +# run.sh exits with this when the requirements file it would install from is absent. +MISSING_REQUIREMENTS_EXIT = 3 + + +def register(subparsers, common): + """Wire up the ``run`` subcommand.""" + parser = subparsers.add_parser( + "run", parents=[common], help="launch a ducktape run on the cluster", + description="Compose the configuration, generate cluster.json / globals.json / " + "run.sh on the runner, and launch ducktape detached.") + parser.add_argument("test_paths", nargs="*", metavar="TEST_PATH", + help="test paths, relative to modules/ducktests/tests as in the " + "Docker flow (./ignitetest/tests/smoke_test.py::Cls.method); " + "paths relative to the checkout root also work") + parser.add_argument("-t", "--tc-path", action="append", default=[], metavar="PATH", + help="test path; repeatable, equivalent to a positional argument") + parser.add_argument("-g", "--global", action="append", default=[], dest="globals_kv", + metavar="KEY=VALUE", + help="dotted globals override, e.g. -g ssl.enabled=true") + parser.add_argument("-p", "--param", action="append", default=[], dest="params_kv", + metavar="KEY=VALUE", help="dotted --parameters override") + parser.add_argument("--globals-json", metavar="JSON", + help="raw globals base layer; paste an existing Jenkins blob here") + parser.add_argument("--globals-file", metavar="PATH", help="raw globals base layer from file") + parser.add_argument("--params-json", metavar="JSON", help="raw --parameters base layer") + parser.add_argument("--cluster-file", metavar="PATH", + help="use this coordinator-side cluster file verbatim instead of " + "generating one from the inventory") + parser.add_argument("-n", "--num-nodes", type=int, default=None, metavar="N", + help="use the first N inventory hosts (default: all)") + parser.add_argument("--source-root", metavar="PATH", + help="Ignite checkout synced to the runner (default: the checkout " + "containing the current directory)") + parser.add_argument("--no-sync", action="store_true", + help="assume the sources are already on the runner") + parser.add_argument("--exclude", action="append", default=[], metavar="PATTERN", + help="extra sync exclusion; repeatable") + parser.add_argument("--work-dir", metavar="PATH", + help="runner-side Ignite checkout to run from; ducktape itself " + "runs in its modules/ducktests/tests directory " + "(default: the synced source root)") + parser.add_argument("--install-sources", action="store_true", + help="pip install the synced sources into the runner venv. Not needed " + "for test discovery; ducktape puts the sources on sys.path itself") + parser.add_argument("--pip-index-url", metavar="URL", + help="package index for the runner venv, when PyPI is unreachable") + parser.add_argument("--pip-extra-index-url", action="append", default=None, metavar="URL", + help="additional index; repeatable, replaces the configured list") + parser.add_argument("--pip-trusted-host", action="append", default=None, metavar="HOST", + help="host whose certificate pip should not verify; repeatable") + parser.add_argument("--pip-timeout", type=int, default=None, metavar="SECONDS", + help="pip socket timeout, for a slow internal mirror") + parser.add_argument("--pip-cert", metavar="PATH", + help="CA bundle for the index, as a RUNNER-side path") + parser.add_argument("--repeat", type=int, default=None, metavar="N") + parser.add_argument("--max-parallel", type=int, default=None, metavar="N") + parser.add_argument("--test-runner-timeout", type=int, default=None, metavar="MS") + parser.add_argument("--results-root", metavar="PATH", + help="runner-side results root (default: /results)") + parser.add_argument("--skip-preflight", action="store_true", help="skip the doctor checks") + group = parser.add_mutually_exclusive_group() + group.add_argument("--follow", dest="follow", action="store_true", default=True, + help="stream the log until the run ends (default)") + group.add_argument("--detach", dest="follow", action="store_false", + help="print the run id and exit immediately") + parser.set_defaults(handler=execute) + + +def execute(ctx): # pylint: disable=too-many-return-statements + """Launch a run. :return: the process exit code.""" + args = ctx.args + console = ctx.console + + raw_test_paths = list(args.test_paths) + list(args.tc_path) + if not raw_test_paths: + raise ConfigError("no tests given; pass a path positionally or with -t/--tc-path") + + source_root = _source_root(ctx) + _check_source_root(ctx, source_root) + + composed, params = _compose_payloads(ctx) + + nodes = ctx.nodes + cluster_payload, cluster_source, cluster_text = _cluster_payload(ctx, nodes) + _warn_about_topology(ctx, nodes, raw_test_paths) + + if not args.skip_preflight and not ctx.dry_run: + console.heading("PREFLIGHT") + checks, diagnoses = doctor.run_checks(ctx, _versions(composed)) + doctor.print_report(ctx, checks, diagnoses) + if doctor.has_failures(checks): + console.error("preflight failed; fix the FAIL rows above or pass --skip-preflight") + return EXIT_PREFLIGHT + + run_id = runs.new_run_id(_coordinator_user()) + state_root = ctx.runner.expand(ctx.state_root) + paths = runs.RunPaths(state_root, run_id) + + work_dir = _work_dir(ctx, paths, source_root) + results_root = args.results_root or paths.results_dir + test_paths = _test_paths(ctx, raw_test_paths, source_root, work_dir) + + run_sh = runs.render_run_script( + version=__version__, timestamp=runs.utc_now_iso(), + author="%s@%s" % (_coordinator_user(), socket.gethostname()), + cwd=_tests_dir(work_dir), results_root=results_root, + cluster_file=paths.cluster_file, globals_file=paths.globals_file, + test_paths=test_paths, venv=_venv_path(ctx), + parameters_file=paths.parameters_file if params else None, + repeat=args.repeat, max_parallel=args.max_parallel, + test_runner_timeout=args.test_runner_timeout, + extra_args=getattr(args, "passthrough", [])) + + _print_artifacts(ctx, paths, cluster_payload, composed, params, run_sh, cluster_source) + + if ctx.dry_run: + console.out("") + console.info("--dry-run: nothing was created, uploaded or started.") + return EXIT_OK + + console.heading("PREPARING %s" % run_id) + ctx.runner.mkdirs(paths.run_dir) + ctx.runner.mkdirs(paths.results_dir) + + if not args.no_sync: + _sync_sources(ctx, source_root, paths) + _ensure_venv(ctx, work_dir) + if args.install_sources: + _install_sources(ctx, work_dir) + + ctx.runner.write_file(cluster_text or cluster_mod.dumps(cluster_payload), + paths.cluster_file) + ctx.runner.write_file(globals_builder.dumps(composed), paths.globals_file, mode=0o600) + if params: + ctx.runner.write_file(globals_builder.dumps(params), paths.parameters_file, mode=0o600) + ctx.runner.write_file(run_sh, paths.run_script, mode=0o755) + ctx.runner.write_file(runs.render_launch_script(paths), paths.launch_script, mode=0o755) + + runs.write_meta(ctx.runner, paths, + _meta(ctx, run_id, test_paths, nodes, work_dir, results_root)) + _update_latest_link(ctx, paths) + + pid = ctx.runner.run_script(runs.render_detach_script(paths)).check().out + console.out("") + console.out("run id : %s" % run_id) + console.out("runner : %s (pid %s)" % (ctx.runner_host, pid)) + console.out("run dir: %s" % paths.run_dir) + console.out("log : %s" % paths.log_file) + + if not args.follow: + _print_reattach(console, run_id) + return EXIT_OK + + return _follow(ctx, paths, run_id) + + +# -- composition ------------------------------------------------------------------- + + +def _compose_payloads(ctx): + args = ctx.args + redactor = ctx.console.redactor + + layers = [] + raw = globals_builder.load_raw_layer(args.globals_json, args.globals_file) + if raw is not None: + layers.append(("--globals-json/--globals-file", raw)) + layers.append(("config globals", ctx.config.get("globals") or {})) + composed, _ = globals_builder.build(layers, args.globals_kv, redactor=redactor) + + param_layers = [] + raw_params = globals_builder.load_raw_layer(args.params_json, None) + if raw_params is not None: + param_layers.append(("--params-json", raw_params)) + param_layers.append(("config parameters", ctx.config.get("parameters") or {})) + params, _ = globals_builder.build(param_layers, args.params_kv, redactor=redactor) + + return composed, params + + +def _cluster_payload(ctx, nodes): + """ + :return: ``(parsed, source, verbatim_text)``. + + ``--cluster-file`` is uploaded byte for byte - it is the migration path from a + hand-written file such as the current ``49_cluster.json``, and rewriting it would + silently drop anything the CLI does not model. It is still parsed here so that a + broken file fails now rather than inside ducktape. + """ + if ctx.args.cluster_file: + path = Path(expand_path(ctx.args.cluster_file)) + if not path.is_file(): + raise ConfigError("cluster file not found: %s" % path) + text = path.read_text(encoding="utf-8") + try: + parsed = json.loads(text) + except ValueError as ex: + raise ConfigError("%s is not valid JSON: %s" % (path, ex)) from ex + if not parsed.get("nodes"): + raise ConfigError("%s has no 'nodes' entry" % path) + return parsed, str(path), text + identity = ctx.runner.expand(ctx.identity_file) if ctx.identity_file else None + return cluster_mod.cluster_json(nodes, identity_file=identity), "inventory", None + + +def _warn_about_topology(ctx, nodes, test_paths): + console = ctx.console + if ctx.runner_host not in ("local", None): + if any(node.host == ctx.runner_host for node in nodes): + console.warn("the runner %s is also listed as a worker. The Docker flow reserves " + "ducker01 for ducktape; on a real cluster that is your call, but " + "ducktape will schedule Ignite nodes onto the machine it runs on." + % ctx.runner_host) + if len(nodes) < 3 and len(test_paths) > 0: + console.warn("only %d worker%s in the inventory. Most ignitetest suites declare " + "@cluster(num_nodes=...) well above that and will be skipped as " + "un-runnable." % (len(nodes), "" if len(nodes) == 1 else "s")) + + +def _versions(composed): + versions = composed.get("ignite_versions") + if isinstance(versions, str): + return [versions] + return list(versions or []) + + +# -- runner-side preparation -------------------------------------------------------- + + +def _source_root(ctx): + """ + :return: the Ignite checkout to sync, as a coordinator-side path. + + An explicit ``--source-root`` / ``run.source_root`` is taken as given. Otherwise the + current directory is walked upwards until a checkout is found, so the command works + from ``modules/ducktests/tests`` - where ``./docker/run_tests.sh`` is run from - as + well as from the repository root. + """ + configured = ctx.args.source_root or ctx.config["run"].get("source_root") + if configured: + return Path(expand_path(configured)).resolve() + cwd = Path(os.getcwd()).resolve() + for candidate in (cwd,) + tuple(cwd.parents): + if is_ignite_checkout(candidate): + return candidate + return cwd + + +def is_ignite_checkout(path): + """:return: True when ``path`` is the root of an Ignite source tree.""" + return Path(path).joinpath(*TESTS_SUBDIR).joinpath(*CHECKOUT_MARKER).is_file() + + +def _check_source_root(ctx, source_root): + """ + Reject a source root that is not an Ignite checkout, before anything is uploaded. + + Getting this wrong is easy - the state root and the tests directory are both plausible + places to stand - and every symptom of it appears much later, as a missing + requirements file or as ducktape finding no tests. + """ + if is_ignite_checkout(source_root): + return + if ctx.args.no_sync and not source_root.exists(): + return # a runner-side path that this machine cannot see; nothing to check + raise ConfigError( + "%s is not an Ignite checkout: it has no %s.\nThe whole source tree is synced to " + "the runner and ducktape is run from its %s directory, so this must be the " + "repository root (the directory that contains `modules/`). Run the command from " + "anywhere inside the checkout, or pass --source-root / set run.source_root." + % (source_root, posixpath.join(*TESTS_SUBDIR, *CHECKOUT_MARKER), + posixpath.join(*TESTS_SUBDIR))) + + +def _tests_dir(work_dir): + """:return: the runner-side directory ducktape runs from.""" + return posixpath.join(work_dir, *TESTS_SUBDIR) + + +def _test_paths(ctx, raw_paths, source_root, work_dir): + """ + Rewrite the given test paths into the form ducktape sees on the runner. + + ducktape's working directory is the tests directory, the same one + ``./docker/run_tests.sh`` uses, so ``./ignitetest/tests/smoke_test.py`` means here + what it means locally. A path is accepted in whatever form resolves on the + coordinator - relative to the current directory, to the tests directory, or to the + checkout root - and is rewritten from there. + """ + tests_local = source_root.joinpath(*TESTS_SUBDIR) + return [_one_test_path(ctx, raw, source_root, tests_local, work_dir) + for raw in raw_paths] + + +def _one_test_path(ctx, raw, source_root, tests_local, work_dir): + body, sep, suffix = raw.partition("::") + resolved = _resolve_test_file(body, source_root, tests_local) + + if resolved is None: + # Legitimate under --no-sync against a checkout this machine does not have, and + # ducktape gives a better message than a guess would. + ctx.console.warn("test path %s does not exist on this machine; passing it to " + "ducktape unchanged" % body) + return raw + + try: + return "./%s%s%s" % (resolved.relative_to(tests_local).as_posix(), sep, suffix) + except ValueError: + pass + + try: + rel = resolved.relative_to(source_root) + except ValueError: + ctx.console.warn("test path %s is outside the source root %s, so it is not synced; " + "it will only run if that exact path exists on the runner" + % (resolved, source_root)) + return raw + return "%s%s%s" % (posixpath.join(work_dir, rel.as_posix()), sep, suffix) + + +def _resolve_test_file(body, source_root, tests_local): + """:return: the coordinator-side file ``body`` names, or None if it names none.""" + expanded = Path(expand_path(body)) + if expanded.is_absolute(): + return expanded.resolve() if expanded.exists() else None + for base in (Path(os.getcwd()), tests_local, source_root): + candidate = base / expanded + if candidate.exists(): + return candidate.resolve() + return None + + +def _work_dir(ctx, paths, source_root): + configured = ctx.args.work_dir or ctx.config["run"].get("work_dir") + if configured: + return ctx.runner.expand(configured) + if ctx.args.no_sync: + # Nothing was uploaded, so the sources must already be where the operator says. + return ctx.runner.expand(str(source_root).replace(os.sep, "/")) + return paths.src_dir + + +def _excludes(ctx, source_root): + if ctx.args.exclude: + return list(ctx.args.exclude) + ignore_file = Path(source_root) / IGNITE_IGNORE_FILE + if ignore_file.is_file(): + return [line.strip() for line in ignore_file.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.strip().startswith("#")] + return list(DEFAULT_EXCLUDES) + list(ctx.config["run"].get("exclude") or []) + + +def _sync_sources(ctx, source_root, paths): + excludes = _excludes(ctx, source_root) + size_mb = _payload_size_mb(source_root, excludes) + limit = int(ctx.config["run"]["max_payload_mb"]) + ctx.console.info("syncing %s (%.1f MB) -> %s:%s" + % (source_root, size_mb, ctx.runner_host, paths.src_dir)) + if size_mb > limit: + raise ConfigError( + "source payload is %.1f MB, above the %d MB limit. That almost always means a " + "build directory leaked into the payload - check --exclude / %s. Distributions " + "belong in `ducktests-remote deploy`, never in the source sync." + % (size_mb, limit, IGNITE_IGNORE_FILE)) + total = int(size_mb * 1024 * 1024) + progress = build_progress(ctx, [ctx.runner_host]) + watcher = None + if progress.watching: + def watcher(sent, fraction): # noqa: F811 - reported only when something draws it + progress.sent(ctx.runner_host, sent, total, fraction=fraction) + with progress: + progress.phase(ctx.runner_host, "sending") + ctx.runner.upload_dir(source_root, paths.src_dir, excludes=excludes, + on_progress=watcher) + progress.done(ctx.runner_host) + + +def _payload_size_mb(source_root, excludes): + from ducktests_remote.transport import is_excluded # pylint: disable=import-outside-toplevel + total = 0 + root = Path(source_root) + for entry in root.rglob("*"): + if entry.is_dir() or entry.is_symlink(): + continue + if is_excluded(entry.relative_to(root), excludes): + continue + try: + total += entry.stat().st_size + except OSError: + continue + return total / (1024.0 * 1024.0) + + +def _venv_path(ctx): + """ + :return: the runner-side venv path, or None when ducktape is expected on PATH. + + The environment is controlled rather than assumed: an explicitly configured path is + used as given, otherwise the default under the state root is used and created when + it does not exist yet. + """ + configured = ctx.config["runner"].get("venv") + if configured: + return ctx.runner.expand(configured) + if not ctx.config["runner"].get("create_venv", True): + return None + return posixpath.join(ctx.runner.expand(ctx.state_root), "venv") + + +def _ensure_venv(ctx, work_dir): + """Create the runner venv and install the pinned requirements when it is missing.""" + venv = _venv_path(ctx) + if not venv: + return + python = ctx.config["runner"].get("python", "python3") + requirements = ctx.config["runner"].get("requirements") or posixpath.join( + _tests_dir(work_dir), *CHECKOUT_MARKER) + index_arg = pipconf.pip_args_str(ctx.config) + + script = """set -eu +venv=%(venv)s +req=%(req)s +if [ ! -x "$venv/bin/python3" ]; then + echo "creating venv at $venv" + %(python)s -m venv "$venv" + created=1 +fi +if ! "$venv/bin/python3" -c 'import ducktape' 2>/dev/null; then + if [ -f "$req" ]; then + echo "installing $req into $venv" + "$venv/bin/pip3" install --disable-pip-version-check %(index)s -r "$req" + else + echo "MISSING_REQUIREMENTS $req" >&2 + exit 3 + fi +fi +"$venv/bin/python3" -c 'import ducktape;print("ducktape " + ducktape.__version__)' +""" % {"venv": shlex.quote(venv), "req": shlex.quote(requirements), + "python": shlex.quote(python), "index": index_arg} + + result = ctx.runner.run_script(script, check=False) + if result.returncode == MISSING_REQUIREMENTS_EXIT: + # Nothing was installed and nothing was reached: the file simply is not there. + raise ConfigError( + "the runner venv at %s has no ducktape, and the requirements file to install " + "it from does not exist on the runner:\n %s\nThat is /%s, so it " + "normally means the synced tree is not an Ignite checkout. Check --source-root, " + "or set runner.requirements to a file the runner does have." + % (venv, requirements, posixpath.join(*TESTS_SUBDIR, *CHECKOUT_MARKER))) + if not result.ok: + raise ConfigError( + "could not prepare the runner venv at %s:\n%s\nInstalling %s from: %s.\nEither " + "point runner.venv at an existing environment, or set pip.index_url to an " + "index the runner can reach." + % (venv, (result.stderr or result.stdout).strip(), requirements, + pipconf.describe(ctx.config, ctx.console.redactor))) + ctx.console.info(result.out.splitlines()[-1] if result.out else "venv ready") + + +def _install_sources(ctx, work_dir): + venv = _venv_path(ctx) + pip = posixpath.join(venv, "bin", "pip3") if venv else "pip3" + tests_dir = _tests_dir(work_dir) + ctx.console.info("installing sources from %s" % tests_dir) + ctx.runner.run([pip, "install", "--disable-pip-version-check"] + + pipconf.pip_args(ctx.config) + ["-e", tests_dir]).check() + + +def _update_latest_link(ctx, paths): + ctx.runner.run_script("set -eu\nln -sfn %s %s\n" + % (shlex.quote(paths.run_dir), shlex.quote(paths.latest_link)), + check=False) + + +def _meta(ctx, run_id, test_paths, nodes, work_dir, results_root): + from ducktests_remote.config import redacted_summary # pylint: disable=import-outside-toplevel + return { + "run_id": run_id, + "cli_version": __version__, + "coordinator": {"user": _coordinator_user(), "host": socket.gethostname(), + "platform": platform.platform()}, + "runner": ctx.runner_host, + "cluster": ctx.cluster_cfg.get("name"), + "nodes": [n.host for n in nodes], + "test_paths": list(test_paths), + "work_dir": work_dir, + "results_root": results_root, + "started_at": runs.utc_now_iso(), + "started_epoch": time.time(), + "config": ctx.console.redactor.redact_structure(redacted_summary(ctx.config)), + } + + +# -- output ------------------------------------------------------------------------ + + +def _print_artifacts(ctx, paths, cluster_payload, composed, params, run_sh, cluster_source): + console = ctx.console + if not (ctx.dry_run or console.verbose): + return + console.heading("cluster.json (from %s) -> %s" % (cluster_source, paths.cluster_file)) + console.out(cluster_mod.dumps(cluster_payload).rstrip()) + console.heading("globals.json (redacted) -> %s [mode 0600]" % paths.globals_file) + console.out(globals_builder.dumps( + console.redactor.redact_structure(composed)).rstrip()) + if params: + console.heading("parameters.json -> %s" % paths.parameters_file) + console.out(globals_builder.dumps( + console.redactor.redact_structure(params)).rstrip()) + console.heading("run.sh -> %s" % paths.run_script) + console.out(run_sh.rstrip()) + + +def _print_reattach(console: Console, run_id): + console.out("") + console.out("The run continues on the runner. To come back to it:") + console.out(" ducktests-remote logs %s -f" % run_id) + console.out(" ducktests-remote status %s" % run_id) + console.out("To stop it and clean the workers:") + console.out(" ducktests-remote stop %s" % run_id) + + +# -- follow ------------------------------------------------------------------------ + + +def _follow(ctx, paths, run_id): + """ + Stream the runner's log until the process ends. + + Ctrl-C detaches; it never kills the run. Losing a three hour run to a reflex is not + a recoverable mistake, so stopping takes a deliberate second interrupt. + """ + console = ctx.console + console.heading("FOLLOWING %s (Ctrl-C detaches, it does not stop the run)" % run_id) + offset = 1 + last_interrupt = 0.0 + + while True: + try: + offset = _drain(ctx, paths, offset, console) + + state = runs.read_state(ctx.runner, paths) + if state.exit_code is not None: + # The process wrote its exit code between the two reads above, so drain + # once more or the last lines of the session report are lost. + _drain(ctx, paths, offset, console) + return _finish(ctx, paths, run_id, state) + time.sleep(FOLLOW_POLL_SEC) + except KeyboardInterrupt: + now = time.monotonic() + if now - last_interrupt < DOUBLE_INTERRUPT_SEC: + return _offer_stop(ctx, paths, run_id) + last_interrupt = now + console.out("") + console.out("Detaching. The run keeps going on %s." % ctx.runner_host) + _print_reattach(console, run_id) + console.out("") + console.out("Press Ctrl-C again within %ds to stop the run instead." + % int(DOUBLE_INTERRUPT_SEC)) + + +def _drain(ctx, paths, offset, console): + """Print whatever the log has gained since ``offset``. :return: the new offset.""" + chunk = ctx.runner.run(["tail", "-c", "+%d" % offset, paths.log_file], check=False) + if chunk.stdout: + offset += len(chunk.stdout.encode("utf-8")) + sys.stdout.write(console.redactor.redact(chunk.stdout)) + sys.stdout.flush() + return offset + + +def _offer_stop(ctx, paths, run_id): + console = ctx.console + console.out("") + if not sys.stdin.isatty(): + console.out("Detached (no terminal to confirm on). Run `ducktests-remote stop %s` " + "to stop it." % run_id) + return EXIT_OK + try: + answer = input("Stop run %s and clean the workers? [y/N] " % run_id) + except (EOFError, KeyboardInterrupt): + answer = "" + if answer.strip().lower() in ("y", "yes"): + from ducktests_remote.commands import stop # pylint: disable=import-outside-toplevel + return stop.stop_run(ctx, paths, timeout=60, kill=False, clean=True) + console.out("Left running. `ducktests-remote logs %s -f` to reattach." % run_id) + return EXIT_OK + + +def _finish(ctx, paths, run_id, state): + console = ctx.console + console.out("") + console.out("run %s %s after %s (ducktape exit %s)" + % (run_id, state.state, runs.format_duration(state.elapsed), state.exit_code)) + console.out("results: %s" % posixpath.join(paths.results_dir, "latest")) + console.out("fetch : ducktests-remote fetch %s" % run_id) + return EXIT_OK if state.exit_code == 0 else EXIT_TESTS_FAILED + + +def _coordinator_user(): + return os.environ.get("USER") or os.environ.get("USERNAME") or "unknown" diff --git a/modules/ducktests/tests/ducktests_remote/commands/status.py b/modules/ducktests/tests/ducktests_remote/commands/status.py new file mode 100644 index 0000000000000..453e085a35d3b --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/status.py @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``status`` - what a run is doing, or a table of every run on the runner.""" + +import json + +from ducktests_remote import runs +from ducktests_remote.cli import EXIT_OK, EXIT_USAGE + + +def register(subparsers, common): + """Wire up the ``status`` subcommand.""" + parser = subparsers.add_parser( + "status", parents=[common], help="show the state of a run", + description="With no run id, report the most recent run on the runner. Run state " + "lives on the runner, so this works from any coordinator.") + parser.add_argument("run_id", nargs="?", help="run id (default: the most recent run)") + parser.add_argument("--all", action="store_true", help="table of every run, newest first") + parser.add_argument("--json", action="store_true", help="machine-readable output") + parser.add_argument("-n", "--lines", type=int, default=15, + help="lines of log tail to show (default 15)") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Print run status. :return: the process exit code.""" + state_root = ctx.state_root_resolved() + + if ctx.args.all: + return _print_all(ctx, state_root) + + run_id = runs.resolve_run_id(ctx.runner, state_root, ctx.args.run_id) + if not run_id: + ctx.console.error("no runs found under %s on %s" % (state_root, ctx.runner_host)) + return EXIT_USAGE + + paths = runs.RunPaths(state_root, run_id) + state = runs.read_state(ctx.runner, paths) + + if ctx.args.json: + ctx.console.out(json.dumps(_as_dict(paths, state), indent=2, sort_keys=True)) + return EXIT_OK + + console = ctx.console + meta = state.meta + console.out("run id : %s" % state.run_id) + console.out("state : %s" % console.paint(state.state, _style(state.state))) + console.out("runner : %s" % ctx.runner_host) + console.out("pid : %s" % (state.pid if state.pid is not None else "-")) + console.out("started : %s" % (state.started_at or "-")) + console.out("elapsed : %s" % runs.format_duration(state.elapsed)) + console.out("exit code : %s" % (state.exit_code if state.exit_code is not None else "-")) + console.out("tests : %s" % ", ".join(meta.get("test_paths") or []) or "-") + console.out("cluster : %s (%d node(s))" % (meta.get("cluster") or "-", + len(meta.get("nodes") or []))) + console.out("run dir : %s" % paths.run_dir) + console.out("results : %s" % (meta.get("results_root") or paths.results_dir)) + + tail = ctx.runner.run(["tail", "-n", str(ctx.args.lines), paths.log_file], check=False) + if tail.stdout.strip(): + console.heading("last %d log lines" % ctx.args.lines) + console.out(tail.stdout.rstrip()) + return EXIT_OK + + +def _print_all(ctx, state_root): + run_ids = runs.list_run_ids(ctx.runner, state_root) + if not run_ids: + ctx.console.out("no runs under %s on %s" % (state_root, ctx.runner_host)) + return EXIT_OK + + rows = [] + for run_id in run_ids: + paths = runs.RunPaths(state_root, run_id) + state = runs.read_state(ctx.runner, paths) + rows.append((run_id, state.state, + runs.format_duration(state.elapsed), + "-" if state.exit_code is None else str(state.exit_code), + ", ".join(state.meta.get("test_paths") or [])[:60])) + + if ctx.args.json: + ctx.console.out(json.dumps( + [dict(zip(("run_id", "state", "elapsed", "exit_code", "tests"), row)) + for row in rows], indent=2)) + return EXIT_OK + + headers = ("RUN ID", "STATE", "ELAPSED", "EXIT", "TESTS") + widths = [max(len(str(r[i])) for r in ([headers] + rows)) for i in range(5)] + ctx.console.out(" ".join(h.ljust(widths[i]) for i, h in enumerate(headers)).rstrip()) + ctx.console.out(" ".join("-" * widths[i] for i in range(5))) + for row in rows: + ctx.console.out(" ".join(str(c).ljust(widths[i]) for i, c in enumerate(row)).rstrip()) + return EXIT_OK + + +def _as_dict(paths, state): + return {"run_id": state.run_id, "state": state.state, "pid": state.pid, + "exit_code": state.exit_code, "run_dir": paths.run_dir, + "elapsed_sec": state.elapsed, "meta": state.meta} + + +def _style(state): + return {runs.RUNNING: "blue", runs.FINISHED: "green", runs.FAILED: "red", + runs.STOPPED: "yellow"}.get(state, "dim") diff --git a/modules/ducktests/tests/ducktests_remote/commands/stop.py b/modules/ducktests/tests/ducktests_remote/commands/stop.py new file mode 100644 index 0000000000000..4cd43c2c96b3e --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/commands/stop.py @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``stop`` - terminate a detached run and, by default, clean the workers behind it.""" + +import shlex + +from ducktests_remote import runs +from ducktests_remote.cli import EXIT_OK, EXIT_USAGE + + +def register(subparsers, common): + """Wire up the ``stop`` subcommand.""" + parser = subparsers.add_parser( + "stop", parents=[common], help="stop a run and clean up after it", + description="SIGTERM the run's process group, wait, then optionally SIGKILL. " + "Cleans the workers afterwards unless --no-clean.") + parser.add_argument("run_id", nargs="?", help="run id (default: the most recent run)") + parser.add_argument("--kill", action="store_true", + help="SIGKILL survivors after the timeout") + parser.add_argument("--timeout", type=int, default=60, + help="seconds to wait for a graceful exit (default 60)") + parser.add_argument("--no-clean", dest="clean", action="store_false", default=True, + help="leave the workers alone; the JVMs will be someone else's problem") + parser.add_argument("-n", "--num-nodes", type=int, default=None, + help="limit the follow-up clean to the first N inventory hosts") + parser.set_defaults(handler=execute) + + +def execute(ctx): + """Stop a run. :return: the process exit code.""" + state_root = ctx.state_root_resolved() + run_id = runs.resolve_run_id(ctx.runner, state_root, ctx.args.run_id) + if not run_id: + ctx.console.error("no runs found under %s on %s" % (state_root, ctx.runner_host)) + return EXIT_USAGE + paths = runs.RunPaths(state_root, run_id) + return stop_run(ctx, paths, timeout=ctx.args.timeout, kill=ctx.args.kill, + clean=ctx.args.clean) + + +def stop_run(ctx, paths, *, timeout=60, kill=False, clean=True): + """Stop the run described by ``paths``. :return: the process exit code.""" + console = ctx.console + state = runs.read_state(ctx.runner, paths) + + if state.exit_code is not None: + console.info("run %s already ended (%s, exit %s)" + % (paths.run_id, state.state, state.exit_code)) + else: + console.info("stopping %s (pid %s, pgid %s)" % (paths.run_id, state.pid, state.pgid)) + result = ctx.runner.run_script(_stop_script(paths, timeout, kill), check=False) + console.detail(result.stdout.strip()) + if not result.ok and not ctx.dry_run: + console.warn("stop script exited %d: %s" % (result.returncode, + result.stderr.strip()[:200])) + + if clean: + console.heading("CLEANING WORKERS") + from ducktests_remote.commands import clean as clean_cmd # pylint: disable=C0415 + clean_cmd.clean_hosts(ctx, ctx.all_nodes, dry_run=ctx.dry_run) + + console.out("stopped %s" % paths.run_id) + return EXIT_OK + + +def _stop_script(paths, timeout, kill): + return """set -u +rd=%(rd)s +touch "$rd/stopped" +pid=$(cat "$rd/pid" 2>/dev/null || true) +pgid=$(cat "$rd/pgid" 2>/dev/null || true) +target="" +if [ -n "$pgid" ]; then target="-$pgid"; elif [ -n "$pid" ]; then target="$pid"; fi +if [ -z "$target" ]; then echo "no pid recorded"; exit 0; fi +kill -TERM -- "$target" 2>/dev/null || true +i=0 +while [ "$i" -lt %(timeout)d ]; do + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then sleep 1; i=$((i+1)); else break; fi +done +if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + if [ %(kill)d -eq 1 ]; then + echo "still alive after %(timeout)ds, sending SIGKILL" + kill -KILL -- "$target" 2>/dev/null || true + else + echo "still alive after %(timeout)ds; rerun with --kill" + fi +else + echo "exited within %(timeout)ds" +fi +[ -f "$rd/exit_code" ] || echo 143 > "$rd/exit_code" +""" % {"rd": shlex.quote(paths.run_dir), "timeout": int(timeout), "kill": 1 if kill else 0} diff --git a/modules/ducktests/tests/ducktests_remote/config.py b/modules/ducktests/tests/ducktests_remote/config.py new file mode 100644 index 0000000000000..b0a0aedbbccf2 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/config.py @@ -0,0 +1,480 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Configuration discovery, layering and validation. + +Layers, later overriding earlier, dicts deep-merged and lists replaced wholesale: + +1. built-in defaults +2. ``~/.ducktests-remote/config.yaml`` +3. ``--config FILE`` (repeatable, in order) +4. ``--profile NAME`` (repeatable, in order) +5. ``DTR_*`` environment variables +6. explicit command-line flags +7. ``-g KEY=VALUE`` dotted overrides (globals only, applied by :mod:`globals_builder`) +""" + +import copy +import difflib +import getpass +import json +import os +import re +from pathlib import Path + +import yaml + +USER_CONFIG_DIR = Path("~/.ducktests-remote").expanduser() +USER_CONFIG_FILE = USER_CONFIG_DIR / "config.yaml" +PACKAGE_EXAMPLES = Path(__file__).resolve().parent / "examples" + +# Pinned requirements the runner venv must satisfy. Read from the repo rather than +# hardcoded so the ducktape pin has exactly one source of truth. +REQUIREMENTS_RELPATH = "modules/ducktests/tests/docker/requirements.txt" + +# System utilities, derived from modules/ducktests/tests/docker/Dockerfile section 2. +# The Dockerfile is the source of truth; when the two disagree, the Dockerfile wins. +# Image-only entries (openssh-server, vim, mc, build-essential, cmake, libfuse-dev...) +# are deliberately not replicated: a real VM already has an sshd and no build toolchain +# is needed to *run* tests. +DOCKERFILE_PACKAGES = [ + "sudo", + "netcat-traditional", + "iptables", + "rsync", + "unzip", + "wget", + "curl", + "jq", + "coreutils", + "net-tools", +] + +# ignitetest shells out to `sudo iptables` only from IgniteAwareService.drop_network +# (services/utils/ignite_aware.py). Exactly two suites reach it. +SUDO_DEPENDENT_TESTS = ( + "ignitetest/tests/discovery_test.py", + "ignitetest/tests/cellular_affinity_test.py", +) + +# Where a JDK is looked for on a worker, in this order of preference within a major +# version. A search path may be a directory *of* JDK homes (/usr/lib/jvm) or a JDK home +# itself; both shapes are probed. +JAVA_SEARCH_PATHS = ["/opt", "/usr/lib/jvm", "/usr/java"] + +# modules/ducktests/tests/docker/Dockerfile pins `ARG jdk_version="eclipse-temurin:17"`. +# The Dockerfile is the source of truth here too; when the two disagree, it wins. +JAVA_MAJOR = 17 + +# ignitetest services launch these main classes; see services/ignite.py, +# services/ignite_app.py, services/utils/cdc/ignite_cdc.py and .../kafka_to_ignite.py. +IGNITE_MAIN_CLASSES = ( + "org.apache.ignite.startup.cmdline.CommandLineStartup", + "org.apache.ignite.startup.cmdline.CdcCommandLineStartup", + "org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService", + "org.apache.ignite.cdc.kafka.KafkaToIgniteCommandLineStartup", +) + +DEFAULTS = { + "profiles_dir": str(USER_CONFIG_DIR / "profiles"), + "cluster": { + "name": "default", + "identity_file": "~/.ssh/id_rsa", + "user": None, # resolved to the coordinator's $USER, never to "ducker" + "port": 22, + "install_root": "/opt", + "runner": "local", + "state_root": "~/.ducktests-remote", + "nodes": [], + "extra_hosts": [], + }, + "runner": { + # Deterministic environment: use venv when given, else the default under the + # state root, creating and populating it when missing. + "venv": None, + "python": "python3", + "create_venv": True, + "requirements": None, # defaults to /modules/.../requirements.txt + }, + "run": { + "source_root": None, + "work_dir": None, + "exclude": [], + "max_payload_mb": 200, + # ducktape's loader walks up from each test file while __init__.py exists and + # appends the resulting top-level directory to sys.path + # (ducktape/tests/loader.py::_add_top_level_dirs_to_sys_path), so the synced + # sources are importable without being installed. Only the third-party + # requirements have to be in the venv. Left as an opt-in escape hatch. + "install_sources": False, + }, + "deploy": { + "dist_dir": "./dist", + "install_root": None, # defaults to cluster.install_root + "sudo": False, # /opt is writable by the ordinary user on these clusters + "owner": None, + "staging_dir": "/tmp/ducktests-remote-staging", + "checksum": False, + # Send only what differs, hardlinking the rest against the deployment already on + # the host. Falls back to a whole tarball when either end has no rsync. + "rsync": True, + # rsync-style patterns left out of every distribution. Empty by default: a + # release directory is shipped whole. The list matters when ignite-dev is a link + # to a source checkout, where only the built jars are wanted on the workers. + "exclude": [], + }, + "clean": { + # pgrep -f patterns; see IGNITE_MAIN_CLASSES above. + "process_pattern": "org.apache.ignite", + # persistent_root default from ignitetest/services/utils/path.py. + "paths": ["/mnt/service"], + "allowed_roots": ["/mnt", "/tmp", "/var/tmp"], + }, + "provision": { + "packages": list(DOCKERFILE_PACKAGES), + "install_jdk": False, + "ssh_env_path_extra": [], + "dirs": ["/mnt/service"], + }, + # Everything pip needs to reach an index that is not PyPI. Only the runner ever runs + # pip: the venv install and `run --install-sources`. Workers are driven over plain + # ssh and never see Python. + "pip": { + "index_url": None, # --index-url + "extra_index_url": [], # --extra-index-url, repeatable; a bare string is ok + "trusted_host": [], # --trusted-host, repeatable; a bare string is ok + "timeout": None, # --timeout, seconds + "retries": None, # --retries + "cert": None, # --cert; a RUNNER-side path to a CA bundle + }, + # Which JVM the workers run the tests under. See ducktests_remote/java.py for the + # resolution ladder and for why PATH matters more here than JAVA_HOME. + "java": { + "major": JAVA_MAJOR, + "home": None, # explicit JDK home on the workers; set -> no search + "search_paths": list(JAVA_SEARCH_PATHS), + "archive": None, # coordinator-side .tar.gz/.tgz/.tar, or a directory + "install_root": None, # defaults to cluster.install_root + "name": None, # target directory name; defaults to the archive's own + "ssh_environment": True, # write ~/.ssh/environment + "bashrc": True, # write a marked block at the top of ~/.bashrc + }, + "ssh": { + "connect_timeout": 15, + }, + "jobs": 16, + "globals": {}, + "parameters": {}, +} + +_FREE_FORM_SECTIONS = ("globals", "parameters") + +_PLACEHOLDER = re.compile(r"\$\{(env|file):([^}]+)\}") + + +class ConfigError(Exception): + """Raised for anything the operator can fix by editing a file or a flag (exit 1).""" + + +class _NullRedactor: + """Accepts resolved values and forgets them; used when no redactor was supplied.""" + + def add(self, value): + """Ignore ``value``.""" + + +def interpolate(value, redactor=None, environ=None, source=""): + """ + Resolve ``${env:NAME}`` and ``${file:PATH}`` placeholders throughout a structure. + + A missing variable or file is a hard error naming both the placeholder and the file + it came from. Substituting an empty string instead would produce a run that fails + three hours later with an authentication error nobody can trace back to here. + + Every resolved value is handed to ``redactor`` so that it is masked in everything the + CLI prints afterwards, ``--dry-run`` included. + """ + environ = os.environ if environ is None else environ + redactor = _NullRedactor() if redactor is None else redactor + + if isinstance(value, dict): + return {k: interpolate(v, redactor, environ, source) for k, v in value.items()} + if isinstance(value, list): + return [interpolate(v, redactor, environ, source) for v in value] + if not isinstance(value, str): + return value + + resolved = value + for match in _PLACEHOLDER.finditer(value): + kind, ref = match.group(1), match.group(2).strip() + if kind == "env": + if ref not in environ: + raise ConfigError( + "%s: environment variable %r referenced by ${env:%s} is not set" + % (source, ref, ref)) + replacement = environ[ref] + else: + path = Path(os.path.expanduser(ref)) + if not path.is_file(): + raise ConfigError( + "%s: file %s referenced by ${file:%s} does not exist" % (source, path, ref)) + replacement = path.read_text(encoding="utf-8").strip() + redactor.add(replacement) + resolved = resolved.replace(match.group(0), replacement) + return resolved + + +def deep_merge(base, overlay): + """ + Merge ``overlay`` into a copy of ``base``. + + Dicts merge recursively; every other type, lists included, replaces outright. + Lists concatenating across profiles would make it impossible to *shrink* a list in a + later layer, which is exactly what an override is for. + """ + if not isinstance(base, dict) or not isinstance(overlay, dict): + return copy.deepcopy(overlay) + merged = copy.deepcopy(base) + for key, value in overlay.items(): + if key in merged and isinstance(merged[key], dict) and isinstance(value, dict): + merged[key] = deep_merge(merged[key], value) + else: + merged[key] = copy.deepcopy(value) + return merged + + +def parse_document(text, source=""): + """ + Parse YAML or JSON, choosing by content rather than by file extension. + + :param text: document body. + :param source: name used in error messages. + """ + stripped = text.lstrip() + try: + if stripped.startswith(("{", "[")): + return json.loads(text) + return yaml.safe_load(text) + except (ValueError, yaml.YAMLError) as ex: + raise ConfigError("%s: not valid YAML or JSON: %s" % (source, ex)) from ex + + +def load_document(path): + """:return: the parsed contents of ``path``.""" + path = Path(path).expanduser() + if not path.is_file(): + raise ConfigError("config file not found: %s" % path) + data = parse_document(path.read_text(encoding="utf-8"), str(path)) + if data is None: + return {} + if not isinstance(data, dict): + raise ConfigError("%s: top level must be a mapping, found %s" % (path, type(data).__name__)) + return data + + +def resolve_profile(name, profiles_dir): + """ + Locate a profile by name. + + Search order: ``/.yaml`` (then ``.yml``/``.json``), a + coordinator-relative path, and finally the profiles shipped in ``examples/``. + """ + candidates = [] + for base in (Path(profiles_dir).expanduser(), Path.cwd()): + for suffix in (".yaml", ".yml", ".json", ""): + candidates.append(base / (name + suffix)) + for suffix in (".yaml", ".yml", ".json"): + candidates.append(PACKAGE_EXAMPLES / ("profile-%s%s" % (name, suffix))) + candidates.append(PACKAGE_EXAMPLES / (name + suffix)) + for candidate in candidates: + if candidate.is_file(): + return candidate + raise ConfigError("profile %r not found; looked in %s" % ( + name, ", ".join(sorted({str(c.parent) for c in candidates})))) + + +def validate(config, reference=None, path=""): + """ + Reject unknown keys with a "did you mean" hint. + + A typo silently ignored in a config that drives a three hour run is expensive; this + turns it into an immediate, named failure. + """ + reference = DEFAULTS if reference is None else reference + for key, value in config.items(): + where = "%s.%s" % (path, key) if path else key + if key not in reference: + hint = difflib.get_close_matches(key, list(reference), n=1) + suffix = "; did you mean %r?" % hint[0] if hint else "" + raise ConfigError("unknown config key %r%s" % (where, suffix)) + if where in _FREE_FORM_SECTIONS: + continue + if isinstance(value, dict) and isinstance(reference[key], dict) and reference[key]: + validate(value, reference[key], where) + + +def env_overrides(environ=None): + """ + Build a config overlay from ``DTR_*`` environment variables. + + ``DTR_CLUSTER__RUNNER=build-vm-01`` sets ``cluster.runner``. A double underscore + separates path segments so single underscores stay usable inside key names + (``DTR_RUN__MAX_PAYLOAD_MB``). A handful of single-segment aliases cover the flags + operators reach for most. + + Any other ``DTR_*`` variable is left alone. Profiles interpolate secrets with + ``${env:...}``, and those variables are frequently named ``DTR_SOMETHING`` too; + treating every one of them as a config path would turn a password into a config + error. The ``__`` form is still validated, so a typo there is caught. + """ + environ = os.environ if environ is None else environ + aliases = {"DTR_RUNNER": "cluster.runner", "DTR_JOBS": "jobs", + "DTR_CLUSTER": "cluster.name", "DTR_STATE_ROOT": "cluster.state_root", + "DTR_INSTALL_ROOT": "cluster.install_root", "DTR_USER": "cluster.user"} + overlay = {} + for name, raw in sorted(environ.items()): + if not name.startswith("DTR_") or name in ("DTR_CONFIG", "DTR_PROFILE"): + continue + if name in aliases: + dotted = aliases[name] + elif "__" in name[len("DTR_"):]: + dotted = name[len("DTR_"):].lower().replace("__", ".") + else: + continue + set_dotted(overlay, dotted, coerce_scalar(raw)) + return overlay + + +def coerce_scalar(raw): + """:return: ``raw`` parsed as JSON when possible, otherwise the string itself.""" + try: + return json.loads(raw) + except (TypeError, ValueError): + return raw + + +def set_dotted(target, dotted, value): + """Set ``a.b.c`` inside a nested dict, creating intermediate dicts as needed.""" + parts = dotted.split(".") + node = target + for part in parts[:-1]: + node = node.setdefault(part, {}) + if not isinstance(node, dict): + raise ConfigError("cannot set %r: %r is not a mapping" % (dotted, part)) + node[parts[-1]] = value + return target + + +def get_dotted(source, dotted, default=None): + """:return: the value at ``a.b.c`` inside a nested dict, or ``default``.""" + node = source + for part in dotted.split("."): + if not isinstance(node, dict) or part not in node: + return default + node = node[part] + return node + + +def load_config(config_files=(), profiles=(), overrides=None, environ=None, + user_config=USER_CONFIG_FILE, redactor=None): + """ + Compose the effective configuration from every layer. + + :param config_files: ``--config`` paths, applied in order. + :param profiles: ``--profile`` names, applied in order. + :param overrides: overlay built from explicit command-line flags. + :param environ: environment mapping, defaults to ``os.environ``. + :param user_config: path to the per-user config file. + :param redactor: collects values resolved from ``${env:}`` / ``${file:}``. + :return: the merged, validated configuration. + """ + config = copy.deepcopy(DEFAULTS) + sources = [] + + user_config = Path(user_config).expanduser() if user_config else None + if user_config and user_config.is_file(): + config = _layer(config, load_document(user_config), str(user_config)) + sources.append(str(user_config)) + + for path in config_files: + config = _layer(config, load_document(path), str(path)) + sources.append(str(path)) + + for name in profiles: + path = resolve_profile(name, config.get("profiles_dir", DEFAULTS["profiles_dir"])) + config = _layer(config, load_document(path), str(path)) + sources.append(str(path)) + + config = _layer(config, env_overrides(environ), "environment") + + if overrides: + config = _layer(config, overrides, "command line") + + config = _interpolate_config(config, sources, redactor, environ) + + config["cluster"]["user"] = config["cluster"]["user"] or _current_user() + config["_sources"] = sources + return config + + +def _interpolate_config(config, sources, redactor, environ): + """ + Resolve placeholders in every section except the free-form ones. + + ``globals`` and ``parameters`` are left alone here: :mod:`globals_builder` resolves + them per layer, so its errors can name the profile a placeholder came from, and doing + it twice would re-scan an already resolved secret for placeholders of its own. + """ + source = "config (%s)" % (", ".join(sources) if sources else "built-in defaults") + resolved = {} + for key, value in config.items(): + if key in _FREE_FORM_SECTIONS: + resolved[key] = value + else: + resolved[key] = interpolate(value, redactor, environ, source) + return resolved + + +def _layer(config, overlay, source): + if not overlay: + return config + try: + validate(overlay) + except ConfigError as ex: + raise ConfigError("%s: %s" % (source, ex)) from ex + return deep_merge(config, overlay) + + +def _current_user(): + try: + return getpass.getuser() + except (KeyError, OSError): # no passwd entry, e.g. some containers + return os.environ.get("USER") or os.environ.get("USERNAME") or "root" + + +def expand_path(path, default=None): + """:return: ``path`` with ``~`` expanded, or ``default`` when path is falsy.""" + if not path: + return default + return os.path.expanduser(str(path)) + + +def redacted_summary(config): + """:return: a config copy safe to write into ``meta.json`` (globals stripped).""" + summary = copy.deepcopy({k: v for k, v in config.items() if not k.startswith("_")}) + summary["globals"] = "" + summary["parameters"] = "" + return summary diff --git a/modules/ducktests/tests/ducktests_remote/docs/commands.md b/modules/ducktests/tests/ducktests_remote/docs/commands.md new file mode 100644 index 0000000000000..0dfc6db103929 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/commands.md @@ -0,0 +1,535 @@ + + +# Command reference + +Ten commands: `run`, `status`, `logs`, `fetch`, `stop`, `provision`, `deploy`, `clean`, +`doctor`, `keys`. + +## Flags every command accepts + +| Flag | Effect | +| --- | --- | +| `--config FILE` | extra config document; repeatable, applied in order | +| `--profile NAME` | named profile; repeatable, applied in order | +| `--runner HOST` | override `cluster.runner`; `local` runs ducktape here | +| `--jobs N` | fan-out parallelism (default 16) | +| `-v, --verbose` | print every command as `+ host$ ...`, and full per-host output | +| `-q, --quiet` | only results and errors | +| `--dry-run` | print what would happen; execute nothing | +| `--no-color` | disable ANSI colour | +| `--fail-fast` | stop scheduling further hosts after the first failure | +| `--no-progress` | do not report per-host progress during long transfers | + +`--version` prints the CLI version. Everything after a bare `--` is passed straight to +ducktape by `run`. + +## What each command transfers + +The question that matters most on a slow link, answered in one table. "Coordinator → X" +means bytes leave your machine. + +| Command | Transfers | To whom | When | +| --- | --- | --- | --- | +| `run` | source tree, `cluster.json`, `globals.json`, `parameters.json`, `run.sh`, `launch.sh`, `meta.json` | **runner only** | every run, unless `--no-sync` (which skips the source tree only) | +| `provision --only jdk` | the `java.archive` tarball | **workers**, and only those with no matching JDK | see [java.md](java.md) | +| `deploy` | one tarball per distribution | **workers**, skipping hosts whose manifest already matches | every invocation, unless skipped or `--dry-run` | +| `keys push` | private key + `.pub` to the runner, public key text to the workers | runner and workers | every invocation | +| `fetch` | *downloads* a results archive | runner → coordinator | every invocation | +| `doctor`, `status`, `logs`, `clean`, `stop` | nothing | — | never | + +Nothing is transferred implicitly by any other command, and `--dry-run` transfers nothing +anywhere. + +--- + +## `run` + +``` +run [TEST_PATH...] [-t PATH]... [-g KEY=VALUE]... [-p KEY=VALUE]... + [--globals-json JSON | --globals-file PATH] [--params-json JSON] + [--cluster-file PATH] [-n N] [--source-root PATH] [--no-sync] + [--exclude PATTERN]... [--work-dir PATH] [--install-sources] + [--pip-index-url URL] [--pip-extra-index-url URL]... [--pip-trusted-host HOST]... + [--pip-timeout SECONDS] [--pip-cert PATH] + [--repeat N] [--max-parallel N] [--test-runner-timeout MS] [--results-root PATH] + [--skip-preflight] [--follow | --detach] [-- EXTRA DUCKTAPE ARGS] +``` + +At least one test path is required, positionally or with `-t`. + +### Where you run it from, and what a test path means + +The whole Ignite checkout is synced to the runner, and ducktape is started in its +`modules/ducktests/tests` directory — the same working directory `docker/run_tests.sh` +uses. So a test path is written exactly as it is locally: + +``` +ducktests-remote run ./ignitetest/tests/smoke_test.py::SmokeServicesTest.test_ignite_start_stop +``` + +The checkout is found by walking up from the current directory, so the command works from +`modules/ducktests/tests`, from the repository root, or from anywhere in between. +`--source-root` / `run.source_root` overrides the search. A directory that is not a +checkout — no `modules/ducktests/tests/docker/requirements.txt` under it — is refused +before anything is uploaded. + +Test paths are accepted in any form that resolves on the coordinator (relative to the +current directory, to the tests directory, or to the checkout root) and are rewritten +into the tests-relative form ducktape sees. A path inside the checkout but outside the +tests directory becomes a runner-side absolute path; one that resolves nowhere is passed +to ducktape unchanged, with a warning. + +### Order of operations + +1. **Compose `globals`**: `--globals-json`/`--globals-file` (the raw layer), then config + `globals`, then `-g` overrides. `parameters` likewise from `--params-json`, config + `parameters`, `-p`. Placeholders resolve here; a missing one aborts now. +2. **Select nodes** (`-n` takes the first N) and build the cluster payload — generated from + the inventory, or `--cluster-file` read and validated but uploaded byte for byte. +3. **Topology warnings**: the runner also appearing in `cluster.nodes`, and an inventory + below three hosts (most suites declare `@cluster(num_nodes=...)` well above that and + would be skipped as un-runnable). Warnings only. +4. **Preflight**: the full `doctor` check set, unless `--skip-preflight` or `--dry-run`. + Any FAIL stops here with exit 2, before anything is created. +5. **Allocate the run id** — `max-20260727-141233-9f2a` — and derive the run directory, + work directory and results root, and normalise the test paths against the checkout. +6. **Render `run.sh`** and, with `--dry-run` or `-v`, print it along with `cluster.json` + and a redacted `globals.json`. **`--dry-run` returns here.** +7. **Create** the run and results directories on the runner. +8. **Sync the source tree** to `/src/` unless `--no-sync`. The payload + is measured first and refused above `run.max_payload_mb` (200 MB): that almost always + means a build directory leaked in. rsync when both ends have it, otherwise a tar stream + over scp. Progress is reported while it runs — see + [deploy's description of the display](#watching-a-long-transfer), which is the same one. +9. **Ensure the venv**: create `/venv` when missing, and install + `docker/requirements.txt` into it when `import ducktape` fails. This is where `pip.*` + applies. Runs after the sync because the requirements file comes from the synced tree. + A missing requirements file is reported as such, separately from an unreachable index. +10. **`--install-sources`** (opt-in): `pip install -e /modules/ducktests/tests`, + with the same `pip.*` flags. +11. **Write the artifacts**: `cluster.json`, `globals.json` (`0600`), `parameters.json` + (`0600`, when non-empty), `run.sh` (`0755`), `launch.sh` (`0755`), `meta.json`, and + update the `runs/latest` symlink. +12. **Launch detached** — `setsid nohup bash launch.sh`, falling back to `nohup` plus + `disown` — and record the pid. +13. **Follow** the log, or print the reattach commands and exit with `--detach`. + +### Following, and Ctrl-C + +The run is detached from second zero, so following is just streaming a file: + +- **Ctrl-C detaches. It does not stop the run.** Losing a three-hour run to a reflex is + not recoverable. +- A **second Ctrl-C within 3 seconds** offers to stop it; in a non-interactive shell it + simply detaches. +- Reattach with `logs -f` from any coordinator. + +### Exit codes + +`0` success · `2` preflight failed · `4` ducktape exited non-zero · `1` configuration +error · `5` transport error · `130` interrupted. See [runs.md](runs.md#exit-codes) for why +4 and 5 are distinct. + +--- + +## `doctor` + +``` +doctor [--json] [-n N] +``` + +Probes the coordinator, the runner and every worker **in parallel**, never stopping at the +first failure, and **changes nothing anywhere**. Also runs implicitly before `run`. + +### What it checks, in order + +1. **Coordinator** — with a remote runner: `ssh` and `scp` on `PATH`. With + `--runner local`: ducktape importable here, and its version against the pin in + `docker/requirements.txt`. +2. **Runner** — `bash`, `python3`, `setsid` present (a missing `setsid` is a WARN: there is + a `nohup` fallback); ducktape importable through the venv python; `identity_file` exists + with sane permissions; free space at `state_root`; the effective pip index. +3. **Workers, reachability** — one connection each from the coordinator, every failure + classified (see [troubleshooting.md](troubleshooting.md#ssh-failure-classes)). +4. **Workers, substance** — one script per reachable host returning: clock skew, free space + at `install_root` and at `/mnt/service`, writability of both, stale JVMs matching + `clean.process_pattern`, passwordless sudo, the distributions present under + `install_root`, and the full JDK probe. +5. **Name resolution** — from the first reachable worker, `getent hosts` for every peer. + Workers that cannot resolve each other fail deep inside discovery with no message that + points here. +6. **Runner → workers** — the connection *ducktape itself* will make, from the runner, with + the runner-side identity. A coordinator that can reach a worker proves nothing about the + runner being able to. + +### The administrator block + +When hosts are unusable, the report ends with a copy-pasteable block naming the hosts, the +account, the key fingerprint being offered, and the exact line to append to +`authorized_keys` — plus, for missing sudo, the precise sudoers line and the two test +suites that need it. Forward it as is. + +### Verdicts + +FAIL stops a run at preflight; WARN never does. `--json` prints every check with its scope, +host, name, status and message, plus an `ok` boolean. Exit `0` when nothing failed, `2` +otherwise. + +--- + +## `provision` + +``` +provision [--only STEP]... [--skip STEP]... [--sudo] [--install-jdk] + [--java-home PATH] [--java-major N] [--java-archive PATH] [--force] + [--create-user NAME] [--authorize-key PATH] [--write-hosts] [-n N] [--json] +``` + +Brings unconfigured VMs up to the state `docker/Dockerfile` guarantees. Steps run in this +order, each idempotent and independently selectable: + +| Step | Does | Needs root | In the default set | +| --- | --- | --- | --- | +| `packages` | installs `provision.packages`; detects apt/dnf/yum, an unknown manager is a clear failure rather than a guess | yes | yes | +| `jdk` | resolves a JDK per host, delivers `java.archive` to hosts that need one — [java.md](java.md) | only with `--install-jdk`, or to write into an unwritable `install_root` | yes | +| `python` | reports whether `python3` exists. **Verifies only**: workers do not need Python, and nothing here will install it | no | yes | +| `user` | `--create-user NAME` plus `--authorize-key` | yes | only with `--create-user` or `--only user` | +| `ssh-env` | points the non-interactive `PATH`/`JAVA_HOME` at the resolved JDK, then verifies over a fresh connection | no | yes | +| `dirs` | creates and chowns `provision.dirs` plus `install_root` | yes | yes | +| `hosts` | rewrites only the block between `# BEGIN ducktests-remote` and `# END ducktests-remote` in `/etc/hosts` | yes | only with `--write-hosts` or `--only hosts` | + +Anything needing root goes through `sudo -n`. If `--sudo` was not given, the step is +reported as skipped and **the remaining steps still run** — a partial provision with an +honest report beats an all-or-nothing failure. Unless `--dry-run`, the command always +finishes by running the `doctor` checks, so it ends with evidence rather than an +assumption; a doctor FAIL makes it exit 2. + +`--dry-run` prints the exact script per host and probes nothing. + +Delivering a JDK is the one step that moves real bytes, so it reports progress per host +while it does — the same display [`deploy` uses](#watching-a-long-transfer). The +probe-only case does not: it is one round trip per host, and only the hosts that come back +without a usable JDK are sent anything. + +--- + +## `deploy` + +``` +deploy [--dist-dir PATH] [--only NAME]... [--exclude PATTERN]... [--install-root PATH] + [--via HOST] [--sudo] [--owner USER] [--force] [--checksum] [--no-rsync] + [-n N] [--json] +``` + +Each subdirectory of `--dist-dir` is copied verbatim to `/`. The name +is never interpreted or checked against version parsing, which is what makes fork layouts +work without special cases — you name the directories to match what the tests expect. + +Per distribution, per host: + +1. Build a manifest on the coordinator: sorted relative paths plus sizes and mtimes, or + content hashes with `--checksum`. +2. Read `.ducktests-deploy.json` from the target; if its hash matches, **skip this host** + (unless `--force`). +3. Refuse if `install_root` is not writable and `--sudo` was not passed. +4. Extract into a staging directory beside the target, then **swap it into place** and + delete the old tree. A half-copied distribution that looks present is worse than an + absent one. + +A staging directory is `/..tmp.`, and holds a whole copy of +the distribution until the swap. One that is never swapped in — a failed host, an +interrupted run — is removed on the way out, and any left by an earlier attempt are swept +before the next one starts. It is scoped to the distribution being deployed: `deploy +ignite-dev` never touches `.ignite-2.17.0.tmp.*`. + +`--via HOST` uploads the payload once to an intermediate host and fans out from there. +`deploy` prints the total bytes before it starts — on a twelve-host cluster a 300 MB +distribution is 3.7 GB from a laptop — and suggests `--via` when that total is large. + +### Incremental redeploys + +Step 4 has two implementations, and the default is the incremental one: + +``` +rsync -a --files-from=- --link-dest=/opt/ignite-dev ./dist/ignite-dev/ host:/opt/.ignite-dev.tmp.ab12/ +``` + +The staging directory is filled by rsync rather than by extracting a tarball. +`--link-dest` points at the deployment already on the host, so every file that has not +changed becomes a **hardlink** to the one already there and is never sent: rebuild one +module and the transfer is one jar, on a tree of any size. The staging directory is still +built from scratch and still swapped in atomically, so this costs nothing in safety; +deleting the old tree afterwards only drops link counts. + +Two details worth knowing: + +- rsync is fed the exact file list on **stdin**, not `--exclude` patterns. Its matching + rules are close to `is_excluded`'s but not identical, and a distribution that differs + from the manifest describing it is the one bug the manifest exists to prevent. +- Nothing is tarred when every host takes this path — the tarball is built lazily, only + for hosts that need it. + +The result line reports what actually moved: + +``` +w01 changed 4.1s rsync: 12 of 8431 file(s) changed, 4.0 MB sent +``` + +### Watching a long transfer + +Sending 3.5 GB to twelve machines takes minutes, and a terminal that prints nothing for +that long is indistinguishable from one that has hung. Every transfer that can take +minutes reports the same way — `deploy`, `run`'s source sync, and `provision`'s JDK +delivery — redrawn in place: + +``` + worker01 █████████████████░░░░░░░ 70% 210.0 MB / 300.0 MB 24.1 MB/s + worker02 ████████░░░░░░░░░░░░░░░░ 32% 96.0 MB / 300.0 MB 18.7 MB/s + worker03 ░░░░░░░░░░░░░░░░░░░░░░░░ extracting + total ████████████░░░░░░░░░░░░ 50% 1/4 host(s) 306.0 MB sent 1:12 +``` + +Hosts drop out of the block as they finish — they are already counted in the total — and +a cluster larger than the block ends with `... N more host(s)`. + +The total is the **mean of the per-host fractions**, not bytes against a predicted total: +rsync sends only what differs, so a denominator of "distribution size × hosts" would stop +at 12% and finish there. Each host owns an equal share of that bar. + +Where the byte counts come from: `--info=progress2` on the rsync path, and the bytes +handed to the remote `cat` on the tarball path. scp has a progress meter of its own, but +it is written for a terminal and suppressed when its output is a pipe — which it always +is here — so a watched tarball upload streams the file through `ssh 'cat > path'` and +counts the chunks itself. + +| Situation | What you get | +| --- | --- | +| a terminal | the redrawn block above | +| not a terminal (a log, CI), or `--verbose` | one `total ...` line every 15 s | +| `--quiet`, `--dry-run`, `--no-progress` | nothing | + +`--verbose` deliberately drops to the single line: it prints a traced command per host, +and a redrawn block would fight with it. + +Counting bytes is not free — rsync grows a meter, and a watched upload streams through +`ssh 'cat > path'` instead of scp — so when nothing will draw the result, nothing is asked +to report and the plain transfer is used. That is what `--no-progress` really turns off. + +The tarball path is used instead when: + +| Condition | Why | +| --- | --- | +| `--no-rsync`, or `deploy.rsync: false` | you asked for it | +| `--via HOST` | the point of `--via` is one upload across the slow link, not one per host | +| no `rsync` on the coordinator, or on that worker | probed per host, so a mixed cluster works | +| the coordinator runs Windows | rsync reads `C:/dist/ignite-dev` as host `C` | + +`provision` installs `rsync` on the workers as part of the Dockerfile-derived package +list, so a provisioned cluster is already on the fast path. `--checksum` is passed +through to rsync, matching the manifest mode. + +### Leaving files out + +Excludes are rsync-style patterns, matched against paths relative to the distribution +root: a pattern matches the whole relative path, a path prefix, or any single path +component. So `src` drops every `modules/*/src`, `*.jar` drops jars anywhere, and +`modules/indexing` drops that one subtree — but `target/libs` matches only a `target/libs` +directly at the root, not `modules/core/target/libs`. They default to nothing: a +distribution without them is shipped byte for byte. Three sources, most specific winning: + +| Source | Scope | +| --- | --- | +| `--exclude PATTERN` (repeatable) | every distribution in this invocation | +| `.ducktests-deploy.ignore` at the root of a distribution | that distribution | +| `deploy.exclude` in the configuration | every distribution | + +The list is read from one source as a whole; sources are never merged. The ignore file is +one pattern per line, `#` comments allowed, and is itself never shipped. + +It is **not** called `.ducktestsignore`: when `ignite-dev` links to a checkout, the +distribution root and the source root are the same directory, and the two lists are +opposites — the source sync drops `target`, `deploy` keeps almost nothing else. + +The manifest is built from the same filtered file list, so a host reported as up to date +holds exactly the files the tarball carried. Change the excludes and every host is +redeployed, as it should be. + +### `ignite-dev` from your own checkout + +`ignite-dev` is the distribution the tests resolve `DEV_BRANCH` to, and on a worker it +must have the layout of a *built source tree*, not of a release: `IgniteSpec` puts +`modules//target` and `modules//target/libs` on the classpath for every +module a test asks for (`ignitetest/services/utils/ignite_spec.py`), `path.py` runs +`bin/ignite.sh` from the same home and reads certificates from +`modules/ducktests/tests/certs`. Everything else in a checkout is ballast for a worker. + +So link the distribution to your checkout and let the excludes do the trimming: + +```bash +mkdir -p ~/dist +ln -sfn ~/Development/vanilla/ignite ~/dist/ignite-dev # relink any time +``` + +`deploy` follows that link: `is_dir()` accepts it as a distribution, the tree is walked +through it, and the workers receive ordinary files. Symlinks *inside* a distribution are +a different matter — they are stored as links and arrive dangling, so keep real files +below the top level. + +Then, in your configuration: + +```yaml +deploy: + dist_dir: ~/dist + exclude: [.git, .idea, src, docs, assembly, classes, test-classes, + generated-sources, generated-test-sources, maven-status, maven-archiver, + surefire-reports, javadoc, "*.tar.gz", "*.zip", __pycache__, "*.pyc"] +``` + +`src` as a pattern drops every `modules/*/src`, `classes` drops the exploded output Java +never reads off a classpath directory, and `target/*.jar` plus `target/libs/*.jar` +survive. The daily loop is then two commands: + +```bash +mvn package -pl :ignite-ducktests -am -DskipTests # in the checkout, however you build +ducktests-remote deploy --only ignite-dev +``` + +Check the damage before the first real transfer — `--dry-run` prints the payload size and +how many files the patterns dropped, and transfers nothing: + +```bash +ducktests-remote deploy --only ignite-dev --dry-run +``` + +Rebuilding one module changes the manifest hash for the whole distribution, so every host +is redeployed — but on the rsync path that redeploy sends only the jars that changed. + +### Where the directory names come from + +ignitetest resolves a distribution home as `/`, where `product` is +`str(IgniteVersion(version))`, and `IgniteVersion.__str__` **normalises**: + +| `ignite_versions` entry | directory | +| --- | --- | +| `dev` | `ignite-dev` | +| `2.17.0` | `ignite-2.17.0` | +| `ise-0-32` | `ise-0-32` | +| `ise--6` | `ise-6` — note the collapsed dash | + +A fork can override `product`, so `doctor` reports a missing directory as a WARN listing +what it *did* find rather than failing on a guessed mapping. + +--- + +## `clean` + +``` +clean [-n N] [--keep-paths] +``` + +Kills processes matching `clean.process_pattern` (SIGTERM, five seconds, then SIGKILL for +survivors) and removes `clean.paths`. `--keep-paths` kills without deleting. + +Every path is validated **on the coordinator, before it is sent anywhere**: it must be +absolute, must sit under one of `clean.allowed_roots`, and must not *be* one of those roots. +A bug here would delete distributions across every machine at once, so the rule is +deliberately blunt. + +`--dry-run` prints the exact process list (pid plus command line) and the exact paths with +their sizes, and kills and deletes nothing. Use it first, always. + +--- + +## `keys push` + +``` +keys push [--identity PATH] [--generate] [-n N] +``` + +Installs the private key on the **runner** (mode `0600`, `.pub` alongside at `0644`) and +appends the public key to `authorized_keys` on every worker and extra host. + +Why it exists: ducktape connects from the runner to the workers with the identity named in +the cluster file, and a detached run outlives your SSH session. Agent forwarding dies with +that session, producing a run that authenticates for the first few minutes and then fails +on every subsequent connection. A real key file on the runner is the only arrangement that +survives. + +`--generate` creates an RSA keypair when the identity does not exist. Already-authorised +hosts report `ok` rather than appending a duplicate. + +--- + +## `status` + +``` +status [RUN_ID] [--all] [--json] [-n LINES] +``` + +With no run id, the most recent run on the runner. Prints state, pid, start time, elapsed, +exit code, test paths, cluster and node count, run directory, results path, and the last +15 log lines (`-n`). `--all` prints a table of every run, newest first. `--json` is the +Jenkins-friendly form. + +Because all state lives on the runner, this works from any coordinator, including one that +did not start the run. + +--- + +## `logs` + +``` +logs [RUN_ID] [-f] [-n LINES] +``` + +Prints the last 200 lines (`-n`) of `ducktape.log`, then streams with `-f` until the run +ends. Ctrl-C stops following and never touches the run. Output passes through the redactor, +so a secret that reached the log is masked on the way to your terminal. + +--- + +## `fetch` + +``` +fetch [RUN_ID] [--dest DIR] [--full] +``` + +Archives the results **on the runner**, downloads one file, and extracts it into +`/` (default `./ducktests-results/`), then downloads `ducktape.log` +alongside. By default only ducktape's reports — `report.html`, `report.txt`, `report.json`, +`test_log.info`, `session.log`; `--full` takes the whole results tree. + +`globals.json` is **always** excluded, at both ends. Extraction refuses any member that +would escape the destination directory. + +--- + +## `stop` + +``` +stop [RUN_ID] [--kill] [--timeout SECONDS] [--no-clean] [-n N] +``` + +1. Touches `stopped` in the run directory, so the final state is reported as `stopped` + rather than `failed`. +2. SIGTERMs the run's **process group**, waits up to `--timeout` (default 60s), and with + `--kill` SIGKILLs whatever survives. Without `--kill` it says so and leaves it. +3. Writes exit code `143` if the process never wrote one itself. +4. Runs `clean` across the workers, unless `--no-clean`. + +Stopping a run that already ended is safe: it says so, and still cleans. diff --git a/modules/ducktests/tests/ducktests_remote/docs/concepts.md b/modules/ducktests/tests/ducktests_remote/docs/concepts.md new file mode 100644 index 0000000000000..2e3b72bfde9cb --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/concepts.md @@ -0,0 +1,125 @@ + + +# Concepts + +## The three roles + +| Role | What it is | What runs there | +| --- | --- | --- | +| **coordinator** | wherever you typed `ducktests-remote` — laptop, cluster VM, Jenkins agent | the CLI itself, and only the CLI | +| **runner** | the host the `ducktape` process lives on: `cluster.runner`, or `local` | ducktape, the Python venv, all run state | +| **workers** | `cluster.nodes` — the machines ducktape drives over SSH | Ignite JVMs, the Java applications, Kafka/ZooKeeper when a test needs them | + +The coordinator can be any of the three at once. `cluster.runner: local` makes the +coordinator the runner; a worker listed in `extra_hosts` can be the runner too. The roles +are about *function*, not about hardware, and every command is written so that the local +and remote cases take the same code path — see [internals.md](internals.md#transport). + +A worker never needs Python and never runs any of this code. ducktape drives it over plain +SSH, which is why `provision --only python` verifies rather than installs, and why the JDK +is the only runtime the workers must actually have. + +## What lives where + +**On the coordinator:** your config files (`~/.ducktests-remote/config.yaml`, profiles), +your source tree, and whatever `deploy --dist-dir` / `java.archive` point at. Nothing else. +Closing the laptop loses nothing. + +**On the runner, under `cluster.state_root`** (default `~/.ducktests-remote`): + +``` +venv/ the Python environment ducktape runs in +src// the synced source tree for one run +runs// one run directory; see runs.md +runs/latest symlink to the newest run +``` + +**On the workers:** `cluster.install_root` (default `/opt`) holds the Ignite +distributions and, when delivered, the JDK. `/mnt/service` (ignitetest's +`persistent_root`) holds everything a test writes. `~/.ssh/environment` and `~/.bashrc` +carry `JAVA_HOME`/`PATH`. + +## Why all run state lives on the runner + +Because a run outlives the thing that started it. A three-hour suite launched from a +laptop must survive the laptop being closed, and a Jenkins job that starts a run must be +followable from a workstation afterwards. + +So: `run` detaches the ducktape process from second zero, writes everything about the run +into a directory on the runner, and every other command (`status`, `logs`, `fetch`, `stop`) +reads that directory. Any coordinator with SSH access to the runner can inspect, follow or +stop a run that a different coordinator started. There is no local state to get out of sync +and nothing to clean up on the coordinator. + +## Invariants + +These hold everywhere in the codebase. Each has a unit check, and breaking one is a bug +even when the result looks correct. + +**The CLI never imports ducktape.** It renders artifacts and drives the ducktape process on +the runner. That keeps it installable on a coordinator with no ducktape at all, and lets +the two versions move independently. `checks/check_remote_transport.py` runs a subprocess +to prove the import boundary holds. + +**Everything remote goes through a `Transport`.** No module above `transport.py` shells out +to `ssh` or `scp`. `--runner local` and `--runner build-vm-01` therefore exercise identical +code paths, and the unit checks can substitute a recording fake. + +**`--dry-run` executes nothing.** Not even a read-only probe. Commands that need facts from +a host to decide report `not probed (--dry-run)` rather than guessing, and commands that +would transfer print the size instead. + +**`doctor` never mutates.** It is safe on a cluster someone else is using. The JDK +discovery script it runs is checked to contain no `mkdir`, `rm`, `mv`, `chmod` or +redirection (`checks/check_remote_java.py`). + +**A fan-out never stops at the first failure** unless `--fail-fast` says so. When the +outcome of a command is a request to whoever administers the machines, a partial list of +broken hosts is worse than useless. + +**Secrets are redacted by value, not by key name.** Anything resolved from `${env:}` or +`${file:}` is registered with a `Redactor` and replaced with `***` in everything the CLI +prints — including `--dry-run` output, streamed logs and error messages. A password that +leaks into an unrelated field is still caught. See +[configuration.md § Secrets](configuration.md#secrets-and-redaction). + +## The two flows this replaces + +**`docker/run_tests.sh`** — the local Docker flow. Every node is a container, `ducker01` is +the runner, and the image guarantees the node state. That flow is unchanged and remains the +right one for developing tests. + +**A Jenkins one-liner** — a very long `ducktape --globals '' ...` +command that only existed inside a job definition. `ducktests-remote` takes that apart: +the inventory becomes `cluster.nodes`, the JSON blob becomes a profile, and the secrets +become `${env:}` placeholders resolved at launch and written to a `0600` file on the +runner instead of appearing on a command line in build logs. `--globals-json` and +`--cluster-file` exist so the blob can be moved over verbatim first and split up +afterwards; see [commands.md § run](commands.md#run). + +## What this tool deliberately does not do + +- **It is not configuration management.** `provision` covers the specific package / + directory / ssh-environment set the Dockerfile installs, and must not grow into Ansible. +- **It does not manage OS repositories.** `pip.*` configures pip; which apt/dnf mirrors a + VM talks to is set when the image is built. +- **It does not download a JDK from the internet.** It moves one you already have. +- **It takes no cluster lease.** One runner, one operator; queueing belongs to whatever + schedules the runs. Exit code 3 is reserved should that ever change. +- **It never removes a host key.** A changed host key is reported with the exact + `ssh-keygen -R` line for you to run after you have verified it. diff --git a/modules/ducktests/tests/ducktests_remote/docs/configuration.md b/modules/ducktests/tests/ducktests_remote/docs/configuration.md new file mode 100644 index 0000000000000..a51252c1e13d7 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/configuration.md @@ -0,0 +1,304 @@ + + +# Configuration reference + +Every default in this document comes from `DEFAULTS` in `config.py`. When the two +disagree, `config.py` is right and this file is a bug. + +## Layering + +Later layers win. Dicts deep-merge; **every other type, lists included, replaces +outright** — a later layer must be able to *shrink* a list, which is exactly what an +override is for. + +1. built-in defaults (`config.py`) +2. `~/.ducktests-remote/config.yaml` +3. `--config FILE`, repeatable, applied in order +4. `--profile NAME`, repeatable, applied in order +5. `DTR_*` environment variables +6. explicit command-line flags +7. `-g KEY=VALUE` / `-p KEY=VALUE` — `globals` and `parameters` only, applied by + `globals_builder` + +After merging, `${env:}` / `${file:}` placeholders are resolved everywhere except +`globals` and `parameters`, which `globals_builder` resolves per layer so its errors can +name the profile a placeholder came from. + +**Unknown keys are a hard error**, with a "did you mean" suggestion from `difflib`: + +``` +ERROR profile.yaml: unknown config key 'cluster.instal_root'; did you mean 'install_root'? +``` + +A typo silently ignored in a config that drives a three-hour run is expensive. The two +free-form sections, `globals` and `parameters`, are passed through unvalidated by design — +they belong to ducktape and to the tests, not to this CLI. + +Files are parsed as **YAML or JSON by content, not by extension**: a document starting with +`{` or `[` is parsed as JSON, anything else as YAML. + +### Profiles + +`--profile NAME` searches, in order: + +1. `/NAME.yaml`, `.yml`, `.json`, or `NAME` exactly +2. the same four under the current directory +3. `examples/profile-NAME.*` and `examples/NAME.*` shipped in the package + +A profile is a normal config document; it is not restricted to `globals`. + +### Environment variables + +- `DTR_CLUSTER__RUNNER=build-vm-01` → `cluster.runner`. A **double** underscore separates + path segments, so single underscores stay usable inside key names + (`DTR_RUN__MAX_PAYLOAD_MB` → `run.max_payload_mb`). +- Short aliases: `DTR_RUNNER` → `cluster.runner`, `DTR_JOBS` → `jobs`, `DTR_CLUSTER` → + `cluster.name`, `DTR_STATE_ROOT` → `cluster.state_root`, `DTR_INSTALL_ROOT` → + `cluster.install_root`, `DTR_USER` → `cluster.user`. +- `DTR_CONFIG` and `DTR_PROFILE` are ignored here (they select files, not values). +- **Any other single-segment `DTR_*` variable is ignored.** Profiles interpolate secrets + with `${env:DTR_SOMETHING}`, and treating every such variable as a config path would turn + a password into a config error. The `__` form is still validated, so a typo there is + caught. +- Values are parsed as JSON when they parse: `DTR_JOBS=8` is an integer, `DTR_RUNNER=vm` is + a string. + +## Sections + +### `cluster` + +| Key | Default | Meaning | +| --- | --- | --- | +| `name` | `default` | label only; appears in `meta.json` and `status` | +| `user` | your `$USER` | SSH account on the workers. Never defaults to `ducker` — that account exists only in the Docker image | +| `identity_file` | `~/.ssh/id_rsa` | private key **as the runner sees it**; this is the path written into `cluster.json` and opened by ducktape | +| `port` | `22` | default SSH port, per-host overridable | +| `install_root` | `/opt` | where distributions live on the workers; `/` is what ignitetest resolves a version to | +| `runner` | `local` | host running ducktape, or `local` | +| `state_root` | `~/.ducktests-remote` | runner-side root for the venv, sources and run directories | +| `nodes` | `[]` | the inventory; see below | +| `extra_hosts` | `[]` | hosts that `provision`/`deploy`/`clean`/`doctor` should also target but that ducktape must **not** schedule Ignite nodes onto — typically the runner itself | + +`identity_file` is the single most misread key. It is a **runner-side** path. If the runner +is not the coordinator, a file that exists on your laptop proves nothing; `doctor` checks +it on the runner and `keys push` installs it there. + +#### The inventory + +An entry is either a bare string or a mapping with `host` and optional `ip`, `user`, +`port`, `identity_file`: + +```yaml +nodes: + - host: worker[01-12].dc.local # expands to twelve hosts + - host: worker13.dc.local + ip: 10.0.0.13 # -> externally_routable_ip in cluster.json + - host: worker14.dc.local + user: someone-else # per-host override, for mixed clusters + - 10.0.0.15 # bare string is shorthand for {host: ...} +``` + +- **Ranges**: `[01-12]` zero-pads to the width of the lower bound, so `worker01`…`worker12`; + `[1-12]` gives `worker1`…`worker12`. Several ranges in one pattern expand as a cartesian + product, left to right. `ip` cannot be combined with a range. +- Duplicate hosts are an error. An unknown key inside an entry is an error. +- `-n/--num-nodes N` takes the **first N** entries, the analogue of + `IGNITE_NUM_CONTAINERS` in `docker/run_tests.sh`. Asking for more than the inventory + holds is an error naming both numbers. + +The generated `cluster.json` is exactly what `ducktape.cluster.json.JsonCluster` reads: +a `nodes` list of `{externally_routable_ip, ssh_config}`, where `ssh_config` is passed +into `RemoteAccountSSHConfig(host, hostname, user, port, password, identityfile)`. +`--cluster-file` bypasses generation entirely and uploads your file byte for byte. + +### `runner` + +| Key | Default | Meaning | +| --- | --- | --- | +| `venv` | `null` | explicit venv path on the runner. Unset means `/venv` | +| `python` | `python3` | interpreter used to create the venv and to probe for ducktape | +| `create_venv` | `true` | when false and `venv` is unset, no venv is used and ducktape is expected on `PATH` | +| `requirements` | `null` | requirements file; unset means `/modules/ducktests/tests/docker/requirements.txt`, so the ducktape pin has one source of truth | + +### `pip` + +Read only on the runner, by `run`: the venv install and `--install-sources`. Workers never +run pip. Full treatment in [commands.md § run](commands.md#run). + +| Key | Default | pip flag | +| --- | --- | --- | +| `index_url` | `null` | `--index-url` | +| `extra_index_url` | `[]` | `--extra-index-url`, repeated; a bare string is accepted | +| `trusted_host` | `[]` | `--trusted-host`, repeated; a bare string is accepted | +| `timeout` | `null` | `--timeout`, seconds | +| `retries` | `null` | `--retries` | +| `cert` | `null` | `--cert`; a **runner-side** path to a CA bundle | + +With nothing set, no flags are added at all and the rendered commands are byte for byte +what they were before this section existed. + +### `java` + +Read by `provision` (`jdk`, `ssh-env`) and by `doctor`. Full treatment in +[java.md](java.md). + +| Key | Default | Meaning | +| --- | --- | --- | +| `major` | `17` | Java major version the tests need. Derived from the Dockerfile's `ARG jdk_version="eclipse-temurin:17"`; the Dockerfile wins if they ever disagree | +| `home` | `null` | an explicit JDK home on the workers. Set means *exactly this*: no search, and a host without it fails | +| `search_paths` | `[/opt, /usr/lib/jvm, /usr/java]` | where to look for an existing JDK. Each entry may be a directory *of* JDK homes or a JDK home itself | +| `archive` | `null` | coordinator-side JDK tarball (`.tar.gz`/`.tgz`/`.tar`/`.tar.bz2`/`.tar.xz`) or an unpacked directory, delivered to hosts that have no matching JDK | +| `install_root` | `null` | where a delivered JDK is unpacked; unset means `cluster.install_root` | +| `name` | `null` | target directory name; unset means the name of the JDK home found inside the archive | +| `ssh_environment` | `true` | write `~/.ssh/environment` | +| `bashrc` | `true` | write a marked block at the top of `~/.bashrc` | + +Setting both `ssh_environment` and `bashrc` to false is an error: there would be no way to +put the JDK on the workers' non-interactive `PATH`. + +### `run` + +| Key | Default | Meaning | +| --- | --- | --- | +| `source_root` | `null` | Ignite checkout synced to the runner; unset means the checkout containing the current directory, found by walking up | +| `work_dir` | `null` | runner-side Ignite checkout to run from; ducktape itself runs in its `modules/ducktests/tests`; unset means the synced source directory | +| `exclude` | `[]` | extra sync exclusions, appended to the built-in list | +| `max_payload_mb` | `200` | refuse to sync more than this. A build directory leaking into the payload is the usual cause | +| `install_sources` | `false` | `pip install -e` the synced sources. Not needed for discovery: ducktape's loader walks up from each test file while `__init__.py` exists and puts the resulting top-level directory on `sys.path` | + +Built-in sync exclusions: `.git`, `target`, `results`, `__pycache__`, `*.pyc`, `.idea`, +`venv`, `.venv`, `*.egg-info`, `.tox`, `.pytest_cache`. A `.ducktestsignore` file at the +source root replaces the built-in list; `--exclude` replaces both. + +### `deploy` + +| Key | Default | Meaning | +| --- | --- | --- | +| `dist_dir` | `./dist` | one subdirectory per distribution | +| `install_root` | `null` | target root; unset means `cluster.install_root` | +| `sudo` | `false` | use `sudo -n` for the remote side | +| `owner` | `null` | `chown -R` the extracted tree | +| `staging_dir` | `/tmp/ducktests-remote-staging` | where `--via` parks the payload | +| `checksum` | `false` | hash file contents for the manifest instead of size+mtime | +| `exclude` | `[]` | rsync-style patterns left out of every distribution | +| `rsync` | `true` | send only what differs, hardlinking the rest against the deployment already on the host | + +`exclude` is empty by default, so a release directory is shipped whole. It earns its keep +when `ignite-dev` is a link to a source checkout, where only the built jars are wanted on +the workers; `--exclude` and a `.ducktests-deploy.ignore` file at the root of one +distribution override it, in that order. See +[commands.md](commands.md#ignite-dev-from-your-own-checkout) for the full recipe. + +`rsync` is the default transfer path: a redeploy sends only the files that changed and +hardlinks the rest against what the host already has. `deploy` falls back to a whole +tarball when either end has no rsync, when `--via` is used, or on a Windows coordinator — +see [commands.md](commands.md#incremental-redeploys). + +### `provision` + +| Key | Default | Meaning | +| --- | --- | --- | +| `packages` | see below | system utilities, derived from `docker/Dockerfile` | +| `install_jdk` | `false` | allow the distribution's JDK package as the last rung | +| `ssh_env_path_extra` | `[]` | extra `PATH` entries appended in `~/.ssh/environment` | +| `dirs` | `[/mnt/service]` | directories to create and chown; `install_root` is always added | + +Default packages: `sudo`, `netcat-traditional`, `iptables`, `rsync`, `unzip`, `wget`, +`curl`, `jq`, `coreutils`, `net-tools`. The Dockerfile is the source of truth; image-only +entries (`openssh-server`, `vim`, `mc`, build toolchain) are deliberately not replicated — +a real VM already has an sshd, and no compiler is needed to *run* tests. + +### `clean` + +| Key | Default | Meaning | +| --- | --- | --- | +| `process_pattern` | `org.apache.ignite` | `pgrep -f` pattern | +| `paths` | `[/mnt/service]` | directories to remove | +| `allowed_roots` | `[/mnt, /tmp, /var/tmp]` | every path in `paths` must sit under one of these | + +The default pattern covers all four main classes ignitetest launches: +`CommandLineStartup`, `CdcCommandLineStartup`, `IgniteAwareApplicationService`, +`KafkaToIgniteCommandLineStartup`. + +### `ssh`, `jobs`, `profiles_dir` + +| Key | Default | Meaning | +| --- | --- | --- | +| `ssh.connect_timeout` | `15` | seconds, passed as `ConnectTimeout` to every connection | +| `jobs` | `16` | fan-out parallelism | +| `profiles_dir` | `~/.ducktests-remote/profiles` | where `--profile` looks first | + +### `globals` and `parameters` + +Free-form. `globals` becomes ducktape's `--globals` payload; `parameters` becomes +`--parameters`. Neither is validated by this CLI, both are deep-merged like everything +else, and both are written to `0600` files on the runner rather than onto a command line. + +`-g` / `-p` take dotted overrides whose values are parsed as JSON when they parse, so +`-g ssl.enabled=true` is a boolean and `-g project=ise` is a string. This mirrors +`_extend_json` in `docker/run_tests.sh`, with nesting added. + +## Placeholders + +`${env:NAME}` and `${file:PATH}` are resolved **on the coordinator, at launch**, in every +section: + +```yaml +cluster: + user: ${env:DTR_SSH_USER} +pip: + index_url: ${env:NEXUS_URL} +globals: + authentication: + password: ${file:~/.secrets/ise-password} +``` + +A missing variable or an unreadable file is a hard error naming both the placeholder and +where it came from — never an empty string, never a run that fails on authentication three +hours later. `${file:}` strips surrounding whitespace, which is what you want for a file +holding one token. + +## Secrets and redaction + +- Every value resolved from `${env:}` or `${file:}` is registered with a `Redactor` and + replaced by `***` in everything the CLI prints: normal output, `--dry-run`, streamed + logs, warnings, error messages, and the config summary inside `meta.json`. +- Redaction is keyed on the **value**, so the same secret is caught in an unrelated field + or in a rendered command line. Key-name matching (`password`, `passwd`, `secret`, + `token`, `keystore_pass`, `truststore_pass`) is only a fallback for values the CLI never + resolved itself. +- Values shorter than three characters are not registered — masking `a` everywhere would + destroy the output. +- `pip` index URLs get a second, independent treatment: any `user:password@` in a URL is + masked wherever the CLI prints it, even when the URL was typed straight into a config + file and the redactor has never seen it. +- The composed `globals.json` is written `0600` on the runner, and `fetch` always excludes + it. +- The example profiles in `examples/` contain no real hostnames, addresses, accounts or + passwords, and must stay that way: this directory is in a public Apache repository. + +## Seeing the result + +```bash +ducktests-remote run --dry-run -t ./ignitetest/tests/smoke_test.py +``` + +prints the composed `cluster.json`, the redacted `globals.json`, and the rendered `run.sh` +without creating, uploading or starting anything. `doctor` additionally prints the +effective pip index and the resolved identity path. When two configurations disagree, diff +their `--dry-run` output rather than reasoning about the layering. diff --git a/modules/ducktests/tests/ducktests_remote/docs/index.md b/modules/ducktests/tests/ducktests_remote/docs/index.md new file mode 100644 index 0000000000000..4012608f1c607 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/index.md @@ -0,0 +1,82 @@ + + +# ducktests-remote manual + +`../README.md` is the tour: install it, describe a cluster, get a run going. This is the +reference: every command, every configuration key, and — the part a README cannot carry — +**exactly what happens in what order, and what crosses the network when**. + +The organising rule of these documents: when the answer is "it depends", the condition is +written down. If you find yourself guessing, that is a documentation bug worth fixing. + +## The documents + +| Document | What it answers | +| --- | --- | +| [concepts.md](concepts.md) | Which machine is which, what lives where, and the invariants the whole tool rests on | +| [configuration.md](configuration.md) | Every config key, its default, who reads it; layering, `${env:}`, secrets | +| [commands.md](commands.md) | Every command and flag, and the exact order of operations inside each | +| [java.md](java.md) | How the workers' JVM is chosen, and precisely when a JDK is copied to them | +| [runs.md](runs.md) | The run directory, run states, detach/follow/stop, exit codes | +| [troubleshooting.md](troubleshooting.md) | Symptom → cause → fix, for the failures this cluster actually produces | +| [internals.md](internals.md) | Module map, the transport contract, fan-out, redaction; where to change things | + +## Find an answer fast + +**"When does X get sent to the workers?"** +Nothing is ever sent implicitly. Three commands transfer files, and nothing else does: +`deploy` (distributions), `provision --only jdk` (a JDK, and only to hosts that lack one), +`keys push` (the public key). `run` uploads only to the *runner*. See +[commands.md § What each command transfers](commands.md#what-each-command-transfers). + +**"Why did my test get the wrong Java?"** → [java.md](java.md), and in particular +[§ Why PATH decides, not JAVA_HOME](java.md#why-path-decides-not-java_home). + +**"Where does this setting come from?"** → [configuration.md § Layering](configuration.md#layering) +then `--dry-run`, which prints the composed result of every layer. + +**"What does this command actually do?"** → [commands.md](commands.md). Each entry lists +its steps in execution order, what it reads, what it writes, and what `--dry-run` skips. + +**"The run ended — where is everything?"** → [runs.md § The run directory](runs.md#the-run-directory). + +**"Is it safe to run twice?"** Yes, for every command; the per-command entry says how +idempotence is achieved. `clean` and `stop` are the two that destroy things, and both have +a `--dry-run` that prints the exact kill list and path list. + +**"What does this exit code mean?"** → [runs.md § Exit codes](runs.md#exit-codes). + +## Two habits worth having + +**`--dry-run` first.** It is genuinely side-effect free on every command: no probes, no +uploads, no processes. It prints the commands it would run and the files it would generate, +including the rendered `run.sh`, `cluster.json` and a redacted `globals.json`. Most +questions about "what would this do" are answered faster by running it than by reading. + +**`-v` when a message is not enough.** Verbose prints every command as it is issued +(`+ host$ ...`) and the full per-host output rather than only the failures. + +## Conventions in these documents + +- **coordinator**, **runner**, **worker** are used precisely; see + [concepts.md](concepts.md#the-three-roles). A sentence that does not name one of them is + about all three. +- Paths like `services/utils/path.py` are relative to `modules/ducktests/tests/ignitetest/`; + paths like `commands/run.py` are relative to `modules/ducktests/tests/ducktests_remote/`. +- Code references name a file and a function rather than a line number, so they survive + edits: "`_deliver_jdk` in `commands/provision.py`". diff --git a/modules/ducktests/tests/ducktests_remote/docs/internals.md b/modules/ducktests/tests/ducktests_remote/docs/internals.md new file mode 100644 index 0000000000000..51b5c4d06996d --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/internals.md @@ -0,0 +1,201 @@ + + +# Internals + +For changing the CLI rather than using it. + +## Module map + +| Module | Responsibility | +| --- | --- | +| `cli.py` | argument parsing, the shared `Context`, `Console`, exit-code mapping | +| `config.py` | defaults, layering, validation, `${env:}`/`${file:}` interpolation | +| `globals_builder.py` | the `--globals` payload and the `Redactor` | +| `cluster.py` | inventory → `Node` list → ducktape's `cluster.json` | +| `transport.py` | the only place that shells out to `ssh`/`scp` | +| `fanout.py` | bounded parallel per-host execution and the result table | +| `progress.py` | the live per-host display for transfers that take minutes | +| `runs.py` | run ids, run directory layout, state derivation, script rendering | +| `sshdiag.py` | SSH failure classification and the administrator block | +| `java.py` | JDK discovery, environment files, archive inspection | +| `pipconf.py` | pip index configuration → command-line flags | +| `commands/*.py` | one module per subcommand, each with `register()` and `execute(ctx)` | +| `templates/run.sh.tmpl` | the generated ducktape command | +| `checks/` | unit checks; no network, no Docker, no ducktape | + +`pipconf` is not called `pip` so that it can never shadow the real `pip` package for +anything that ends up with this directory on `sys.path`. + +## Context + +`Context` (in `cli.py`) is what every command receives. It holds the composed config, the +parsed args and the console, and lazily builds transports: + +- `ctx.nodes` — inventory, truncated by `-n` +- `ctx.all_nodes` — inventory plus `extra_hosts`; used by `provision`, `deploy`, `clean`, + `doctor`, `keys` +- `ctx.runner` — a transport to the runner, created once +- `ctx.worker(node)` — a transport per worker, cached by host +- `ctx.state_root_resolved()` — the state root with `~` expanded against the *runner's* home + +A subtlety worth knowing: `cluster.identity_file` is a runner-side path, so the coordinator +uses it only when a file of that name happens to exist locally; otherwise the system ssh +client falls back to `~/.ssh/config` and the agent. + +## Transport + +Three implementations behind one interface: + +| Class | Used for | +| --- | --- | +| `LocalTransport` | `--runner local`, and anything else on this machine | +| `SshTransport` | a remote runner or a worker, through the **system** `ssh`/`scp` | +| `ProxiedTransport` | a second hop: `deploy --via`, and probing a worker over the connection ducktape itself will make | + +The interface is `run(argv)`, `run_script(script)`, `upload`, `download`, `upload_dir`, +`exists`, `mkdirs`, `write_file`, `read_file`, `home`, `expand`. + +Design decisions that are load-bearing: + +- **`argv` is always a list, never a shell string.** Quoting is done once, in the transport. +- **`run_script` feeds a script to `bash -s` over stdin.** Long quoted one-liners are the + single largest source of remote-execution bugs; anything longer than a couple of words + belongs in a script. +- **The system ssh client, not paramiko.** It brings `~/.ssh/config`, `ProxyJump`, agent and + Kerberos support for free, and it is the same client an engineer uses by hand when + reproducing a failure. +- **Connection multiplexing** (`ControlMaster`) is enabled everywhere except Windows, where + OpenSSH has no support for it and the options fail hard. `doctor` and `deploy` open many + connections per host; multiplexing makes every extra one a free channel. +- **`write_file` writes bytes with explicit LF endings.** A coordinator on Windows would + otherwise translate every newline to CRLF, and a shell script with carriage returns fails + on the runner with a message naming the wrong line. +- **`expand`** resolves a leading `~` against the *remote* home, once per connection, because + paths are shell-quoted before they reach the remote side and a literal tilde would never + be expanded there. +- **`has_rsync()` is probed once per transport and cached.** Both the source sync and + `deploy`'s incremental path ask for it; a mixed cluster where one host lacks rsync falls + back per host rather than for the whole run. `deploy` does not route rsync through the + transport, though — it runs rsync on the coordinator with the transport's own + `ssh_options()` in `-e`, because the payload never passes through a shell. + +## Fan-out + +`fanout(hosts, operation, jobs=…, fail_fast=…)` runs `operation(host)` in a bounded thread +pool and returns results **in inventory order**. An exception inside one operation becomes +that host's `FAILED` result rather than killing the batch — per-host isolation is the whole +point. Statuses: `ok`, `changed`, `skipped`, `warn`, `failed`. + +`render_table` prints failures' detail by default and everything's detail with `-v`; +`summarise` prints `9 ok, 2 failed`, ordered worst-first. + +## Progress + +`Progress` is written to from the fan-out threads and drawn by one daemon thread of its +own, so a slow terminal cannot slow a transfer down and an update never blocks on a +redraw. Three rules it does not break: + +- **It never raises.** A display that takes a deploy down with it would be worse than no + display, so a stream that cannot be written to just turns the display off. +- **The block is erased before anything else prints.** `deploy` renders the result table + after the `with progress:` body, on the lines the block occupied. +- **`NullProgress` has the same surface.** Callers never test for `None` or for whether + the display is on; `--quiet`, `--dry-run` and `--no-progress` simply swap the object. + +Byte counts come from `--info=progress2` (rsync) and from `upload_watched`, which streams +a file into `ssh 'cat > path'` because scp suppresses its own meter when its output is a +pipe. `run_local_streaming` reads a child's output as it arrives, treating a carriage +return as a line ending, and drains stderr on a thread so a chatty failure cannot deadlock +the pipe. + +`build_progress(ctx, hosts)` is the single decision point, shared by `deploy`, +`run`'s source sync and `provision`'s JDK delivery. Callers gate the wiring on +`progress.watching` rather than on which object they got: reporting costs something — +rsync grows a meter, an upload leaves scp for a streamed `cat` — and none of it is worth +paying when the display is a `NullProgress`. + +## Redaction + +`Redactor` keys on resolved **values**, not on key names. Anything coming out of `${env:}` +or `${file:}` is registered at config load, and `Console` runs every line through it, so +redaction cannot be bypassed by printing from an unusual place. `redact_structure` also +masks a small set of sensitive key names as a fallback for values the CLI never resolved +itself. Independently, `pipconf.mask_credentials` masks `user:password@` in URLs. + +If you add a new output path, print through `Console` — not `print`. + +## Adding a command + +1. Create `commands/.py` with `register(subparsers, common)` and `execute(ctx)`. +2. Add it to the import and the loop in `cli.build_parser`. The order there is the order in + `--help`. +3. Return one of the `EXIT_*` constants from `cli.py`. +4. Support `--dry-run` honestly: print what would happen, execute nothing, probe nothing. +5. Route every remote action through `ctx.runner` / `ctx.worker(node)`, and every host loop + through `fanout`. +6. Add checks under `checks/check_remote_.py` using `FakeTransport`. + +## Conventions + +- ASF licence header on every file, Python and Markdown alike. +- flake8, max line length **120**; config in `tox.ini`. +- Docstrings in the `:param:` / `:return:` style used throughout. +- Comments explain *why*, not *what*. The reason a decision was made is the part that + cannot be recovered from the code later. +- No new runtime dependency. The package uses the standard library plus `PyYAML`, and + **never imports ducktape**. + +## Tests + +```bash +cd modules/ducktests/tests +pytest ducktests_remote/checks # unit only: no network, no Docker, no ducktape +flake8 ducktests_remote +``` + +`[pytest]` in `tox.ini` collects `check_*.py` files, `Check` classes and `check_*` +functions, which is why the files are named that way. `checks/fake_transport.py` provides a +recording transport that simulates a small filesystem and returns canned output for +commands matching a needle. Two kinds of subprocess are deliberate: the one that proves +the ducktape import boundary still holds, and the short `python -c` scripts that +`check_remote_progress.py` uses to prove output really is read as it arrives. Nothing +else spawns anything. + +Checks are named as sentences — `check_lists_replace_and_do_not_concatenate` — because the +name is the specification and shows up in the failure output. + +## Where the ignitetest facts come from + +Several behaviours are pinned to things in `ignitetest` rather than invented here. When +those move, these move: + +| Fact | Source | +| --- | --- | +| distribution home is `/` | `services/utils/path.py`, `services/utils/ignite_aware.py` | +| version strings normalise (`ise--6` → `ise-6`) | `utils/version.py` | +| `persistent_root` defaults to `/mnt/service` | `services/utils/path.py` | +| `sudo iptables` is used by exactly two suites | `IgniteAwareService.drop_network` | +| the four Ignite main classes `clean` must match | `services/ignite.py`, `services/ignite_app.py`, `services/utils/cdc/*` | +| Java major parsing | `services/utils/jvm_utils.py` | +| which Java consumers use `PATH` vs `JAVA_HOME` | `services/utils/ignite_spec.py`, `jvm_utils.py`, `kafka/kafka.py`, `jmx_utils.py` | +| the package list and the JDK version | `docker/Dockerfile` | +| the ducktape pin | `docker/requirements.txt` | + +`docker/Dockerfile` is the source of truth for what a prepared node looks like. Where it +and this CLI disagree, the Dockerfile wins — and the derived lists in `config.py` carry a +comment saying so, to keep the drift visible at the next review. diff --git a/modules/ducktests/tests/ducktests_remote/docs/java.md b/modules/ducktests/tests/ducktests_remote/docs/java.md new file mode 100644 index 0000000000000..01ba4b67caac5 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/java.md @@ -0,0 +1,227 @@ + + +# Choosing the workers' JVM + +## Why PATH decides, not JAVA_HOME + +`ignitetest` reaches a JVM four different ways, and only one of them respects `JAVA_HOME`: + +| Consumer | Mechanism | +| --- | --- | +| `ignite.sh`, via `IgniteSpec.envs()` → `export …;` | honours `JAVA_HOME` | +| `jvm_utils.java_version()` → `java -version` over ssh | bare `java`, so **PATH** | +| `services/kafka/kafka.py` → `nohup java …` | bare `java`, so **PATH** | +| `jmx_utils` → `java -jar jmxterm.jar` | bare `java`, so **PATH** | + +All four run over **non-interactive** SSH, where `~/.profile` is never sourced. So setting +`JAVA_HOME` alone changes what `ignite.sh` uses and nothing else, and a `java` that works +perfectly when you log in by hand can be absent or wrong during a test run. Both are set, +and — because neither mechanism is guaranteed to work at a given site — what a fresh +non-interactive session actually gets is then *measured*. + +This is also why `doctor` judges the JVM on `PATH` rather than the JDK it can find: a +perfect Java 17 sitting in `/opt` that the non-interactive `PATH` does not point at is not +the JVM the suite will run under. + +The runner needs no JVM at all. ducktape is pure Python. Java is purely a worker concern. + +## The resolution ladder + +Run by `provision --only jdk`, per host. Rungs 1–3 are pure discovery — one round trip, +no writes — which is why `doctor` and `provision --only ssh-env` can run the identical +script. + +**1. `java.home` is set.** Verify `$home/bin/java`. If it is missing or unusable on a host, +that host **fails, by name**. There is no fallback: an explicit `java.home` that silently +resolves to a different JVM would defeat the point of saying it. This is checked before +anything else, so an explicit home is never overridden by a lucky match elsewhere. + +**2. The JVM already on the non-interactive PATH matches `java.major`.** Use it as is, +report `ok`. A correctly prepared VM never gets touched. + +**3. A JDK under `java.search_paths` matches.** Each search path is probed both as a +directory *of* JDK homes (`/usr/lib/jvm/*/bin/java`) and as a JDK home itself +(`/opt/bin/java`), so both layouts work. Every candidate's `java -version` is parsed the +same way `jvm_utils.java_major_version` does: `1.8.0_292` → 8, `11.0.19` → 11, +`17.0.11+9` → 17. Among candidates of the requested major, the **highest patch level** +wins, compared numerically — string sorting would put `17.0.9` above `17.0.11` — and the +path breaks ties, so the choice is deterministic. + +**4. `java.archive` is delivered.** Only now, and only to the hosts that reached this rung. +Details below. + +**5. Nothing worked.** The host fails with the list of every JDK that *was* found and the +config keys that would fix it. With `--install-jdk` **and** `--sudo`, the distribution's +own `openjdk-N-jdk` / `java-N-openjdk-devel` package is attempted first as a last rung. + +## When the archive is actually sent + +The single question this section exists to answer. + +**Only `provision`, step `jdk`, ever sends a JDK.** Not `doctor` (it never mutates +anything), not `ssh-env` (it only points `PATH` at a JDK that is already there), not `run`, +not `deploy`. + +```bash +ducktests-remote provision --only jdk # just this step +ducktests-remote provision --sudo # all steps; jdk runs before ssh-env +``` + +Within that step: + +| Phase | Where | What happens | +| --- | --- | --- | +| 1 | coordinator, once | `java.archive` is opened with `tarfile`, searched for `bin/java`, and its size printed. **A bad archive fails here**, before a byte moves. A directory is packed here too, once for the whole run | +| 2 | each host, in parallel | the discovery script runs rungs 1–3 | +| 3 | each host that found nothing | delivery, below | + +A host reaches delivery only when **all** of these hold: + +- rungs 1–3 found no JDK of `java.major` on it, **and** +- `java.home` is unset — if it is set and missing, that is a failure, never a delivery, **and** +- `java.archive` is set — otherwise the host fails with the "set `java.archive` or + `java.home`" message. + +So on a twelve-host cluster where eleven already carry Java 17, exactly one upload happens. + +### What delivery does on the host + +1. Read `.ducktests-java.json` in the target directory. If its hash matches the archive, + **skip** — nothing is uploaded. `--force` overrides. +2. Check `install_root` is writable; if not and `--sudo` was not passed, fail naming the + directory and the account. +3. Create a staging directory beside the target, upload the archive into it, and unpack — + with `--strip-components` set to whatever wraps the JDK home (1 for a stock Temurin + tarball) and the decompression flag taken from the file's suffix. +4. Verify `bin/java` exists in the staging tree; if not, fail. +5. Write the manifest, then **swap** the staging tree into place and delete the old one. + +The staging-and-swap is `deploy`'s, reused rather than reimplemented: a half-extracted JDK +that looks present is exactly as bad as a half-extracted distribution, and a staging tree +that is not swapped in is discarded the same way. + +### Archive formats + +`.tar.gz`, `.tgz`, `.tar`, `.tar.bz2`/`.tbz2` and `.tar.xz`/`.txz` are all accepted; the +`tar` flag on the worker follows the suffix. + +The JDK home is *located* rather than assumed: the shallowest `bin/java` in the archive +wins, and everything above it is stripped. A second, deeper `bin/java` (a bundled JRE) +cannot pull the depth with it. + +| Given | Result | +| --- | --- | +| one top-level dir, `jdk-17.0.11+9/bin/java` | stripped by 1; target defaults to that dir's name, e.g. `/opt/jdk-17.0.11+9` | +| flat, `bin/java` at the root | not stripped; target defaults to the file name without its suffix | +| wrapped deeper, `openjdk-17/jdk-17.0.11+9/bin/java` | stripped by 2; target defaults to `jdk-17.0.11+9` | +| extra entries beside the JDK (a stray `LICENSE`, AppleDouble `._*` files) | ignored; they do not change where `bin/java` is | +| an unpacked directory containing `bin/java` | packed once on the coordinator and sent to every host that needs it | +| no `bin/java` anywhere | **refused on the coordinator**, naming the archive | +| a macOS build (`Contents/Home` in the way) | **refused on the coordinator** — it unpacks fine and then fails on every worker | +| `.zip` | **refused**, naming `.tar.gz`. Linux JDKs ship as tarballs, and a zip branch would be untested code on every real run | + +`java.name` overrides the target directory name; `java.install_root` overrides where it +goes (defaulting to `cluster.install_root`). + +### Cost + +The archive is uploaded from the coordinator to each host that needs it. `provision` prints +the archive size up front and, above 200 MB with more than three hosts, warns with the +worst-case total. `provision` has no `--via`; if that total is painful, a good pattern is +to deliver to one machine, or to place the JDK in your `deploy` dist directory once and +point `java.home` at the result. + +## Making the choice stick + +The `ssh-env` step writes **both** files, from one resolved value, in one step, so they +cannot drift apart: + +**`~/.ssh/environment`** — what the Dockerfile does. Contains `JAVA_HOME`, a `PATH` with +`$JAVA_HOME/bin` **first** (plus `provision.ssh_env_path_extra`), and `LANG=C.UTF-8`. +Silently ignored unless sshd carries `PermitUserEnvironment yes`; the step says so when it +is absent. The existing `PATH` is filtered of the entries about to be prepended, so running +the step repeatedly neither grows the variable nor reports a change for ever. + +**`~/.bashrc`** — a block between `# BEGIN ducktests-remote` and `# END ducktests-remote` +at the **top** of the file, above the `case $- in *i*) ;; *) return;; esac` guard the stock +Ubuntu file opens with. That guard exists precisely because bash *does* source `~/.bashrc` +for non-interactive ssh commands, so appending would be writing to `/dev/null`. Only the +block between the markers is ever rewritten. This does nothing when the account's login +shell is not bash. + +Either can be switched off with `java.ssh_environment: false` / `java.bashrc: false`; +switching off both is an error. + +**Then it verifies.** A fresh non-interactive connection runs `java -version` and reads +`$JAVA_HOME`. That result is the outcome of the step: if the session still gets the wrong +JVM, the step fails there — with the advice that sshd may be ignoring `~/.ssh/environment` +*and* the login shell may not be bash — rather than reporting success and letting a test +fail three hours later. + +## What `doctor` says + +| Situation | Verdict | Message | +| --- | --- | --- | +| no `java` on the non-interactive PATH | **FAIL** | run `provision --only jdk --only ssh-env` | +| PATH java's major ≠ `java.major`, but a matching JDK is on the host | **FAIL** | names the JDK it found and the command that would point PATH at it | +| PATH java's major ≠ `java.major`, nothing matching installed | **FAIL** | lists what was found; names `java.archive` and `java.home` | +| major matches, but an explicit `java.home` is not the JDK in effect | **WARN** | the tests will run on the right version; the pin is simply not active | +| major matches | **OK** | version and resolved path | +| hosts disagree on version | **WARN** on a `java-consistency` row | majority version and the outliers | + +A FAIL blocks `run` at preflight with exit 2. `--skip-preflight` remains the escape hatch, +and `java.major: null` disables the version requirement entirely (whatever is on the host +is then accepted). + +## Worked scenarios + +**VMs already have the right JDK.** `java.major: 17`, nothing else. Rung 2 matches +everywhere, `ssh-env` makes sure non-interactive sessions see it, no transfers at all. + +**A JDK sits in `/opt/jdk-17.0.11` but `java` is 11.** Rung 3 finds it. +`provision --only jdk --only ssh-env` selects it and points `PATH` at it. No transfers. + +**Fresh VMs, JDK pre-downloaded on your machine.** + +```yaml +java: + major: 17 + archive: ~/jdk/OpenJDK17U-jdk_x64_linux_hotspot.tar.gz +``` + +```bash +ducktests-remote provision --dry-run --only jdk # validates the archive, prints the size +ducktests-remote provision --only jdk --only ssh-env +``` + +Rung 4 delivers to every host, unpacks to `/opt/jdk-17.0.11+9`, and `ssh-env` points at it. +Re-running transfers nothing: the manifest matches. + +**A vendor JDK you must use exactly.** `java.home: /opt/corp-jdk-17`. Rungs 2–4 are skipped +entirely; a host without that directory fails by name. + +**Mixed cluster, some hosts short.** Set both `java.major` and `java.archive`. Hosts that +already have a match are untouched; the rest are delivered to. One command, one honest +per-host table. + +## Configuration recap + +See [configuration.md § java](configuration.md#java) for the full table. The short version: +`major` is what the tests need, `home` is an exact pin, `search_paths` is where to look, +`archive` is what to fall back on, and `ssh_environment`/`bashrc` control how the answer is +made to stick. diff --git a/modules/ducktests/tests/ducktests_remote/docs/runs.md b/modules/ducktests/tests/ducktests_remote/docs/runs.md new file mode 100644 index 0000000000000..13611e5bab583 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/runs.md @@ -0,0 +1,143 @@ + + +# Runs + +## The run directory + +Everything about a run lives on the runner, under +`/runs//`: + +| File | Written by | Contents | +| --- | --- | --- | +| `meta.json` | `run`, at launch | run id, CLI version, coordinator user/host/platform, runner, cluster name, node list, test paths, work dir, results root, start time, and a redacted config summary | +| `cluster.json` | `run` | exactly what ducktape was given | +| `globals.json` | `run`, mode `0600` | the composed globals, secrets resolved | +| `parameters.json` | `run`, mode `0600` | present only when `parameters` is non-empty | +| `run.sh` | `run`, mode `0755` | the exact ducktape command. SSH in and execute it to reproduce the run by hand | +| `launch.sh` | `run`, mode `0755` | wrapper that waits on `run.sh` and records the exit code | +| `pid`, `pgid` | `launch.sh` / the detach script | what `stop` signals | +| `exit_code` | `launch.sh`, last | written after ducktape exits | +| `stopped` | `stop` | marker that distinguishes "stopped" from "failed" | +| `ducktape.log` | `launch.sh` | combined stdout and stderr | +| `results/` | ducktape | `--results-root`, with ducktape's own `latest` symlink inside | + +Alongside: `/runs/latest` points at the newest run, and +`/src//` holds that run's synced sources. + +Run ids look like `max-20260727-141233-9f2a`: the account and timestamp make a directory +listing readable, and four hex characters keep two runs started in the same second apart. + +## Reproducing a run by hand + +```bash +ssh build-vm-01 +cd ~/.ducktests-remote/runs/max-20260727-141233-9f2a +cat run.sh # every path is shell-quoted; nothing is hidden +bash run.sh +``` + +`run.sh` activates the venv and execs ducktape with `--results-root`, `--cluster-file`, +`--globals` and the test paths. `--globals` takes a *file path*: ducktape 0.13 checks +`os.path.isfile` before parsing the argument as JSON, so the composed blob never crosses a +command line and never lands in shell history or a process listing. + +## Run states + +`status` derives the state from three observable facts — is the pid alive, is there an +`exit_code`, is there a `stopped` marker: + +| State | Meaning | +| --- | --- | +| `running` | pid alive, no exit code yet | +| `finished` | exit code 0 | +| `failed` | non-zero exit code, no `stopped` marker | +| `stopped` | exit code present *and* a `stopped` marker, or the marker with no live pid | +| `unknown` | no pid, no exit code, no marker — usually a run that died before `launch.sh` got going | + +`exit_code` wins over liveness because the file is written last: a process that has already +exited is never reported as running just because its pid got reused. + +## Detach, follow, reattach + +`run` detaches from second zero — `setsid nohup bash launch.sh`, with `nohup` plus `disown` +where `setsid` is missing — and then attaches to the log. `setsid` matters: it puts +ducktape in its own session so a dropped SSH connection cannot SIGHUP it and leave Ignite +JVMs alive on every worker. + +Consequences: + +- **Ctrl-C detaches; it does not stop the run.** A second Ctrl-C within 3 seconds offers to + stop it (and simply detaches in a non-interactive shell). +- `--detach` skips following entirely and prints the reattach commands. +- `logs -f` reattaches from **any** coordinator, including one that did not start + the run. +- Following is just streaming a file by byte offset, so it costs the runner nothing and can + be interrupted freely. + +## Stopping + +`stop` touches `stopped`, SIGTERMs the process **group**, waits `--timeout` seconds +(default 60), and with `--kill` SIGKILLs survivors. If the process never wrote an exit code, +`143` is recorded. Then it cleans the workers, unless `--no-clean`. + +The follow-up clean is the important half: killing ducktape does not kill the Ignite JVMs +it started on twelve machines, and those are what break the *next* run. See +[commands.md § clean](commands.md#clean). + +## Exit codes + +| Code | Meaning | +| --- | --- | +| `0` | success | +| `1` | usage or configuration error — a bad key, a missing file, an unset `${env:}` | +| `2` | preflight failed; `run` stopped before creating anything | +| `3` | reserved: this deployment has a single runner and takes no cluster lease | +| `4` | ducktape ran and reported test failures | +| `5` | transport or infrastructure error | +| `130` | interrupted by the operator | + +`4` and `5` are deliberately distinct: Jenkins needs "tests failed" separate from "the +cluster is broken". Note that ducktape itself exits `1` both for test failures and for its +own startup errors, so a `4` strictly means "ducktape ran and exited non-zero" — the log +distinguishes the two. + +## Getting results back + +```bash +ducktests-remote fetch # newest run, reports only +ducktests-remote fetch --full # the whole results tree +``` + +Results land in `./ducktests-results//`, with `ducktape.log` beside them. +`globals.json` is always excluded. + +The results also stay on the runner indefinitely — nothing is pruned automatically. On a +long-lived runner, `/runs` and `/src` grow one directory per run; +`status --all` lists them, and removing old ones is an ordinary `rm -rf` on the runner. + +## Jenkins + +```bash +ducktests-remote --profile ise-perf run -t "$TC_PATHS" --detach +ducktests-remote status --json > status.json +``` + +`--detach` returns immediately with the run id; `status --json` is machine-readable; the +exit code separates test failures from infrastructure failures. Because run state lives on +the runner, a job that times out and is retried can pick the same run back up rather than +starting a second one on top of it. diff --git a/modules/ducktests/tests/ducktests_remote/docs/troubleshooting.md b/modules/ducktests/tests/ducktests_remote/docs/troubleshooting.md new file mode 100644 index 0000000000000..a427c39580796 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/docs/troubleshooting.md @@ -0,0 +1,173 @@ + + +# Troubleshooting + +Start with `ducktests-remote doctor`. It probes everything in parallel, never stops at the +first failure, and changes nothing, so it is always safe to run — including on a cluster +someone else is using. + +## The classic failures + +### Stale Ignite JVMs + +The single most common source of baffling failures on a shared cluster: a previous run was +killed and its JVMs still hold ports and `/mnt/service`. Tests fail in ways that have +nothing to do with the change you are testing. + +`doctor` reports it as a FAIL with the host list. + +```bash +ducktests-remote clean --dry-run # prints every pid and command line it would kill +ducktests-remote clean +``` + +### `java: command not found`, or the wrong JDK, deep inside a test + +Non-interactive SSH does not source `~/.profile`, so the `java` you get when you log in by +hand is not necessarily the one the tests get. + +```bash +ducktests-remote provision --only jdk --only ssh-env +``` + +If it still reports the wrong JVM afterwards, sshd is ignoring `~/.ssh/environment` **and** +the login shell is not bash. Set `java.home` to a JDK the site already puts on the default +`PATH`. Full detail in [java.md](java.md). + +### `identity_file` does not exist on the runner + +`cluster.identity_file` is the path *ducktape* will open, **on the runner**. A file that +exists on your laptop proves nothing. `doctor` checks it there and reports its mode; +`keys push` installs it. + +### Agent forwarding does not survive a detached run + +`ssh -A` gives you an agent for the lifetime of your session; a run that lasts hours +outlives it, and every subsequent worker connection then fails — typically after the first +few minutes, which makes it look like an intermittent cluster problem. Use a real key file +on the runner (`keys push`). + +### Discovery failures with no useful message + +Workers that cannot resolve each other's hostnames fail inside Ignite discovery, far from +the cause. `doctor` runs an N-way resolution probe from one worker. + +```bash +ducktests-remote provision --sudo --write-hosts # escape hatch when cluster DNS cannot be fixed +``` + +### `deploy` wants to send a gigabyte of `ignite-dev` + +The distribution is a link to a checkout and nothing is being filtered. The workers need +`modules/*/target/*.jar`, `modules/*/target/libs/*.jar`, `bin/` and +`modules/ducktests/tests/certs`; `.git` and every `src` tree are ballast. Set +`deploy.exclude` (or `--exclude`, or a `.ducktests-deploy.ignore` at the root of the +distribution) and confirm with `--dry-run`, which prints the payload size and how many +files the patterns dropped. The recipe is in +[commands.md](commands.md#ignite-dev-from-your-own-checkout). + +Do not reach for `.ducktestsignore` here — that file is the *source sync* list and its +patterns are the opposite ones. + +### "source payload is N MB, above the limit" + +A build directory leaked into the sync. Distributions go through `deploy`, never through +the source sync. Adjust `--exclude`, add a `.ducktestsignore` at the source root, or raise +`run.max_payload_mb` if the payload really is that big. + +### The venv cannot be prepared + +The message names the index it tried. On a network without PyPI access, set `pip.index_url` +(see [configuration.md § pip](configuration.md#pip)) or point `runner.venv` at an +environment that already has ducktape. `doctor` prints the effective index on the runner +row, with credentials masked. + +### Tests are skipped as un-runnable + +Most `ignitetest` suites declare `@cluster(num_nodes=...)` above three. An inventory smaller +than the largest declaration means those tests are never scheduled. `run` warns when the +inventory is below three hosts; ducktape's own report lists what it skipped and why. + +### No distribution for a version + +`doctor` reports a WARN listing what it *did* find under `install_root`. Remember that +ignitetest normalises version strings: `ise--6` maps to `/opt/ise-6`. See +[commands.md § deploy](commands.md#where-the-directory-names-come-from). + +## SSH failure classes + +Every failed connection is classified and mapped to a concrete next action, because the +first-time experience is usually not a subtle bug but "nobody has added me to these +machines yet". + +| Class | Meaning | Next action | +| --- | --- | --- | +| `unresolved` | the hostname does not resolve from here | check VPN/DNS, or put the address in `cluster.nodes[].ip` | +| `no-sshd` | the host answers but nothing listens on the port | sshd is down or on another port | +| `unreachable` | no network path | firewall, routing, or the host is down | +| `no-access` | your key is not authorised for that account | `keys push`, or the administrator block | +| `no-user` | the account does not exist there | ask for it, or use a per-host `user` override | +| `hostkey` | the host key changed | **verify first**, then run the printed `ssh-keygen -R` yourself; this tool will never remove a host key for you | +| `no-sudo` | passwordless sudo is missing | only the two network-segmentation suites need it | +| `unknown` | unrecognised | rerun with `-v` for the full stderr | + +When any host is unusable, the report ends with a **"what to ask your administrator"** +block: the hosts, the account, the fingerprint of the key being offered, the exact line to +append to `authorized_keys`, and — for `no-sudo` — the precise sudoers line plus the two +suites that need it. It is meant to be forwarded verbatim. + +The classification patterns come from OpenSSH's own message strings and have not been +replayed against every distribution's build. Adding one is a one-line change in +`sshdiag.py`; `checks/check_remote_sshdiag.py` is table-driven over recorded samples. + +## Privileges the tests actually need + +Grepped from the `ignitetest` sources, not assumed. Useful when an administrator offers you +a privileged account you do not need: + +- **An ordinary unprivileged account** for everything except the two suites below. It needs + write access to `persistent_root` (default `/mnt/service`) and read access to + `install_root` (default `/opt`). +- **Passwordless `sudo` for `iptables` only**, and only for + `ignitetest/tests/discovery_test.py` and `ignitetest/tests/cellular_affinity_test.py`, + which reach `sudo iptables`, `iptables-save` and `iptables-restore` through + `IgniteAwareService.drop_network`. +- **Write access to `install_root`** only if you use `deploy` without `--sudo`. + +Nothing else in `ignitetest` needs root. + +## When the message is not enough + +```bash +ducktests-remote -v doctor # every command as it is issued, full per-host output +ducktests-remote --dry-run # the exact scripts and files, executed nowhere +ducktests-remote status --json # machine-readable run state +``` + +On the runner, the run directory holds the whole truth: `run.sh` is the exact command, +`ducktape.log` is the combined output, and `meta.json` records the configuration the run +was launched with. See [runs.md](runs.md). + +If a failure looks like a bug in this CLI rather than in the cluster, the unit checks run +in under a second and touch no network: + +```bash +cd modules/ducktests/tests +pytest ducktests_remote/checks +flake8 ducktests_remote +``` diff --git a/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml b/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml new file mode 100644 index 0000000000000..c7508f95fa1dd --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/examples/cluster.yaml @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Example cluster inventory. Copy to ~/.ducktests-remote/config.yaml and edit. +# Everything here is a placeholder: this file lives in a public repository, so it +# contains no real hostnames, addresses, accounts or secrets. + +cluster: + name: example + + # SSH account used on every worker. Defaults to your own $USER when omitted. + # There is no dedicated test account on a real VM cluster unless your site created + # one; nothing in this tool assumes one exists. + user: ${env:DTR_SSH_USER} + + # Path to the private key AS THE RUNNER SEES IT, not as this machine sees it. + # `ducktests-remote keys push` installs it there. + identity_file: ~/.ssh/id_rsa + + port: 22 + + # Where distributions live on the workers. ignitetest resolves a home directory as + # /, product being str(IgniteVersion(version)). + install_root: /opt + + # Host that runs the ducktape process. "local" runs it on this machine. + runner: ${env:DTR_RUNNER} + + # Run state (run dirs, logs, the venv) lives here on the runner. + state_root: ~/.ducktests-remote + + # Any number of workers: 2, 12, 49. Ranges expand, and a bare string is shorthand + # for {host: ...}. + nodes: + - host: worker[01-12].example.invalid + - host: worker13.example.invalid + ip: 203.0.113.13 # optional -> externally_routable_ip + - host: worker14.example.invalid + user: someone-else # per-host override for mixed clusters + + # Hosts that provision/deploy/clean/doctor should also target, but that ducktape + # must not schedule Ignite nodes onto - typically the runner itself. + extra_hosts: [] + +runner: + # Leave unset to use /venv, created on first run with the pinned + # requirements from modules/ducktests/tests/docker/requirements.txt. + venv: null + python: python3 + create_venv: true + +# Where the runner installs those requirements from. Only the runner ever runs pip; +# workers are driven over plain ssh and need no Python. Leave the whole section out to +# use PyPI. A token in the URL belongs in the environment, not in this file - and +# anything resolved from ${env:} is masked in everything the CLI prints. +pip: + # index_url: ${env:DTR_PIP_INDEX_URL} # a missing variable is a hard error, never "" + # extra_index_url: [https://nexus.example.invalid/repository/pypi-internal/simple] + # trusted_host: [nexus.example.invalid] # plain http, or a certificate you cannot fix + # cert: /etc/pki/tls/certs/corp-ca.pem # a RUNNER-side path, like identity_file + timeout: 60 + retries: 5 + +# Which JVM the workers run the tests under. `major` alone is enough when the VMs already +# carry a suitable JDK somewhere under search_paths; `archive` covers the ones that do not. +java: + major: 17 + # home: /opt/jdk-17.0.11 # use exactly this JDK, no search + search_paths: [/opt, /usr/lib/jvm, /usr/java] + # archive: ~/jdk/OpenJDK17U-jdk_x64_linux_hotspot.tar.gz # coordinator-side + # install_root: /opt # defaults to cluster.install_root + # name: jdk-17 # target directory; defaults to the archive's own name + ssh_environment: true # write ~/.ssh/environment + bashrc: true # and a marked block at the top of ~/.bashrc + +deploy: + # One subdirectory per distribution. `ignite-dev` is usually a symlink to a checkout: + # ln -sfn ~/dev/ignite ./dist/ignite-dev + dist_dir: ./dist + + # Patterns left out of every distribution, rsync-style. Empty by default, so a release + # directory is shipped whole. The list below is for the symlinked-checkout case: the + # workers need modules/*/target/*.jar, modules/*/target/libs/*.jar, bin/ and the + # ducktests certs, and nothing else in the tree. + # exclude: [.git, .idea, src, docs, assembly, classes, test-classes, + # generated-sources, generated-test-sources, maven-status, maven-archiver, + # surefire-reports, javadoc, "*.tar.gz", "*.zip", __pycache__, "*.pyc"] + + # Send only what differs from the deployment already on each host, hardlinking the + # rest. Falls back to a whole tarball when either end has no rsync, or with --via. + rsync: true + +jobs: 16 diff --git a/modules/ducktests/tests/ducktests_remote/examples/profile-ise-perf.yaml b/modules/ducktests/tests/ducktests_remote/examples/profile-ise-perf.yaml new file mode 100644 index 0000000000000..3f811a378986e --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/examples/profile-ise-perf.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A fork profile: what the long Jenkins --globals blob becomes once it is split up. +# Every value that would identify a real environment is an ${env:...} placeholder, +# because this file is committed to a public Apache repository. +# +# export ISE_USER=... ISE_PASSWORD=... +# ducktests-remote --profile ise-perf run +# +# ${file:PATH} is also available and reads a trimmed single-line file, which is what +# you want for a password kept outside the environment. + +globals: + project: ${env:DTR_PROJECT} + + ignite_versions: + - ${env:DTR_IGNITE_VERSION} + + ssl: + enabled: true + + authentication: + enabled: true + username: ${env:ISE_USER} + password: ${env:ISE_PASSWORD} + + # Fork spec override; resolve_spec() in ignitetest/services/utils/ignite_spec.py + # reads these from globals to swap in the fork's node startup. + NodeSpec: ${env:DTR_NODE_SPEC} + + # ignitetest defaults: persistent_root /mnt/service, install_root /opt. + # Override them here when the site layout differs. + # persistent_root: /data/ducktests + # install_root: /opt + +# Injected into the tests themselves, mapped to ducktape's --parameters. +# Only emitted when non-empty, matching run_tests.sh. +parameters: {} diff --git a/modules/ducktests/tests/ducktests_remote/examples/profile-smoke.yaml b/modules/ducktests/tests/ducktests_remote/examples/profile-smoke.yaml new file mode 100644 index 0000000000000..2daf8ac482f05 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/examples/profile-smoke.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Smallest useful profile: the smoke suite against the locally built master. +# +# ducktests-remote --profile smoke run ./ignitetest/tests/smoke_test.py +# +globals: + ignite_versions: ["dev"] diff --git a/modules/ducktests/tests/ducktests_remote/fanout.py b/modules/ducktests/tests/ducktests_remote/fanout.py new file mode 100644 index 0000000000000..dcd0d24d7ddfb --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/fanout.py @@ -0,0 +1,153 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Parallel per-host execution and the table it prints. + +The batch never aborts on the first failure unless asked to. When an operator has to +send a single request to whoever administers the machines, a partial list of broken +hosts is worse than useless. +""" + +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from typing import Any, Optional + +OK = "ok" +CHANGED = "changed" +SKIPPED = "skipped" +WARN = "warn" +FAILED = "failed" + +_ORDER = {FAILED: 0, WARN: 1, CHANGED: 2, SKIPPED: 3, OK: 4} + + +@dataclass +class HostResult: + """Outcome of one per-host operation.""" + + host: str + status: str = OK + message: str = "" + detail: str = "" + duration: float = 0.0 + data: Optional[Any] = field(default=None) + + @property + def ok(self): + """:return: True unless the operation failed.""" + return self.status != FAILED + + +class FanoutAborted(Exception): + """Raised when ``--fail-fast`` cut a batch short.""" + + +def fanout(hosts, operation, *, jobs=16, fail_fast=False): + """ + Apply ``operation(host)`` across ``hosts`` in a bounded thread pool. + + :param operation: callable returning a :class:`HostResult` (or a ``(status, message)`` + tuple, or None for plain success). + :param jobs: pool size. + :param fail_fast: stop scheduling further hosts after the first failure. + :return: results in inventory order. + """ + hosts = list(hosts) + if not hosts: + return [] + results = [None] * len(hosts) + stop = {"flag": False} + + def wrapped(index, host): + if stop["flag"]: + return HostResult(_host_name(host), SKIPPED, "skipped after earlier failure") + started = time.monotonic() + try: + outcome = operation(host) + except Exception as ex: # noqa: BLE001 - per-host isolation is the whole point + outcome = HostResult(_host_name(host), FAILED, _short(ex), detail=repr(ex)) + outcome = _normalise(outcome, host) + outcome.duration = time.monotonic() - started + if fail_fast and outcome.status == FAILED: + stop["flag"] = True + results[index] = outcome + return outcome + + with ThreadPoolExecutor(max_workers=max(1, int(jobs))) as pool: + list(pool.map(lambda pair: wrapped(*pair), list(enumerate(hosts)))) + + return [r for r in results if r is not None] + + +def _normalise(outcome, host): + name = _host_name(host) + if outcome is None: + return HostResult(name, OK) + if isinstance(outcome, HostResult): + return outcome + if isinstance(outcome, tuple): + status, message = (list(outcome) + [""])[:2] + return HostResult(name, status, message) + return HostResult(name, OK, str(outcome)) + + +def _host_name(host): + return getattr(host, "host", None) or str(host) + + +def _short(ex): + text = str(ex).strip().splitlines() + return text[0][:160] if text else ex.__class__.__name__ + + +def render_table(results, *, verbose=False, headers=("HOST", "STATUS", "TIME", "DETAIL")): + """:return: an aligned table of fan-out results, failures first within equal status.""" + if not results: + return "(no hosts)" + rows = [(r.host, r.status, "%.1fs" % r.duration, r.message or "") for r in results] + widths = [max(len(str(row[i])) for row in ([headers] + rows)) for i in range(4)] + lines = [" ".join(str(h).ljust(widths[i]) for i, h in enumerate(headers)).rstrip()] + lines.append(" ".join("-" * widths[i] for i in range(4))) + for row in rows: + lines.append(" ".join(str(c).ljust(widths[i]) for i, c in enumerate(row)).rstrip()) + if verbose: + for result in results: + if result.detail: + lines.append("") + lines.append("--- %s ---" % result.host) + lines.append(result.detail.rstrip()) + else: + for result in results: + if result.status == FAILED and result.detail: + lines.append("") + lines.append("--- %s ---" % result.host) + lines.append(result.detail.rstrip()) + return "\n".join(lines) + + +def summarise(results): + """:return: a compact ``9 ok, 2 failed`` style summary line.""" + counts = {} + for result in results: + counts[result.status] = counts.get(result.status, 0) + 1 + ordered = sorted(counts.items(), key=lambda kv: _ORDER.get(kv[0], 9)) + return ", ".join("%d %s" % (count, status) for status, count in ordered) or "nothing to do" + + +def any_failed(results): + """:return: True when at least one host failed.""" + return any(r.status == FAILED for r in results) diff --git a/modules/ducktests/tests/ducktests_remote/globals_builder.py b/modules/ducktests/tests/ducktests_remote/globals_builder.py new file mode 100644 index 0000000000000..9d081bd5c4501 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/globals_builder.py @@ -0,0 +1,163 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Composition of the ``--globals`` payload, and the redactor that keeps its secrets out of +everything the CLI prints. + +The Jenkins one-liner this replaces carries a kilobyte of JSON with a password and a set +of internal IPs on the command line, where it lands in shell history, process listings +and build logs. Here the same content is layered from profiles, resolved from the +environment at the last moment, and written straight to a 0600 file on the runner. +""" + +import json +import os +from pathlib import Path + +from ducktests_remote.config import ConfigError, deep_merge, interpolate, set_dotted + +# Re-exported: ``interpolate`` lives in `config` so that the non-globals sections can use +# it too, without `config` having to import this module back. +__all__ = ["Redactor", "build", "dumps", "interpolate", "load_raw_layer", "parse_kv_override"] + + +class Redactor: + """ + Replaces resolved secret *values* with ``***`` in anything the CLI emits. + + Keying on values rather than on key names is what makes this reliable: a password + that leaks into an unrelated field, a rendered command line or a traceback is still + caught. Key-name matching is only a fallback for values we never resolved ourselves. + """ + + MASK = "***" + SENSITIVE_KEYS = ("password", "passwd", "secret", "token", "keystore_pass", "truststore_pass") + + def __init__(self): + self._values = set() + + def add(self, value): + """Register a resolved secret value.""" + if isinstance(value, str) and len(value.strip()) >= 3: + self._values.add(value.strip()) + + @property + def values(self): + """:return: the registered secret values.""" + return frozenset(self._values) + + def redact(self, text): + """:return: ``text`` with every registered secret replaced by ``***``.""" + if text is None: + return None + result = str(text) + for secret in sorted(self._values, key=len, reverse=True): + result = result.replace(secret, self.MASK) + return result + + def redact_structure(self, data): + """:return: a copy of ``data`` with secret values, and sensitive keys, masked.""" + if isinstance(data, dict): + out = {} + for key, value in data.items(): + if isinstance(key, str) and key.lower() in self.SENSITIVE_KEYS: + out[key] = self.MASK + else: + out[key] = self.redact_structure(value) + return out + if isinstance(data, list): + return [self.redact_structure(item) for item in data] + if isinstance(data, str): + return self.redact(data) + return data + + +def parse_kv_override(item): + """ + Parse a ``-g a.b.c=value`` argument. + + Values are parsed as JSON when they parse, so ``ssl.enabled=true`` yields a boolean + and ``project=ise`` yields a string. This mirrors ``_extend_json`` in + ``docker/run_tests.sh``, with nesting added. + """ + if "=" not in item: + raise ConfigError("expected KEY=VALUE, found %r" % item) + key, raw = item.split("=", 1) + key = key.strip() + if not key: + raise ConfigError("empty key in %r" % item) + try: + value = json.loads(raw) + except ValueError: + value = raw + return key, value + + +def build(base_layers=(), overrides=(), redactor=None, environ=None): + """ + Compose the final globals mapping. + + :param base_layers: ``(source_name, mapping)`` pairs, later layers winning. + :param overrides: raw ``KEY=VALUE`` strings from ``-g``. + :param redactor: :class:`Redactor` collecting resolved secrets; created when omitted. + :param environ: environment mapping used for ``${env:}``. + :return: ``(globals_dict, redactor)``. + """ + redactor = Redactor() if redactor is None else redactor + composed = {} + for source, layer in base_layers: + if not layer: + continue + if not isinstance(layer, dict): + raise ConfigError("%s: globals must be a mapping, found %s" + % (source, type(layer).__name__)) + composed = deep_merge(composed, interpolate(layer, redactor, environ, source)) + + for item in overrides: + key, value = parse_kv_override(item) + set_dotted(composed, key, interpolate(value, redactor, environ, "-g %s" % key)) + + return composed, redactor + + +def load_raw_layer(json_text=None, json_file=None): + """ + Parse a raw base layer from ``--globals-json`` / ``--globals-file``. + + This is the migration path: paste the existing Jenkins blob verbatim, get a working + run, then split it into YAML profiles one key at a time. + """ + if json_text and json_file: + raise ConfigError("pass only one of --globals-json / --globals-file") + if json_file: + path = Path(os.path.expanduser(json_file)) + if not path.is_file(): + raise ConfigError("globals file not found: %s" % path) + json_text = path.read_text(encoding="utf-8") + if not json_text: + return None + try: + data = json.loads(json_text) + except ValueError as ex: + raise ConfigError("globals is not valid JSON: %s" % ex) from ex + if not isinstance(data, dict): + raise ConfigError("globals must be a JSON object, found %s" % type(data).__name__) + return data + + +def dumps(data): + """:return: canonical JSON for a globals/parameters payload.""" + return json.dumps(data, indent=2, sort_keys=True) + "\n" diff --git a/modules/ducktests/tests/ducktests_remote/java.py b/modules/ducktests/tests/ducktests_remote/java.py new file mode 100644 index 0000000000000..ae16f92bfb826 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/java.py @@ -0,0 +1,636 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Choosing the JVM the workers run the tests under. + +**Why PATH matters more than JAVA_HOME here.** ``ignitetest`` reaches a JVM four +different ways, and only one of them respects ``JAVA_HOME``: + +=================================================== ========================== +consumer mechanism +=================================================== ========================== +``ignite.sh``, via ``IgniteSpec.envs()`` honours ``JAVA_HOME`` +``jvm_utils.java_version()`` -> ``java -version`` bare ``java``, so PATH +``services/kafka/kafka.py`` -> ``nohup java ...`` bare ``java``, so PATH +``jmx_utils`` -> ``java -jar jmxterm.jar`` bare ``java``, so PATH +=================================================== ========================== + +All four run over *non-interactive* ssh, where ``~/.profile`` is never sourced. Setting +``JAVA_HOME`` alone therefore changes what ``ignite.sh`` uses and nothing else; what the +rest of the suite gets is decided by the non-interactive PATH. Both are written, and the +result is then verified over a fresh connection rather than assumed - see +:func:`env_script` and :func:`verify_script`. + +The resolution ladder, in :func:`discovery_script` (rungs 1-3, pure discovery, safe to run +from ``doctor``) and in ``commands/provision.py`` (rungs 4-5, which mutate): + +1. ``java.home`` is set - verify it, and fail naming the host when it is not there. + Explicit means explicit; falling back would defeat the point of saying it. +2. the JVM the non-interactive shell already provides matches ``java.major`` - use it. +3. a JDK of the right major exists under one of ``java.search_paths`` - use it. +4. ``java.archive`` is set - deliver it (``commands/provision.py``). +5. otherwise fail, listing every JDK that *was* found. +""" + +import posixpath +import shlex +import tarfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Optional + +from ducktests_remote.config import ConfigError, expand_path + +BLOCK_BEGIN = "# BEGIN ducktests-remote" +BLOCK_END = "# END ducktests-remote" + +# Sources, in the order the ladder tries them. +EXPLICIT = "explicit" +CURRENT = "current" +SEARCH = "search" +DELIVERED = "delivered" +NONE = "none" + +_TAR_SUFFIXES = (".tar.gz", ".tgz", ".tar", ".tar.bz2", ".tbz2", ".tar.xz", ".txz") + +# `tar` decompression flag per suffix. Explicit rather than relying on GNU tar's +# auto-detection, so the worker-side command does not depend on which tar is installed. +_TAR_FLAGS = ((".tar.gz", "z"), (".tgz", "z"), (".tar.bz2", "j"), (".tbz2", "j"), + (".tar.xz", "J"), (".txz", "J"), (".tar", "")) + + +@dataclass +class JavaConfig: + """The ``java`` config section, with the cluster-level fallbacks already applied.""" + + major: Optional[int] = None + home: Optional[str] = None + search_paths: List[str] = field(default_factory=list) + archive: Optional[str] = None + install_root: str = "/opt" + name: Optional[str] = None + ssh_environment: bool = True + bashrc: bool = True + + +@dataclass +class Resolution: + """What one worker reported, and what should be done about it.""" + + host: str = "-" + home: Optional[str] = None # the JDK the ladder selected + version: str = "" + major: Optional[int] = None + source: str = NONE + path_java: Optional[str] = None # what bare `java` resolves to right now + path_real: Optional[str] = None # ... with symlinks resolved + path_version: str = "" + path_major: Optional[int] = None + env_home: Optional[str] = None # $JAVA_HOME as the non-interactive shell sees it + candidates: List[tuple] = field(default_factory=list) # [(home, major, version)] + + @property + def selected(self): + """:return: True when the ladder found a JDK of the requested major.""" + return bool(self.home) + + def path_matches(self, requested_major): + """ + :return: True when the JVM the tests will actually get has the requested major. + + This, not :attr:`selected`, is the question ``doctor`` has to answer: a perfect + JDK sitting in ``/opt`` that the non-interactive PATH does not point at is not the + JVM the suite will run under. + """ + if self.path_major is None: + return False + return requested_major is None or self.path_major == requested_major + + @property + def home_in_effect(self): + """ + :return: True when the JDK on PATH is the one the ladder selected. + + Compared on the resolved path, so ``/usr/bin/java`` symlinked into the selected + home counts as a match. A same-major JDK from somewhere else does not, which is + worth a warning but not a failure - the tests still get the version they asked + for. + """ + return _same_home(self.path_real or self.path_java, self.home) + + def summary(self): + """:return: a short human description of the selected JDK.""" + if not self.selected: + return "no JDK of the requested version" + return "%s (%s, found by %s)" % (self.home, self.version or "unknown version", + self.source) + + +def config_of(ctx): + """:return: the :class:`JavaConfig` for this context, cluster fallbacks applied.""" + section = ctx.config.get("java") or {} + major = section.get("major") + if major is not None: + try: + major = int(major) + except (TypeError, ValueError) as ex: + raise ConfigError("java.major must be a whole number, found %r" % (major,)) from ex + search = section.get("search_paths") or [] + if isinstance(search, str): + search = [search] + return JavaConfig( + major=major, + home=section.get("home") or None, + search_paths=[str(p) for p in search], + archive=section.get("archive") or None, + install_root=(section.get("install_root") + or ctx.cluster_cfg.get("install_root") or "/opt"), + name=section.get("name") or None, + ssh_environment=bool(section.get("ssh_environment", True)), + bashrc=bool(section.get("bashrc", True))) + + +def major_of(version): + """ + :return: the major version of a Java version string, or None. + + Mirrors ``ignitetest.services.utils.jvm_utils.java_major_version``: ``1.8.0_292`` is + 8, ``11.0.19`` is 11, ``17.0.11+9`` is 17. + """ + text = str(version or "").strip().strip('"') + if not text: + return None + parts = text.split(".") + chosen = parts[1] if parts[0] == "1" and len(parts) > 1 else parts[0] + digits = "" + for char in chosen: + if not char.isdigit(): + break + digits += char + return int(digits) if digits else None + + +def discovery_script(cfg: JavaConfig): + """ + :return: a POSIX shell script that reports the JDK situation on one host. + + It only ever *reads*, which is what makes it safe to run from ``doctor`` and from + ``provision --only ssh-env`` as well as from the ``jdk`` step, and it always exits 0: + the status travels in the fields, so one unusable host cannot abort a fan-out. + """ + return _DISCOVERY % { + "requested": shlex.quote(str(cfg.major) if cfg.major else ""), + "explicit": shlex.quote(cfg.home or ""), + "search": " ".join(shlex.quote(p) for p in cfg.search_paths), + } + + +_DISCOVERY = r""" +set -u +say() { printf '%%s=%%s\n' "$1" "$2"; } +requested=%(requested)s +explicit=%(explicit)s + +jv_version() { "$1" -version 2>&1 | head -n1 | sed -n 's/.*version "\([^"]*\)".*/\1/p'; } +jv_major() { + case "$1" in + "") printf '' ;; + 1.*) printf '%%s' "$1" | cut -d. -f2 | tr -cd '0-9' ;; + *) printf '%%s' "$1" | cut -d. -f1 | tr -cd '0-9' ;; + esac +} +home_of() { + p=$(readlink -f "$1" 2>/dev/null || printf '%%s' "$1") + dirname "$(dirname "$p")" +} + +cands="" +add_cand() { + [ -n "$1" ] || return 0 + [ -x "$1/bin/java" ] || return 0 + for seen in $cands; do [ "$seen" = "$1" ] && return 0; done + cands="$cands $1" +} + +say java_requested "$requested" + +# What the tests will actually get: bare `java`, over non-interactive ssh. +cur=$(command -v java 2>/dev/null || true) +say java "$(java -version 2>&1 | head -n1 | tr -d '\r' || echo missing)" +if [ -n "$cur" ]; then + cv=$(jv_version "$cur"); say java_path "$cur"; say java_path_version "$cv" + say java_path_major "$(jv_major "$cv")" + say java_path_real "$(readlink -f "$cur" 2>/dev/null || printf '%%s' "$cur")" + add_cand "$(home_of "$cur")" +fi +say java_env_home "${JAVA_HOME:-}" +add_cand "${JAVA_HOME:-}" +add_cand "$explicit" + +for root in %(search)s; do + add_cand "$root" + for entry in "$root"/*; do add_cand "$entry"; done +done + +report="" +for c in $cands; do + v=$(jv_version "$c/bin/java") + m=$(jv_major "$v") + [ -n "$m" ] || continue + report="$report,$c:$m:$v" +done +say java_candidates "${report#,}" +exit 0 +""" + + +def parse_probe(host, text, cfg: JavaConfig): + """:return: :func:`parse_facts` applied to one host's raw discovery output.""" + return parse_facts(host, _parse_kv(text), cfg) + + +def parse_facts(host, facts, cfg: JavaConfig): + """ + Turn one host's discovery facts into a :class:`Resolution`. + + The selection happens here rather than in the shell so that it is unit-testable and + identical for every caller. ``doctor`` already parses the worker probe into a dict, + which is why this takes facts and :func:`parse_probe` takes text. + """ + res = Resolution(host=host) + res.path_java = facts.get("java_path") or None + res.path_real = facts.get("java_path_real") or None + res.path_version = facts.get("java_path_version") or "" + res.path_major = _int_or_none(facts.get("java_path_major")) + res.env_home = facts.get("java_env_home") or None + res.candidates = _parse_candidates(facts.get("java_candidates")) + + versions = {home: version for home, _, version in res.candidates} + majors = {home: major for home, major, _ in res.candidates} + + if cfg.home: + if majors.get(cfg.home) is None: + return res # explicit home absent or unusable: the caller fails hard + res.home, res.major, res.version = cfg.home, majors[cfg.home], versions[cfg.home] + res.source = EXPLICIT + return res + + wanted = cfg.major + current_home = _home_of(res.path_real or res.path_java) if res.path_java else None + if wanted is None: + # No requested version: whatever the host already provides is the answer. + if current_home and res.path_major is not None: + res.home = current_home + res.major, res.version, res.source = res.path_major, res.path_version, CURRENT + return res + + if res.path_major == wanted and current_home: + res.home = current_home + res.major, res.version, res.source = wanted, res.path_version, CURRENT + return res + + matching = [(home, version) for home, major, version in res.candidates if major == wanted] + if matching: + # Highest patch level wins, compared numerically: sorting the strings would put + # 17.0.9 above 17.0.11. The path breaks ties, so the choice is deterministic. + res.home = max(matching, key=lambda item: (version_key(item[1]), item[0]))[0] + res.major, res.version, res.source = wanted, versions[res.home], SEARCH + return res + + +def version_key(version): + """:return: a numeric tuple for ordering Java version strings (17.0.9 < 17.0.11).""" + numbers = [] + current = "" + for char in str(version or ""): + if char.isdigit(): + current += char + elif current: + numbers.append(int(current)) + current = "" + if current: + numbers.append(int(current)) + return tuple(numbers) + + +def env_script(cfg: JavaConfig, java_home, path_extra=()): + """ + :return: a script writing ``JAVA_HOME``/``PATH`` where non-interactive ssh will see it. + + Two mechanisms, written from the same resolved value in the same step so they cannot + drift apart: + + ``~/.ssh/environment`` + what the Dockerfile does, and silently ignored unless sshd carries + ``PermitUserEnvironment yes``, which no site guarantees. + + ``~/.bashrc`` + a marked block at the very *top* of the file, above the + ``case $- in *i*) ;; *) return;; esac`` guard that the stock Ubuntu file opens + with - that guard exists precisely because bash does source ``~/.bashrc`` for + non-interactive ssh commands. It does nothing when the login shell is not bash. + + Neither is trusted afterwards: :func:`verify_script` re-asks over a fresh connection. + """ + if not (cfg.ssh_environment or cfg.bashrc): + raise ConfigError("java.ssh_environment and java.bashrc are both false, so there " + "is no way to put the JDK on the workers' non-interactive PATH") + fallback = ("echo \"NOTE the ~/.bashrc block below is what will carry JAVA_HOME.\"" + if cfg.bashrc else + "echo \"NOTE java.bashrc is off, so nothing else will carry it: ask for\"\n" + " echo \"NOTE PermitUserEnvironment, or name a JDK already on PATH.\"") + body = _ENV_HEAD % {"home": shlex.quote(str(java_home))} + if cfg.ssh_environment: + body += _ENV_SSH % {"extra": "".join(":%s" % p for p in (path_extra or [])), + "fallback": fallback} + if cfg.bashrc: + body += _ENV_BASHRC % {"begin": BLOCK_BEGIN, "end": BLOCK_END} + return body + _ENV_TAIL + + +_ENV_HEAD = r""" +set -u +jh=%(home)s +[ -x "$jh/bin/java" ] || { echo "no java under $jh" >&2; exit 1; } +changed=0 +""" + +_ENV_SSH = r""" +mkdir -p ~/.ssh +chmod 700 ~/.ssh +touch ~/.ssh/environment +chmod 600 ~/.ssh/environment +# Drop the entries we are about to prepend before reading $PATH back, or a host that +# honours ~/.ssh/environment would grow one more copy of $jh/bin on every run and this +# step would report CHANGED for ever. +base_path=$(printf '%%s' "$PATH" | awk -v RS=: -v strip="$jh/bin%(extra)s" ' + BEGIN { ORS=""; n = split(strip, s, ":") } + { + keep = ($0 != "") + for (i = 1; i <= n; i++) if ($0 == s[i]) keep = 0 + if (keep) { if (out++) printf ":"; printf "%%s", $0 } + }') +want_path="PATH=$jh/bin:$base_path%(extra)s" +want_home="JAVA_HOME=$jh" +for line in "$want_path" "$want_home" "LANG=C.UTF-8"; do + key=${line%%%%=*} + if grep -q "^$key=" ~/.ssh/environment 2>/dev/null; then + current=$(grep "^$key=" ~/.ssh/environment | head -n1) + [ "$current" = "$line" ] && continue + grep -v "^$key=" ~/.ssh/environment > ~/.ssh/environment.tmp || true + mv ~/.ssh/environment.tmp ~/.ssh/environment + fi + printf '%%s\n' "$line" >> ~/.ssh/environment + changed=1 +done +chmod 600 ~/.ssh/environment +[ "$changed" -eq 1 ] && echo "CHANGED wrote ~/.ssh/environment" +if ! grep -qi '^ *PermitUserEnvironment *yes' /etc/ssh/sshd_config 2>/dev/null; then + echo "NOTE sshd has no 'PermitUserEnvironment yes'; ~/.ssh/environment is ignored." + %(fallback)s +fi +""" + +_ENV_BASHRC = r""" +# The block goes at the TOP, above the `case $- in *i*)` early return that the stock +# ~/.bashrc opens with, or it would never run for a non-interactive ssh command. +tmp=$(mktemp) +{ + printf '%%s\n' "%(begin)s" + printf '%%s\n' "export JAVA_HOME=$jh" + printf '%%s\n' 'export PATH="$JAVA_HOME/bin:$PATH"' + printf '%%s\n' "%(end)s" +} > "$tmp" +if [ -f ~/.bashrc ]; then + awk 'BEGIN{skip=0} + /^# BEGIN ducktests-remote$/{skip=1; next} + /^# END ducktests-remote$/{skip=0; next} + skip==0{print}' ~/.bashrc >> "$tmp" +fi +if cmp -s "$tmp" ~/.bashrc 2>/dev/null; then + rm -f "$tmp" +else + mv "$tmp" ~/.bashrc + changed=1 + echo "CHANGED wrote the ducktests-remote block in ~/.bashrc" +fi +""" + +_ENV_TAIL = r""" +if [ "$changed" -eq 0 ]; then echo "up to date"; fi +exit 0 +""" + + +def verify_script(): + """ + :return: a script reporting what a *fresh* non-interactive session actually gets. + + This is the authority. ``~/.ssh/environment`` and ``~/.bashrc`` are both best effort, + and the failure they exist to prevent - ``java`` missing three hours into a run - is + only really excluded by asking the way ducktape will ask. + """ + return r""" +set -u +say() { printf '%s=%s\n' "$1" "$2"; } +jv_version() { "$1" -version 2>&1 | head -n1 | sed -n 's/.*version "\([^"]*\)".*/\1/p'; } +jv_major() { + case "$1" in + "") printf '' ;; + 1.*) printf '%s' "$1" | cut -d. -f2 | tr -cd '0-9' ;; + *) printf '%s' "$1" | cut -d. -f1 | tr -cd '0-9' ;; + esac +} +say java "$(java -version 2>&1 | head -n1 | tr -d '\r' || echo missing)" +say java_env_home "${JAVA_HOME:-}" +cur=$(command -v java 2>/dev/null || true) +if [ -n "$cur" ]; then + cv=$(jv_version "$cur") + say java_path "$cur" + say java_path_version "$cv" + say java_path_major "$(jv_major "$cv")" + say java_path_real "$(readlink -f "$cur" 2>/dev/null || printf '%s' "$cur")" +fi +exit 0 +""" + + +@dataclass +class ArchivePlan: + """A coordinator-side JDK archive or directory, inspected before anything is sent.""" + + path: Path + kind: str # "tar" or "dir" + top_level: Optional[str] # directory prefix holding the JDK home, if any + strip: int # --strip-components for tar + bytes: int + name: str # default target directory name + tar_flag: str = "z" # tar decompression flag matching the suffix + + +def archive_plan(archive, name=None): + """ + Inspect ``java.archive`` on the coordinator. + + Reading the member list with :mod:`tarfile` rather than guessing in the shell is what + lets a bad archive fail *before* it is copied to every host. The JDK home is located + by finding ``bin/java`` and stripping whatever wraps it, so a Temurin tarball + (``jdk-17.0.11+9/bin/java``), a flat one (``bin/java``) and a doubly wrapped one all + work. A macOS build is still refused: ``Contents/Home`` unpacks fine and then fails + on every worker, which is a far worse place to find out. + """ + path = Path(expand_path(archive)) + if not path.exists(): + raise ConfigError("java.archive %s does not exist on this machine" % path) + + if path.is_dir(): + if not (path / "bin" / "java").exists(): + raise ConfigError("java.archive %s has no bin/java; point it at a JDK home" + % path) + total = sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) + return ArchivePlan(path=path, kind="dir", top_level=None, strip=0, bytes=total, + name=name or path.name, tar_flag="z") + + lowered = path.name.lower() + if lowered.endswith(".zip"): + raise ConfigError( + "java.archive %s is a zip; ducktests-remote unpacks .tar.gz/.tgz/.tar only. " + "Linux JDKs ship as tarballs, and an untested zip branch on every run is " + "worse than this message." % path) + if not lowered.endswith(_TAR_SUFFIXES): + raise ConfigError("java.archive %s is neither a directory nor a tar archive" + % path) + + try: + with tarfile.open(path) as tar: + members = tar.getmembers() + except (tarfile.TarError, OSError) as ex: + raise ConfigError("java.archive %s could not be read: %s" % (path, ex)) from ex + + names = [_member_name(m) for m in members if m.name not in (".", "./")] + names = [n for n in names if n] + if not names: + raise ConfigError("java.archive %s is empty" % path) + total = sum(m.size for m in members if m.isfile()) + + prefix = _java_home_prefix(names, path) + strip = prefix.count("/") + 1 if prefix else 0 + return ArchivePlan(path=path, kind="tar", top_level=prefix or None, strip=strip, + bytes=total, name=name or (posixpath.basename(prefix) if prefix + else _strip_suffix(path.name)), + tar_flag=_tar_flag(path.name)) + + +def _member_name(member): + """:return: ``member``'s path with a leading ``./`` removed, and nothing else.""" + name = member.name + while name.startswith("./"): + name = name[2:] + return name.rstrip("/") + + +def _java_home_prefix(names, path): + """ + :return: the directory prefix inside the tarball that *is* the JDK home ("" if flat). + + The shallowest ``bin/java`` wins, so a JDK that happens to ship a second one deeper + down (a bundled JRE, a jmod staging directory) cannot pull the strip depth with it. + """ + best = None + for name in names: + if name == "bin/java": + return "" + if name.endswith("/bin/java"): + candidate = name[: -len("/bin/java")] + if best is None or candidate.count("/") < best.count("/"): + best = candidate + if best is None: + raise ConfigError( + "java.archive %s does not contain bin/java anywhere. Point it at a JDK " + "tarball or at an unpacked JDK home." % path) + if _is_macos_layout(best): + raise ConfigError( + "java.archive %s is a macOS JDK: bin/java sits under %s/bin/java, with " + "Contents/Home in between. It cannot run on the Linux workers - download the " + "Linux x64 build instead." % (path, best)) + return best + + +def _is_macos_layout(prefix): + parts = prefix.split("/") + return "Contents" in parts and "Home" in parts + + +def target_dir(cfg: JavaConfig, plan: ArchivePlan): + """:return: where ``plan`` is installed on a worker.""" + return posixpath.join(cfg.install_root, cfg.name or plan.name) + + +def _strip_suffix(filename): + for suffix in _TAR_SUFFIXES: + if filename.lower().endswith(suffix): + return filename[: -len(suffix)] + return filename + + +def _tar_flag(filename): + """:return: the ``tar`` decompression flag for ``filename``'s suffix.""" + lowered = filename.lower() + for suffix, flag in _TAR_FLAGS: + if lowered.endswith(suffix): + return flag + return "z" + + +def _home_of(java_binary): + return posixpath.dirname(posixpath.dirname(str(java_binary))) + + +def _same_home(java_binary, home): + if not java_binary or not home: + return False + return _home_of(java_binary).rstrip("/") == str(home).rstrip("/") + + +def _parse_candidates(raw): + out = [] + for item in (raw or "").split(","): + if not item.strip(): + continue + parts = item.split(":") + if len(parts) < 2: + continue + major = _int_or_none(parts[1]) + if major is None: + continue + out.append((parts[0], major, parts[2] if len(parts) > 2 else "")) + return out + + +def _parse_kv(text): + fields = {} + for line in (text or "").split("\n"): + if "=" in line: + key, value = line.split("=", 1) + fields[key.strip()] = value.strip() + return fields + + +def _int_or_none(value): + try: + return int(str(value).strip()) + except (TypeError, ValueError): + return None diff --git a/modules/ducktests/tests/ducktests_remote/pipconf.py b/modules/ducktests/tests/ducktests_remote/pipconf.py new file mode 100644 index 0000000000000..8947fc271dcd9 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/pipconf.py @@ -0,0 +1,159 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Index configuration for the pip commands the CLI runs on the runner. + +A runner inside a corporate network usually cannot reach PyPI at all, only an internal +mirror, and frequently one behind a private CA. Everything needed for that is expressed +in the ``pip`` config section and turned into *command-line flags* here. + +Flags rather than a generated ``pip.conf`` or ``PIP_*`` environment variables: ``--dry-run`` +printing the literal command it would run is this CLI's contract, and configuration hidden +in a file the operator cannot see breaks it. pip carries its finder settings into PEP 517 +build isolation, so ``pip install -e`` picks the same index up for its build dependencies. + +The module is named ``pipconf`` rather than ``pip`` so that it can never shadow the real +``pip`` package for anything that ends up with this directory on ``sys.path``. +""" + +import os +import re +import shlex + +from ducktests_remote.config import ConfigError + +# Matches the credentials in https://user:token@host/simple, which is how an internal +# index is usually handed out. +_USERINFO = re.compile(r"(?<=://)[^/@\s]+(?=@)") + + +def pip_args(config): + """ + :return: the pip flags implied by the ``pip`` config section, as an argv list. + + Empty configuration yields an empty list, so a command line with nothing configured + stays byte for byte what it was before this section existed. + """ + section = (config or {}).get("pip") or {} + args = [] + + index_url = _text(section.get("index_url"), "pip.index_url") + if index_url: + args += ["--index-url", index_url] + for url in _as_list(section.get("extra_index_url"), "pip.extra_index_url"): + args += ["--extra-index-url", url] + for host in _as_list(section.get("trusted_host"), "pip.trusted_host"): + args += ["--trusted-host", host] + + timeout = _positive_int(section.get("timeout"), "pip.timeout") + if timeout is not None: + args += ["--timeout", str(timeout)] + retries = _positive_int(section.get("retries"), "pip.retries") + if retries is not None: + args += ["--retries", str(retries)] + + cert = _text(section.get("cert"), "pip.cert") + if cert: + # A runner-side path, like cluster.identity_file: it is opened by pip on the + # runner, so a file that exists on this machine proves nothing. + args += ["--cert", os.path.expanduser(cert) if cert.startswith("~") else cert] + + return args + + +def pip_args_str(config): + """:return: :func:`pip_args` shell-quoted for splicing into a generated script.""" + return " ".join(shlex.quote(arg) for arg in pip_args(config)) + + +def describe(config, redactor=None): + """ + :return: a one-line, credential-free summary of where pip will install from. + + The value-keyed :class:`~ducktests_remote.globals_builder.Redactor` only knows about + values it resolved itself, so an index URL typed straight into a config file would + otherwise reach the terminal with its token attached. Masking the userinfo here + covers that case regardless of where the URL came from. + """ + section = (config or {}).get("pip") or {} + parts = [] + index_url = _text(section.get("index_url"), "pip.index_url") + parts.append("index %s" % mask_credentials(index_url) if index_url + else "index default (PyPI)") + extras = _as_list(section.get("extra_index_url"), "pip.extra_index_url") + if extras: + parts.append("extra %s" % ", ".join(mask_credentials(url) for url in extras)) + hosts = _as_list(section.get("trusted_host"), "pip.trusted_host") + if hosts: + parts.append("trusted %s" % ", ".join(hosts)) + if section.get("cert"): + parts.append("cert %s" % section["cert"]) + timeout = _positive_int(section.get("timeout"), "pip.timeout") + if timeout is not None: + parts.append("timeout %ds" % timeout) + retries = _positive_int(section.get("retries"), "pip.retries") + if retries is not None: + parts.append("retries %d" % retries) + line = ", ".join(parts) + return redactor.redact(line) if redactor is not None else line + + +def mask_credentials(url): + """:return: ``url`` with any ``user:password@`` replaced by ``***@``.""" + return _USERINFO.sub("***", str(url or "")) + + +def _text(value, key): + if value is None: + return None + if not isinstance(value, str): + raise ConfigError("%s must be a string, found %s" % (key, type(value).__name__)) + return value.strip() or None + + +def _as_list(value, key): + """A single string is accepted wherever a list is: that is what operators type.""" + if value is None: + return [] + if isinstance(value, str): + values = [value] + elif isinstance(value, (list, tuple)): + values = list(value) + else: + raise ConfigError("%s must be a string or a list, found %s" + % (key, type(value).__name__)) + out = [] + for item in values: + if not isinstance(item, str): + raise ConfigError("%s: every entry must be a string, found %s" + % (key, type(item).__name__)) + if item.strip(): + out.append(item.strip()) + return out + + +def _positive_int(value, key): + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, str)): + raise ConfigError("%s must be a whole number of seconds, found %r" % (key, value)) + try: + number = int(value) + except ValueError as ex: + raise ConfigError("%s must be a whole number, found %r" % (key, value)) from ex + if number <= 0: + raise ConfigError("%s must be greater than zero, found %d" % (key, number)) + return number diff --git a/modules/ducktests/tests/ducktests_remote/progress.py b/modules/ducktests/tests/ducktests_remote/progress.py new file mode 100644 index 0000000000000..0ecd93655005a --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/progress.py @@ -0,0 +1,377 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Live progress for the transfers that take minutes. + +Sending 3.5 GB to twelve machines is a normal deploy, and a terminal that prints nothing +for six minutes is indistinguishable from one that has hung. This renders what every +host is doing, from the same fan-out threads that do the work. + +Two modes and no third: + +* **live** - a redrawn block on a terminal, one row per host plus a total. +* **plain** - one aggregate line every ``plain_interval`` seconds, for a log file, a CI + job, ``--verbose`` (where a redrawn block would fight with the traced command lines) + and anything else that is not a terminal. + +The aggregate is the mean of the per-host fractions, never a byte total against a +predicted one: rsync sends only what differs, so a denominator of "the whole +distribution times the host count" would stall at 12% and finish there. Each host +contributes an equal share of the bar, scaled by how far along it is. +""" + +import os +import shutil +import sys +import threading +import time + +WAITING = "waiting" +SENDING = "sending" +DONE = "done" + +_UP = "\033[%dA" +_CLEAR_LINE = "\033[2K" +_HIDE_CURSOR = "\033[?25l" +_SHOW_CURSOR = "\033[?25h" + + +def human_bytes(size): + """:return: ``1.4 GB``; the same shape deploy prints elsewhere.""" + value = float(size) + for unit in ("B", "KB", "MB", "GB", "TB"): + if value < 1024 or unit == "TB": + return "%.0f %s" % (value, unit) if unit in ("B", "KB") else "%.1f %s" % (value, unit) + value /= 1024 + return "%.1f TB" % value + + +def human_duration(seconds): + """:return: ``4:12`` or ``1:04:12``.""" + seconds = int(max(0, seconds)) + hours, rest = divmod(seconds, 3600) + minutes, secs = divmod(rest, 60) + if hours: + return "%d:%02d:%02d" % (hours, minutes, secs) + return "%d:%02d" % (minutes, secs) + + +class _HostState: # pylint: disable=too-few-public-methods + """What one host is doing, and how far along it is.""" + + def __init__(self, name): + self.name = name + self.phase = WAITING + self.sent = 0 + self.total = 0 + self.fraction = 0.0 + self.started = None + self.message = "" + + @property + def rate(self): + """:return: bytes per second since this host started, or 0.""" + if not self.started or self.sent <= 0: + return 0 + elapsed = time.monotonic() - self.started + return self.sent / elapsed if elapsed > 0 else 0 + + +class Progress: + """ + A thread-safe progress display shared by every host in a fan-out. + + Nothing here raises: a display that breaks a deploy would be worse than no display, + so a stream that cannot be written to simply stops being written to. + """ + + # Callers ask this before wiring a transfer up to report: counting bytes has a cost + # (rsync grows a meter, an upload streams through ssh instead of scp) that is only + # worth paying when something will draw the result. + watching = True + + def __init__(self, hosts, *, stream=None, live=None, redactor=None, + interval=0.25, plain_interval=15.0, max_rows=12, unicode_bar=None): + self.stream = stream if stream is not None else sys.stderr + self.hosts = {name: _HostState(name) for name in hosts} + self.order = list(hosts) + self.redactor = redactor + self.interval = interval + self.plain_interval = plain_interval + self.max_rows = max_rows + self.live = is_a_terminal(self.stream) if live is None else bool(live) + self.blocks = _unicode_bar(self.stream) if unicode_bar is None else bool(unicode_bar) + self.title = "" + self._lock = threading.Lock() + self._thread = None + self._stop = threading.Event() + self._drawn = 0 + self._started = time.monotonic() + self._last_plain = 0.0 + + # -- lifecycle --------------------------------------------------------------- + + def start(self, title=""): + """Begin redrawing. Safe to call when the display is disabled.""" + self.title = title + self._started = time.monotonic() + self._last_plain = time.monotonic() + if self._thread is not None: + return self + self._stop.clear() + self._thread = threading.Thread(target=self._loop, name="dtr-progress", daemon=True) + self._thread.start() + return self + + def close(self): + """Stop redrawing and leave the terminal as it was found.""" + if self._thread is None: + return + self._stop.set() + self._thread.join(timeout=2 * self.interval + 1) + self._thread = None + if self.live: + self._write(self._erase() + _SHOW_CURSOR) + + def __enter__(self): + return self.start() + + def __exit__(self, *_exc): + self.close() + return False + + # -- updates, called from the fan-out threads -------------------------------- + + def phase(self, host, phase): + """Record what ``host`` is doing now: sending, extracting, swapping.""" + with self._lock: + state = self.hosts.get(host) + if state is None: + return + state.phase = phase + if state.started is None: + state.started = time.monotonic() + + def sent(self, host, sent, total=None, fraction=None): + """ + Record transferred bytes for ``host``. + + :param fraction: 0..1 when the sender knows better than ``sent/total`` - rsync + reports its own percentage, which accounts for files it decided not to send. + """ + with self._lock: + state = self.hosts.get(host) + if state is None: + return + if state.started is None: + state.started = time.monotonic() + state.phase = SENDING + state.sent = max(state.sent, int(sent or 0)) + if total: + state.total = int(total) + if fraction is not None: + state.fraction = min(1.0, max(0.0, float(fraction))) + elif state.total: + state.fraction = min(1.0, state.sent / float(state.total)) + + def done(self, host, message=""): + """Mark ``host`` finished, however it finished.""" + with self._lock: + state = self.hosts.get(host) + if state is None: + return + state.phase = DONE + state.fraction = 1.0 + state.message = message + + # -- rendering --------------------------------------------------------------- + + def _loop(self): + while not self._stop.is_set(): + self._tick() + self._stop.wait(self.interval) + self._tick(final=True) + + def _tick(self, final=False): + if self.live: + self._write(self._frame()) + return + now = time.monotonic() + if final or now - self._last_plain >= self.plain_interval: + self._last_plain = now + line = self.aggregate_line() + if line: + self._write(line + "\n") + + def _frame(self): + lines = self.rows() + [self.aggregate_line()] + out = [_HIDE_CURSOR] + if self._drawn: + out.append(_UP % self._drawn) + for line in lines: + out.append(_CLEAR_LINE + self._fit(line) + "\n") + for _ in range(max(0, self._drawn - len(lines))): + out.append(_CLEAR_LINE + "\n") + overshoot = max(0, self._drawn - len(lines)) + if overshoot: + out.append(_UP % overshoot) + self._drawn = len(lines) + return "".join(out) + + def _erase(self): + if not self._drawn: + return "" + return (_UP % self._drawn) + (_CLEAR_LINE + "\n") * self._drawn + (_UP % self._drawn) + + def rows(self): + """:return: one rendered line per host worth showing.""" + with self._lock: + states = [self.hosts[name] for name in self.order] + active = [s for s in states if s.phase not in (DONE, WAITING)] + # Finished hosts leave the block: they are already counted in the total line, and + # on a large cluster they would otherwise crowd out the hosts still working. + candidates = active or states + shown = candidates[:self.max_rows] + width = max((len(s.name) for s in states), default=4) + rows = [self._row(state, width) for state in shown] + hidden = len(candidates) - len(shown) + if hidden > 0: + rows.append(" ... %d more host(s)" % hidden) + return rows + + def _row(self, state, width): + name = state.name.ljust(width) + if state.phase == DONE: + return " %s %s %s" % (name, self._bar(1.0), state.message or DONE) + if state.phase == WAITING: + return " %s %s %s" % (name, self._bar(0.0), WAITING) + if state.phase != SENDING: + return " %s %s %s" % (name, self._bar(state.fraction), state.phase) + detail = "%3d%% %s" % (round(state.fraction * 100), human_bytes(state.sent)) + if state.total: + detail += " / %s" % human_bytes(state.total) + if state.rate: + detail += " %s/s" % human_bytes(state.rate) + return " %s %s %s" % (name, self._bar(state.fraction), detail) + + def aggregate_line(self): + """:return: the one line that also stands on its own in a log file.""" + with self._lock: + states = [self.hosts[name] for name in self.order] + if not states: + return "" + finished = sum(1 for s in states if s.phase == DONE) + fraction = sum(s.fraction for s in states) / len(states) + moved = sum(s.sent for s in states) + width = max((len(s.name) for s in states), default=5) + text = " %s %s %3d%% %d/%d host(s) %s sent %s" % ( + "total".ljust(width), self._bar(fraction), round(fraction * 100), finished, + len(states), human_bytes(moved), + human_duration(time.monotonic() - self._started)) + return text if self.live else " ".join(text.split()) + + def _bar(self, fraction, width=24): + filled, empty = ("█", "░") if self.blocks else ("#", ".") + done = int(round(min(1.0, max(0.0, fraction)) * width)) + return filled * done + empty * (width - done) + + def _fit(self, line): + columns = terminal_width(self.stream) + return line if len(line) <= columns else line[:max(0, columns - 1)] + + def _write(self, text): + if not text: + return + if self.redactor is not None: + text = self.redactor.redact(text) + try: + self.stream.write(text) + self.stream.flush() + except (OSError, ValueError): + # A closed or broken stream must never take a deploy down with it. + self.live = False + + +class NullProgress: + """The same surface, doing nothing. Callers never test for None.""" + + live = False + watching = False + + def start(self, title=""): + """:return: self, so ``with`` reads the same either way.""" + return self + + def close(self): + """Nothing to close.""" + + def __enter__(self): + return self + + def __exit__(self, *_exc): + return False + + def phase(self, host, phase): + """Ignore the update.""" + + def sent(self, host, sent, total=None, fraction=None): + """Ignore the update.""" + + def done(self, host, message=""): + """Ignore the update.""" + + +def build_progress(ctx, hosts): + """ + :return: a :class:`Progress` for this command, or a :class:`NullProgress`. + + Off for ``--dry-run`` (nothing moves), ``--quiet`` and ``--no-progress``. Live only + on a terminal: ``--verbose`` prints a traced command line per host, which a redrawn + block would fight with, so that combination reports one aggregate line every few + seconds instead - the same shape a CI log gets. + """ + if ctx.dry_run or ctx.console.quiet or getattr(ctx.args, "no_progress", False): + return NullProgress() + names = [getattr(host, "host", host) for host in hosts] + if not names: + return NullProgress() + return Progress(names, live=is_a_terminal(sys.stderr) and not ctx.console.verbose, + redactor=ctx.console.redactor) + + +def is_a_terminal(stream): + """:return: True when ``stream`` can be redrawn in place.""" + try: + if not stream.isatty(): + return False + except (AttributeError, ValueError): + return False + return os.environ.get("TERM") != "dumb" + + +def terminal_width(stream, default=100): + """:return: usable columns, falling back when the size cannot be read.""" + try: + columns = shutil.get_terminal_size().columns + except (OSError, ValueError): + return default + del stream + return columns if columns and columns > 20 else default + + +def _unicode_bar(stream): + encoding = (getattr(stream, "encoding", "") or "").lower() + return "utf" in encoding diff --git a/modules/ducktests/tests/ducktests_remote/runs.py b/modules/ducktests/tests/ducktests_remote/runs.py new file mode 100644 index 0000000000000..584f474b1422e --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/runs.py @@ -0,0 +1,361 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Run directory layout and state, all of it living on the runner. + +Nothing about a run is kept on the coordinator. That is what lets any coordinator +inspect, follow or stop a run that a different one started - including after the machine +that launched it has been closed and put in a bag. +""" + +import json +import os +import posixpath +import re +import secrets +import shlex +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import List, Optional + +RUN_ID_RE = re.compile(r"^[A-Za-z0-9._-]+-\d{8}-\d{6}-[0-9a-f]{4}$") + +RUNNING = "running" +FINISHED = "finished" +FAILED = "failed" +STOPPED = "stopped" +UNKNOWN = "unknown" + + +def new_run_id(user, now=None, entropy=None): + """ + :return: a run id of the form ``max-20260727-141233-9f2a``. + + The user and timestamp make it readable in a directory listing; the four hex + characters keep two runs started in the same second apart. + """ + now = now or datetime.now() + safe_user = re.sub(r"[^A-Za-z0-9._-]", "_", str(user or "unknown")) or "unknown" + suffix = entropy if entropy is not None else secrets.token_hex(2) + return "%s-%s-%s" % (safe_user, now.strftime("%Y%m%d-%H%M%S"), suffix) + + +def is_run_id(value): + """:return: True when ``value`` looks like a run id.""" + return bool(value) and bool(RUN_ID_RE.match(value)) + + +@dataclass +class RunPaths: + """Absolute, runner-side paths for one run.""" + + state_root: str + run_id: str + + @property + def runs_root(self): + """:return: ``/runs``.""" + return posixpath.join(self.state_root, "runs") + + @property + def run_dir(self): + """:return: ``/runs/``.""" + return posixpath.join(self.runs_root, self.run_id) + + @property + def latest_link(self): + """:return: the ``latest`` symlink beside the run directories.""" + return posixpath.join(self.runs_root, "latest") + + @property + def src_dir(self): + """:return: where this run's sources are synced.""" + return posixpath.join(self.state_root, "src", self.run_id) + + def path(self, *parts): + """:return: a path inside the run directory.""" + return posixpath.join(self.run_dir, *parts) + + meta = property(lambda self: self.path("meta.json")) + cluster_file = property(lambda self: self.path("cluster.json")) + globals_file = property(lambda self: self.path("globals.json")) + parameters_file = property(lambda self: self.path("parameters.json")) + run_script = property(lambda self: self.path("run.sh")) + launch_script = property(lambda self: self.path("launch.sh")) + pid_file = property(lambda self: self.path("pid")) + pgid_file = property(lambda self: self.path("pgid")) + exit_code_file = property(lambda self: self.path("exit_code")) + stopped_file = property(lambda self: self.path("stopped")) + log_file = property(lambda self: self.path("ducktape.log")) + results_dir = property(lambda self: self.path("results")) + + +@dataclass +class RunState: + """Everything ``status`` needs, derived from files on the runner.""" + + run_id: str + state: str = UNKNOWN + pid: Optional[int] = None + pgid: Optional[int] = None + exit_code: Optional[int] = None + meta: dict = field(default_factory=dict) + + @property + def started_at(self): + """:return: ISO timestamp recorded at launch, or None.""" + return self.meta.get("started_at") + + @property + def elapsed(self): + """:return: seconds since launch, or None when the start time is unknown.""" + started = self.meta.get("started_epoch") + if not started: + return None + end = self.meta.get("finished_epoch") or time.time() + return max(0.0, float(end) - float(started)) + + +def derive_state(*, pid_alive, exit_code, stopped): + """ + Turn the three observable facts into a state. + + ``exit_code`` present wins over liveness: the file is written last, so a process that + has already exited is never reported as running just because a pid got reused. + """ + if exit_code is not None: + if stopped: + return STOPPED + return FINISHED if exit_code == 0 else FAILED + if pid_alive: + return RUNNING + if stopped: + return STOPPED + return UNKNOWN + + +def read_state(transport, paths: RunPaths) -> RunState: + """Read one run's state from the runner in a single round trip.""" + script = _STATE_SCRIPT % {"run_dir": shlex.quote(paths.run_dir)} + result = transport.run_script(script, check=False) + fields = _parse_kv(result.stdout) + + pid = _as_int(fields.get("pid")) + pgid = _as_int(fields.get("pgid")) + exit_code = _as_int(fields.get("exit_code")) + meta = {} + if fields.get("meta"): + try: + meta = json.loads(fields["meta"]) + except ValueError: + meta = {} + state = derive_state(pid_alive=fields.get("alive") == "1", + exit_code=exit_code, + stopped=fields.get("stopped") == "1") + return RunState(paths.run_id, state, pid, pgid, exit_code, meta) + + +_STATE_SCRIPT = """ +set -u +rd=%(run_dir)s +say() { printf '%%s=%%s\\n' "$1" "$2"; } +pid="" +[ -f "$rd/pid" ] && pid=$(cat "$rd/pid" 2>/dev/null || true) +say pid "$pid" +[ -f "$rd/pgid" ] && say pgid "$(cat "$rd/pgid" 2>/dev/null || true)" +[ -f "$rd/exit_code" ] && say exit_code "$(cat "$rd/exit_code" 2>/dev/null || true)" +[ -f "$rd/stopped" ] && say stopped 1 +if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then say alive 1; else say alive 0; fi +if [ -f "$rd/meta.json" ]; then printf 'meta=%%s\\n' "$(tr -d '\\n' < "$rd/meta.json")"; fi +""" + + +def list_run_ids(transport, state_root) -> List[str]: + """:return: run ids present on the runner, newest first.""" + runs_root = posixpath.join(state_root, "runs") + result = transport.run(["ls", "-1", runs_root], check=False) + if not result.ok: + return [] + ids = [line.strip() for line in result.stdout.splitlines() if is_run_id(line.strip())] + return sorted(ids, reverse=True) + + +def resolve_run_id(transport, state_root, run_id=None): + """:return: ``run_id`` when given, otherwise the most recent run on the runner.""" + if run_id: + return run_id + ids = list_run_ids(transport, state_root) + if not ids: + return None + return ids[0] + + +def write_meta(transport, paths: RunPaths, meta): + """Write ``meta.json`` into the run directory.""" + transport.write_file(json.dumps(meta, indent=2, sort_keys=True) + "\n", paths.meta) + + +def utc_now_iso(): + """:return: the current time as an ISO-8601 UTC string.""" + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def format_duration(seconds): + """:return: ``1h 04m 12s`` style duration, or ``-`` for None.""" + if seconds is None: + return "-" + seconds = int(seconds) + hours, rest = divmod(seconds, 3600) + minutes, secs = divmod(rest, 60) + if hours: + return "%dh %02dm %02ds" % (hours, minutes, secs) + if minutes: + return "%dm %02ds" % (minutes, secs) + return "%ds" % secs + + +def _parse_kv(text): + fields = {} + for line in (text or "").split("\n"): + if "=" in line: + key, value = line.split("=", 1) + fields[key.strip()] = value.strip() + return fields + + +def _as_int(value): + try: + return int(str(value).strip()) + except (TypeError, ValueError): + return None + + +def default_state_root(state_root): + """:return: the state root with ``~`` left alone (it is expanded on the runner).""" + return state_root or "~/.ducktests-remote" + + +def local_fetch_dir(base, run_id): + """:return: the coordinator-side directory ``fetch`` writes into.""" + return os.path.join(str(base), run_id) + + +TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates") + + +def render_template(name, mapping): + """Substitute ``{{key}}`` placeholders in a packaged template.""" + with open(os.path.join(TEMPLATE_DIR, name), "r", encoding="utf-8") as handle: + text = handle.read() + for key, value in mapping.items(): + text = text.replace("{{%s}}" % key, str(value)) + left = re.search(r"\{\{(\w+)\}\}", text) + if left: + raise KeyError("template %s has an unfilled placeholder %r" % (name, left.group(1))) + return text + + +def render_run_script(*, version, timestamp, author, cwd, results_root, cluster_file, + globals_file, test_paths, venv=None, parameters_file=None, + repeat=None, max_parallel=None, test_runner_timeout=None, + extra_args=()): + """ + Render the ``run.sh`` that the runner executes. + + ``cwd`` is the directory ducktape is started in - the checkout's + ``modules/ducktests/tests``, matching ``docker/run_tests.sh`` - so the test paths in + the generated script read the same as they do in a local Docker run. + + ducktape 0.13 accepts a *file path* for ``--globals`` (``command_line/main.py`` + checks ``os.path.isfile`` before parsing the argument as JSON), so the composed + blob is referenced by path and never crosses a shell command line. + + Every interpolated value is shell-quoted, which is why paths with spaces and JSON + with braces survive the round trip. + """ + extra = [] + if parameters_file: + extra.append("--parameters %s" % shlex.quote(str(parameters_file))) + if repeat: + extra.append("--repeat %d" % int(repeat)) + if max_parallel: + extra.append("--max-parallel %d" % int(max_parallel)) + if test_runner_timeout: + extra.append("--test-runner-timeout %d" % int(test_runner_timeout)) + for arg in extra_args: + extra.append(shlex.quote(str(arg))) + + extra_lines = "".join(" \\\n %s" % item for item in extra) + + if venv: + activate = shlex.quote(posixpath.join(str(venv), "bin", "activate")) + venv_activate = ("# activate the runner venv; `set +u` because older activate\n" + "# scripts read unset variables\nset +u\n" + ". %s\nset -u\n" % activate) + else: + venv_activate = "# no venv configured: ducktape is expected on PATH\n" + + return render_template("run.sh.tmpl", { + "version": version, + "timestamp": timestamp, + "author": author, + "cwd": shlex.quote(str(cwd)), + "venv_activate": venv_activate, + "results_root": shlex.quote(str(results_root)), + "cluster_file": shlex.quote(str(cluster_file)), + "globals_file": shlex.quote(str(globals_file)), + "extra_lines": extra_lines, + "test_paths": " ".join(shlex.quote(str(p)) for p in test_paths), + }) + + +def render_launch_script(paths: RunPaths): + """ + Render the wrapper that detaches the run and records its exit code. + + The run is detached from second zero rather than "detached later": the failure worth + surviving is the coordinator going away, and a run that is only detachable after the + fact does not survive it. ``setsid`` puts ducktape in its own session so a dropped + SSH connection cannot SIGHUP it and leave Ignite JVMs alive on every worker; where + ``setsid`` is missing we fall back to ``nohup`` plus ``disown``. + """ + run_dir = shlex.quote(paths.run_dir) + return """#!/usr/bin/env bash +# Generated by ducktests-remote. Waits on run.sh and records its exit code. +rd=%s +ps -o pgid= -p $$ 2>/dev/null | tr -d ' ' > "$rd/pgid" || true +bash "$rd/run.sh" >> "$rd/ducktape.log" 2>&1 < /dev/null +echo $? > "$rd/exit_code" +""" % run_dir + + +def render_detach_script(paths: RunPaths): + """Render the snippet that starts ``launch.sh`` detached and records its pid.""" + run_dir = shlex.quote(paths.run_dir) + return """set -eu +rd=%s +cd "$rd" +: > "$rd/ducktape.log" +if command -v setsid >/dev/null 2>&1; then + setsid nohup bash "$rd/launch.sh" > /dev/null 2>&1 < /dev/null & +else + nohup bash "$rd/launch.sh" > /dev/null 2>&1 < /dev/null & + disown 2>/dev/null || true +fi +echo $! > "$rd/pid" +cat "$rd/pid" +""" % run_dir diff --git a/modules/ducktests/tests/ducktests_remote/sshdiag.py b/modules/ducktests/tests/ducktests_remote/sshdiag.py new file mode 100644 index 0000000000000..2037571e5f7e6 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/sshdiag.py @@ -0,0 +1,271 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +SSH failure classification and the "what to ask your administrator" block. + +The common first-time experience is not a subtle bug, it is *"nobody has added me to +these machines yet"*. A raw ``Permission denied (publickey)`` repeated twelve times +tells the operator nothing about what to ask for, so every SSH failure is classified and +mapped to a concrete next action, and the whole inventory is always probed before +anything is reported. + +The patterns below are derived from OpenSSH's own message strings. They have not been +replayed against every target distribution's build, so treat the table as data: adding a +distro-specific string is a one-line change and ``checks/check_remote_sshdiag.py`` is +table-driven over recorded samples. +""" + +import re +import shutil +from dataclasses import dataclass +from typing import List, Optional + +from ducktests_remote.transport import Result, TransportError, run_local + +OK = "ok" +UNRESOLVED = "unresolved" +NO_SSHD = "no-sshd" +UNREACHABLE = "unreachable" +NO_ACCESS = "no-access" +NO_USER = "no-user" +HOSTKEY = "hostkey" +NO_SUDO = "no-sudo" +UNKNOWN = "unknown" + +# Ordered: the first pattern that matches wins, so the specific ones come first. +PATTERNS = ( + (HOSTKEY, r"REMOTE HOST IDENTIFICATION HAS CHANGED|Host key verification failed|" + r"host key .* has changed"), + (UNRESOLVED, r"Could not resolve hostname|Name or service not known|" + r"nodename nor servname provided|Temporary failure in name resolution|" + r"Name does not resolve"), + (NO_SSHD, r"Connection refused|port \d+: Connection refused"), + (UNREACHABLE, r"Connection timed out|Operation timed out|No route to host|" + r"Network is unreachable|Host is unreachable|" + r"kex_exchange_identification: Connection closed"), + (NO_USER, r"Invalid user|no such user|Please login as the user|" + r"This account is currently not available"), + (NO_ACCESS, r"Permission denied \(publickey|Permission denied \(.*publickey|" + r"Too many authentication failures|no matching host key type found|" + r"Authentication failed"), + (NO_SUDO, r"sudo: a (password|terminal) is required|sudo: no tty present|" + r"a terminal is required to read the password|" + r"is not in the sudoers file"), +) + +_ADVICE = { + UNRESOLVED: "hostname does not resolve from here. Check VPN/DNS, or put the IP in " + "cluster.nodes[].ip.", + NO_SSHD: "host answers but nothing is listening on port {port}; sshd is down or on " + "another port.", + UNREACHABLE: "no network path to {host}:{port}. Firewall, routing or the host is down.", + NO_ACCESS: "your key is not authorised for user {user!r} on this host.", + NO_USER: "account {user!r} does not exist on this host.", + HOSTKEY: "the host key changed. Inspect it, then remove the stale line yourself; this " + "tool will never do it for you.", + NO_SUDO: "passwordless sudo is missing for {user!r}.", + UNKNOWN: "unrecognised ssh failure; rerun with -v for the full stderr.", +} + + +@dataclass +class SshDiagnosis: + """One host's SSH state.""" + + host: str + classification: str = OK + user: Optional[str] = None + port: int = 22 + stderr: str = "" + returncode: int = 0 + + @property + def ok(self): + """:return: True when the connection succeeded.""" + return self.classification == OK + + @property + def advice(self): + """:return: a one-line, host-specific next action.""" + template = _ADVICE.get(self.classification, _ADVICE[UNKNOWN]) + return template.format(host=self.host, user=self.user, port=self.port) + + +def classify(returncode, stderr): + """ + Map an ssh exit status plus stderr onto one of the classification constants. + + :param returncode: process exit status; 0 always means ``ok``. + :param stderr: captured stderr, matched case-insensitively. + """ + if returncode == 0: + return OK + text = stderr or "" + for name, pattern in PATTERNS: + if re.search(pattern, text, re.IGNORECASE): + return name + return UNKNOWN + + +def diagnose(result: Result, *, host, user=None, port=22): + """:return: an :class:`SshDiagnosis` built from a command :class:`Result`.""" + return SshDiagnosis(host=host, + classification=classify(result.returncode, result.stderr), + user=user, port=port, + stderr=(result.stderr or "").strip(), + returncode=result.returncode) + + +def diagnose_exception(ex, *, host, user=None, port=22): + """:return: an :class:`SshDiagnosis` built from a raised transport error.""" + stderr = str(ex) + result = getattr(ex, "result", None) + if isinstance(result, Result): + return diagnose(result, host=host, user=user, port=port) + return SshDiagnosis(host=host, classification=classify(255, stderr), user=user, port=port, + stderr=stderr, returncode=255) + + +def key_fingerprint(identity_file): + """ + :return: the fingerprint of the public key being offered, or None. + + The administrator needs this to confirm they are authorising the right key, and the + operator usually cannot produce it from memory. + """ + if not identity_file: + return None + keygen = shutil.which("ssh-keygen") + if not keygen: + return None + for candidate in (str(identity_file) + ".pub", str(identity_file)): + try: + result = run_local([keygen, "-l", "-f", candidate]) + except TransportError: + continue + if result.ok and result.out: + return result.out + return None + + +def public_key(identity_file): + """:return: the contents of ``.pub`` when readable, else None.""" + if not identity_file: + return None + try: + with open(str(identity_file) + ".pub", "r", encoding="utf-8") as handle: + return handle.read().strip() + except OSError: + return None + + +def summarise(diagnoses: List[SshDiagnosis]): + """:return: ``9 ok, 2 no-access, 1 no-user``.""" + counts = {} + for diagnosis in diagnoses: + counts[diagnosis.classification] = counts.get(diagnosis.classification, 0) + 1 + ordered = sorted(counts.items(), key=lambda kv: (kv[0] != OK, kv[0])) + return ", ".join("%d %s" % (count, name) for name, count in ordered) + + +def admin_request_block(diagnoses: List[SshDiagnosis], *, user, identity_file=None): + """ + Render the copy-pasteable block the operator forwards to whoever owns the machines. + + This block is the point of the whole classification exercise; the table above is + plumbing. It names the hosts, the account, the key, and the exact line to append. + """ + failures = [d for d in diagnoses if not d.ok] + if not failures: + return "" + + by_class = {} + for diagnosis in failures: + by_class.setdefault(diagnosis.classification, []).append(diagnosis.host) + + lines = ["", "=" * 72, + "WHAT TO ASK YOUR ADMINISTRATOR", + "=" * 72, + "%d of %d hosts are not usable. Summary: %s" + % (len(failures), len(diagnoses), summarise(diagnoses)), + ""] + + fingerprint = key_fingerprint(identity_file) + pubkey = public_key(identity_file) + + if NO_ACCESS in by_class: + hosts = by_class[NO_ACCESS] + lines += ["Please authorise my SSH key for the account %r on these %d host(s):" + % (user, len(hosts)), + _host_list(hosts), ""] + if fingerprint: + lines += [" key fingerprint offered: %s" % fingerprint] + lines += [" append this line to ~%s/.ssh/authorized_keys on each host:" % user, + " %s" % (pubkey or "" % (identity_file or "your key")), + ""] + + if NO_USER in by_class: + hosts = by_class[NO_USER] + working = [d.host for d in diagnoses if d.ok] + lines += ["The account %r does not exist on these %d host(s):" % (user, len(hosts)), + _host_list(hosts)] + if working: + lines += [" (it does exist on: %s)" % ", ".join(sorted(working)[:10])] + lines += [" please create it, or tell me which account to use instead.", ""] + + if NO_SSHD in by_class: + lines += ["sshd is not listening on these host(s):", + _host_list(by_class[NO_SSHD]), ""] + + if UNREACHABLE in by_class: + lines += ["No network path to these host(s) on port %d; please check the firewall:" + % (failures[0].port or 22), + _host_list(by_class[UNREACHABLE]), ""] + + if UNRESOLVED in by_class: + lines += ["These hostnames do not resolve from my machine. Either grant DNS/VPN " + "access, or give me their IP addresses:", + _host_list(by_class[UNRESOLVED]), ""] + + if NO_SUDO in by_class: + lines += ["Passwordless sudo is missing for %r on these host(s):" % user, + _host_list(by_class[NO_SUDO]), + " It is needed only by the network-segmentation suites", + " (ignitetest/tests/discovery_test.py, " + "ignitetest/tests/cellular_affinity_test.py), which call", + " `sudo iptables` from IgniteAwareService.drop_network. Every other test " + "runs unprivileged.", + " Required line: %s ALL=(ALL) NOPASSWD: /usr/sbin/iptables, " + "/usr/sbin/iptables-save, /usr/sbin/iptables-restore" % user, + ""] + + if HOSTKEY in by_class: + lines += ["Host key mismatch on these host(s) - verify before removing anything:", + _host_list(by_class[HOSTKEY])] + for host in by_class[HOSTKEY]: + lines.append(" ssh-keygen -R %s" % host) + lines.append("") + + if UNKNOWN in by_class: + lines += ["Unclassified ssh failures (rerun with -v for the full output):", + _host_list(by_class[UNKNOWN]), ""] + + lines.append("=" * 72) + return "\n".join(lines) + + +def _host_list(hosts): + return "\n".join(" %s" % host for host in sorted(hosts)) diff --git a/modules/ducktests/tests/ducktests_remote/templates/run.sh.tmpl b/modules/ducktests/tests/ducktests_remote/templates/run.sh.tmpl new file mode 100644 index 0000000000000..e8c0e102b3ff9 --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/templates/run.sh.tmpl @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated by ducktests-remote {{version}} at {{timestamp}} by {{author}}. +# Do not edit: rerun `ducktests-remote run` instead. You can, however, ssh to this +# host and execute this exact script to reproduce the run by hand. +set -euo pipefail + +cd {{cwd}} +{{venv_activate}} +exec ducktape \ + --results-root {{results_root}} \ + --cluster-file {{cluster_file}} \ + --globals {{globals_file}}{{extra_lines}} \ + {{test_paths}} diff --git a/modules/ducktests/tests/ducktests_remote/transport.py b/modules/ducktests/tests/ducktests_remote/transport.py new file mode 100644 index 0000000000000..f9bf8307a370a --- /dev/null +++ b/modules/ducktests/tests/ducktests_remote/transport.py @@ -0,0 +1,728 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Command transport. + +Every command the CLI runs, locally or remotely, goes through a :class:`Transport`. +Nothing above this boundary may shell out to ``ssh`` directly, so ``--runner local`` and +``--runner some-host`` exercise identical code paths. +""" + +import abc +import os +import posixpath +import re +import shlex +import shutil +import subprocess +import tarfile +import tempfile +import threading +import uuid +from dataclasses import dataclass, field +from fnmatch import fnmatch +from pathlib import Path +from typing import List, Optional, Sequence, Union + +# OpenSSH on Windows has no ControlMaster support; multiplexing options make it fail hard. +_MULTIPLEXING_SUPPORTED = os.name != "nt" + +DEFAULT_SSH_OPTIONS = ( + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ServerAliveInterval=30", +) + + +class TransportError(Exception): + """Raised when a transport-level operation fails (exit code 5 territory).""" + + def __init__(self, message, result=None): + super().__init__(message) + self.result = result + + +@dataclass +class Result: + """Outcome of a single command.""" + + argv: List[str] + returncode: int + stdout: str = "" + stderr: str = "" + host: str = "local" + + @property + def ok(self): + """:return: True when the command exited successfully.""" + return self.returncode == 0 + + @property + def out(self): + """:return: stdout with surrounding whitespace removed.""" + return self.stdout.strip() + + def check(self): + """Raise :class:`TransportError` unless the command succeeded.""" + if not self.ok: + raise TransportError( + "command failed on %s (exit %d): %s\n%s" + % (self.host, self.returncode, shlex.join(self.argv), self.stderr.strip()), + self) + return self + + +@dataclass +class Transport(abc.ABC): # pylint: disable=too-many-instance-attributes + """Runs commands and moves files on exactly one host.""" + + name: str = "local" + dry_run: bool = False + verbose: bool = False + printer: Optional[object] = None + _home: Optional[str] = field(default=None, init=False, repr=False) + + def home(self): + """:return: the home directory of the account this transport connects as.""" + if self._home is None: + if self.dry_run: + # Nothing is executed in a dry run, so the remote home is unknown; keep + # the tilde so the preview stays readable rather than inventing a path. + self._home = "~" + else: + result = self.run(["printenv", "HOME"], check=False) + self._home = result.out or "/root" + return self._home + + def expand(self, path): + """ + Expand a leading ``~`` against the *remote* home directory. + + Paths are shell-quoted before they reach the remote side, so a literal tilde + would never be expanded by the remote shell. Resolving it here, once per + connection, keeps ``~/.ducktests-remote`` working without giving up quoting. + """ + text = str(path) + if text == "~": + return self.home() + if text.startswith("~/"): + return posixpath.join(self.home(), text[2:]) + return text + + def _echo(self, message): + if self.printer is not None: + self.printer(message) + + def _trace(self, argv): + if self.dry_run: + self._echo("[dry-run] %s$ %s" % (self.name, shlex.join(argv))) + elif self.verbose: + self._echo("+ %s$ %s" % (self.name, shlex.join(argv))) + + @abc.abstractmethod + def run(self, argv, *, check=True, timeout=None, input=None): # noqa: A002 - matches spec + """Run ``argv`` (a list, never a string) and return a :class:`Result`.""" + + def run_script(self, script, **kw): + """ + Feed a shell script to ``bash -s`` over stdin. + + Long quoted one-liners are the single largest source of remote-execution bugs; + anything longer than a couple of words belongs here instead. + """ + return self.run(["bash", "-s"], input=script, **kw) + + @abc.abstractmethod + def upload(self, local_path, remote_path, *, mode=None): + """Copy a single local file to ``remote_path``.""" + + def upload_watched(self, local_path, remote_path, *, mode=None, on_bytes=None): + """ + Copy a file, reporting bytes as they go out. + + The default is :meth:`upload` with no reporting at all; only transports that can + see the bytes leave override it. Callers therefore never have to ask whether + progress is available. + """ + del on_bytes + return self.upload(local_path, remote_path, mode=mode) + + @abc.abstractmethod + def download(self, remote_path, local_path): + """Copy a single remote file to ``local_path``.""" + + @abc.abstractmethod + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False, + on_progress=None): + """ + Copy a directory tree; ``local_dir`` contents land inside ``remote_dir``. + + ``on_progress(sent_bytes, fraction)`` is called as the transfer goes, with + ``fraction`` set only when the sender knows it. Implementations that cannot see + the bytes leave simply never call it. + """ + + def exists(self, remote_path): + """:return: True when ``remote_path`` exists on this host.""" + if self.dry_run: + return True + return self.run(["test", "-e", remote_path], check=False).ok + + def mkdirs(self, remote_path, *, mode=None): + """Create ``remote_path`` and any missing parents.""" + self.run(["mkdir", "-p", remote_path]).check() + if mode is not None: + self.run(["chmod", "%o" % mode, remote_path]).check() + + def write_file(self, content, remote_path, *, mode=None): + """ + Write ``content`` to ``remote_path`` without going through a shell quote. + + Written as bytes with explicit LF endings: a coordinator on Windows would + otherwise translate every newline to CRLF, and a shell script with carriage + returns fails on the runner with a message that names the wrong line. + """ + with tempfile.TemporaryDirectory() as tmp: + local = Path(tmp) / "payload" + local.write_bytes(content.replace("\r\n", "\n").encode("utf-8")) + self.upload(local, remote_path, mode=mode) + + def read_file(self, remote_path, *, missing_ok=True): + """:return: contents of ``remote_path``, or None when it does not exist.""" + result = self.run(["cat", remote_path], check=False) + if result.ok: + return result.stdout + if missing_ok: + return None + return result.check().stdout + + +class LocalTransport(Transport): + """Runs everything in the coordinator's own filesystem.""" + + def __init__(self, **kw): + super().__init__(name="local", **kw) + + def run(self, argv, *, check=True, timeout=None, input=None): # noqa: A002 - matches spec + argv = [str(a) for a in argv] + self._trace(argv) + if self.dry_run: + return Result(argv, 0, host=self.name) + return _spawn(argv, host=self.name, check=check, timeout=timeout, input=input) + + def upload(self, local_path, remote_path, *, mode=None): + self._trace(["cp", str(local_path), remote_path]) + if self.dry_run: + return + Path(remote_path).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(str(local_path), remote_path) + if mode is not None: + os.chmod(remote_path, mode) + + def upload_watched(self, local_path, remote_path, *, mode=None, on_bytes=None): + if on_bytes is None: + return self.upload(local_path, remote_path, mode=mode) + self._trace(["cp", str(local_path), remote_path]) + if self.dry_run: + return None + Path(remote_path).parent.mkdir(parents=True, exist_ok=True) + sent = 0 + with open(local_path, "rb") as source, open(remote_path, "wb") as target: + for block in iter(lambda: source.read(CHUNK), b""): + target.write(block) + sent += len(block) + on_bytes(sent) + if mode is not None: + os.chmod(remote_path, mode) + return None + + def download(self, remote_path, local_path): + self._trace(["cp", remote_path, str(local_path)]) + if self.dry_run: + return + Path(local_path).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(remote_path, str(local_path)) + + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False, + on_progress=None): + del on_progress # a local copy is not worth watching + self._trace(["cp", "-r", str(local_dir), remote_dir]) + if self.dry_run: + return + if delete and Path(remote_dir).exists(): + shutil.rmtree(remote_dir) + ignore = shutil.ignore_patterns(*[_exclude_to_glob(e) for e in excludes]) if excludes else None + shutil.copytree(str(local_dir), remote_dir, dirs_exist_ok=True, ignore=ignore, symlinks=True) + + +@dataclass +class SshTransport(Transport): + """ + Runs commands on a remote host through the *system* ``ssh``/``scp`` binaries. + + paramiko is deliberately not used: the system client brings ``~/.ssh/config``, + ``ProxyJump``, agent and Kerberos support along for free, and it is the same client + an engineer would use by hand when reproducing a failure. + """ + + user: Optional[str] = None + port: int = 22 + identity_file: Optional[str] = None + connect_timeout: int = 15 + control_dir: Optional[str] = None + _rsync: Optional[bool] = field(default=None, init=False, repr=False) + + def __post_init__(self): + if self.control_dir is None: + self.control_dir = tempfile.gettempdir() + + @property + def target(self): + """:return: ``user@host`` or plain host when no user is configured.""" + return "%s@%s" % (self.user, self.name) if self.user else self.name + + def ssh_options(self, *, for_scp=False): + """:return: the base ssh option list shared by every connection this class makes.""" + opts = list(DEFAULT_SSH_OPTIONS) + opts += ["-o", "ConnectTimeout=%d" % self.connect_timeout] + if _MULTIPLEXING_SUPPORTED: + # doctor and deploy open many connections per host; multiplexing turns each + # extra one into a no-cost channel on the first connection. + opts += ["-o", "ControlMaster=auto", + "-o", "ControlPersist=60s", + "-o", "ControlPath=%s" % os.path.join(self.control_dir, "dtr-%r@%h:%p")] + if self.identity_file: + opts += ["-o", "IdentitiesOnly=yes", "-i", os.path.expanduser(self.identity_file)] + if self.port and int(self.port) != 22: + opts += ["-P" if for_scp else "-p", str(self.port)] + return opts + + def run(self, argv, *, check=True, timeout=None, input=None): # noqa: A002 - matches spec + argv = [str(a) for a in argv] + remote = shlex.join(argv) + full = ["ssh"] + self.ssh_options() + ["-T", self.target, remote] + self._trace(argv) + if self.dry_run: + return Result(argv, 0, host=self.name) + result = _spawn(full, host=self.name, check=check, timeout=timeout, input=input) + return Result(argv, result.returncode, result.stdout, result.stderr, self.name) + + def upload(self, local_path, remote_path, *, mode=None): + self._trace(["scp", str(local_path), "%s:%s" % (self.target, remote_path)]) + if self.dry_run: + return + remote = "%s:%s" % (self.target, remote_path) + argv = ["scp"] + self.ssh_options(for_scp=True) + [str(local_path), remote] + _spawn(argv, host=self.name, check=True) + if mode is not None: + self.run(["chmod", "%o" % mode, remote_path]).check() + + def upload_watched(self, local_path, remote_path, *, mode=None, on_bytes=None): + """ + Send a file through ``ssh 'cat > path'``, counting the bytes on the way in. + + scp is the better tool and stays the default, but its progress meter is written + for a terminal and suppressed when stdout is a pipe - which it always is here. + Feeding the file to a remote ``cat`` ourselves is the only way to know how far a + 300 MB payload has got, and the write blocks on a slow link exactly as scp does. + """ + if on_bytes is None: + return self.upload(local_path, remote_path, mode=mode) + self._trace(["cat", str(local_path), ">", "%s:%s" % (self.target, remote_path)]) + if self.dry_run: + return None + command = "cat > %s" % shlex.quote(remote_path) + argv = ["ssh"] + self.ssh_options() + ["-T", self.target, command] + _stream_to_stdin(argv, local_path, on_bytes, host=self.name) + if mode is not None: + self.run(["chmod", "%o" % mode, remote_path]).check() + return None + + def download(self, remote_path, local_path): + self._trace(["scp", "%s:%s" % (self.target, remote_path), str(local_path)]) + if self.dry_run: + return + Path(local_path).parent.mkdir(parents=True, exist_ok=True) + remote = "%s:%s" % (self.target, remote_path) + argv = ["scp"] + self.ssh_options(for_scp=True) + [remote, str(local_path)] + _spawn(argv, host=self.name, check=True) + + def has_rsync(self): + """:return: True when rsync is usable on both ends. Probed once, then cached.""" + if self._rsync is None: + if self.dry_run: + self._rsync = False + else: + local = shutil.which("rsync") is not None + remote = local and self.run(["command", "-v", "rsync"], check=False).ok + self._rsync = bool(remote) + return self._rsync + + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False, + on_progress=None): + local_dir = str(local_dir) + self._trace(["rsync", local_dir, "%s:%s" % (self.target, remote_dir)]) + if self.dry_run: + return + self.mkdirs(remote_dir) + if self.has_rsync(): + argv = ["rsync", "-az", "-e", shlex.join(["ssh"] + self.ssh_options())] + if delete: + argv.append("--delete") + for pattern in excludes: + argv += ["--exclude", pattern] + argv += [local_dir.rstrip("/\\") + "/", "%s:%s/" % (self.target, remote_dir)] + if on_progress is None: + _spawn(argv, host=self.name, check=True) + return + argv.insert(1, "--info=progress2") + run_local_streaming(argv, host=self.name, + on_output=rsync_reporter(on_progress)).check() + return + # rsync missing on one of the ends: fall back to a tar stream through scp. + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / "payload.tar.gz" + make_tarball(local_dir, archive, excludes=excludes) + staged = "/tmp/dtr-upload-%s.tar.gz" % uuid.uuid4().hex[:8] + if on_progress is None: + self.upload(archive, staged) + else: + total = archive.stat().st_size + self.upload_watched(archive, staged, + on_bytes=lambda sent: on_progress(sent, sent / total)) + script = "set -eu\nmkdir -p %s\n" % shlex.quote(remote_dir) + if delete: + script += "rm -rf -- %s/*\n" % shlex.quote(remote_dir) + script += "tar -xzf %s -C %s\nrm -f -- %s\n" % ( + shlex.quote(staged), shlex.quote(remote_dir), shlex.quote(staged)) + self.run_script(script).check() + + +@dataclass +class ProxiedTransport(Transport): + """ + Runs commands on a host by hopping through another transport. + + Two uses: probing a worker over the connection *ducktape itself* will make (from the + runner, with the runner-side identity file), and ``deploy --via HOST``, where a + payload is uploaded once and then fanned out from a machine that is close to the + workers instead of once per host from a laptop. + """ + + via: Optional[Transport] = None + user: Optional[str] = None + port: int = 22 + identity_file: Optional[str] = None + connect_timeout: int = 15 + staging_dir: str = "/tmp" + + @property + def target(self): + """:return: ``user@host`` as seen from the intermediate host.""" + return "%s@%s" % (self.user, self.name) if self.user else self.name + + def ssh_options(self, *, for_scp=False): + """:return: ssh options for the second hop, evaluated on the intermediate host.""" + opts = list(DEFAULT_SSH_OPTIONS) + opts += ["-o", "ConnectTimeout=%d" % self.connect_timeout] + if self.identity_file: + opts += ["-o", "IdentitiesOnly=yes", "-i", self.identity_file] + if self.port and int(self.port) != 22: + opts += ["-P" if for_scp else "-p", str(self.port)] + return opts + + def run(self, argv, *, check=True, timeout=None, input=None): # noqa: A002 - matches spec + argv = [str(a) for a in argv] + hop = ["ssh"] + self.ssh_options() + ["-T", self.target, shlex.join(argv)] + result = self.via.run(hop, check=False, timeout=timeout, input=input) + result = Result(argv, result.returncode, result.stdout, result.stderr, self.name) + return result.check() if check else result + + def _staged(self, name): + return posixpath.join(self.staging_dir, "dtr-%s-%s" % (uuid.uuid4().hex[:8], name)) + + def upload(self, local_path, remote_path, *, mode=None): + staged = self._staged(Path(local_path).name) + self.via.upload(local_path, staged) + argv = ["scp"] + self.ssh_options(for_scp=True) + [staged, + "%s:%s" % (self.target, remote_path)] + try: + self.via.run(argv).check() + finally: + self.via.run(["rm", "-f", "--", staged], check=False) + if mode is not None: + self.run(["chmod", "%o" % mode, remote_path]).check() + + def download(self, remote_path, local_path): + staged = self._staged(posixpath.basename(remote_path)) + argv = ["scp"] + self.ssh_options(for_scp=True) + ["%s:%s" % (self.target, remote_path), + staged] + self.via.run(argv).check() + try: + self.via.download(staged, local_path) + finally: + self.via.run(["rm", "-f", "--", staged], check=False) + + def upload_dir(self, local_dir, remote_dir, *, excludes=(), delete=False, + on_progress=None): + del on_progress # the interesting hop is the second one, which scp cannot watch + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / "payload.tar.gz" + make_tarball(local_dir, archive, excludes=excludes) + staged = self._staged("payload.tar.gz") + self.via.upload(archive, staged) + try: + self.push_archive(staged, remote_dir, delete=delete) + finally: + self.via.run(["rm", "-f", "--", staged], check=False) + + def push_archive(self, staged_on_via, remote_dir, *, delete=False): + """Send an archive that already sits on the intermediate host to this host.""" + remote_tmp = self._staged("payload.tar.gz") + argv = ["scp"] + self.ssh_options(for_scp=True) + [ + staged_on_via, "%s:%s" % (self.target, remote_tmp)] + self.via.run(argv).check() + script = "set -eu\nmkdir -p %s\n" % shlex.quote(remote_dir) + if delete: + script += "rm -rf -- %s/*\n" % shlex.quote(remote_dir) + script += "tar -xzf %s -C %s\nrm -f -- %s\n" % ( + shlex.quote(remote_tmp), shlex.quote(remote_dir), shlex.quote(remote_tmp)) + self.run_script(script).check() + + +def make_tarball(source_dir, archive_path, *, excludes=()): + """Create a gzip tarball of the *contents* of ``source_dir``, honouring rsync-ish excludes.""" + source = Path(source_dir) + with tarfile.open(archive_path, "w:gz") as tar: + for entry in sorted(source.rglob("*")): + if entry.is_dir(): + continue + rel = entry.relative_to(source) + if is_excluded(rel, excludes): + continue + tar.add(str(entry), arcname=rel.as_posix()) + + +def is_excluded(rel_path, excludes): + """ + :return: True when ``rel_path`` matches one of the rsync-style ``excludes``. + + A pattern matches a whole relative path, any prefix of it, or any single path + component, so ``.git/``, ``*.pyc`` and ``target/`` all behave as expected. + """ + rel = Path(rel_path) + posix = rel.as_posix() + for raw in excludes: + pattern = raw.rstrip("/") + if not pattern: + continue + if fnmatch(posix, pattern) or fnmatch(posix, pattern + "/*"): + return True + if any(fnmatch(part, pattern) for part in rel.parts): + return True + return False + + +def _exclude_to_glob(exclude): + return exclude.rstrip("/") or exclude + + +def _spawn(argv, *, host, check=True, timeout=None, input=None): # noqa: A002 - matches spec + payload = input.encode("utf-8") if isinstance(input, str) else input + try: + completed = subprocess.run( # noqa: S603 - argv is always a list, never a shell string + argv, input=payload, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=timeout, check=False) + except FileNotFoundError as ex: + raise TransportError("%s: %s" % (argv[0], ex)) from ex + except subprocess.TimeoutExpired as ex: + raise TransportError("timed out after %ss on %s: %s" % (timeout, host, shlex.join(argv))) from ex + result = Result(argv, completed.returncode, + _decode(completed.stdout), _decode(completed.stderr), host) + if check: + result.check() + return result + + +def _decode(raw): + if raw is None: + return "" + return raw.decode("utf-8", errors="replace") + + +def run_local(argv, *, check=False, timeout=None, input=None): # noqa: A002 - matches spec + """Run a command on the coordinator itself, outside any transport (probes only).""" + return _spawn([str(a) for a in argv], host="local", check=check, timeout=timeout, input=input) + + +CHUNK = 1024 * 1024 + +# `--info=progress2`: ` 1,234,567 35% 12.34MB/s 0:00:12`. The separators are +# locale-dependent, the layout is not. +_RSYNC_PROGRESS = re.compile(r"^\s*([\d,.]+)\s+(\d+)%\s") + + +def parse_rsync_progress(line): + """:return: ``(bytes_so_far, fraction)`` from one ``--info=progress2`` line, or None.""" + match = _RSYNC_PROGRESS.match(line or "") + if not match: + return None + try: + return int(re.sub(r"[,.]", "", match.group(1))), int(match.group(2)) / 100.0 + except ValueError: + return None + + +def rsync_reporter(on_progress): + """:return: an ``on_output`` callback that turns rsync's meter into ``on_progress``.""" + def watch(line): + reading = parse_rsync_progress(line) + if reading: + on_progress(reading[0], reading[1]) + return watch + + +def run_local_streaming(argv, *, on_output, host="local"): + """ + Run ``argv`` locally, handing each output line to ``on_output`` as it appears. + + :return: a :class:`Result` holding the whole output, so a caller can still parse a + summary the command printed at the end. + + Progress meters separate their updates with carriage returns rather than newlines, + so both count as line endings here. stderr is drained by a thread: a command that + fills the stderr pipe while nobody reads it deadlocks, and rsync is chatty on stderr + when a host misbehaves. + """ + argv = [str(a) for a in argv] + try: + # pylint: disable=consider-using-with + process = subprocess.Popen( # noqa: S603 - argv is always a list, never a string + argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0) + except FileNotFoundError as ex: + raise TransportError("%s: %s" % (argv[0], ex)) from ex + + errors = {} + drain = _drain(process.stderr, "stderr", errors) + + collected = [] + pending = "" + while True: + block = process.stdout.read(4096) + if not block: + break + text = _decode(block) + collected.append(text) + pending += text + pending = _emit_lines(pending, on_output) + if pending.strip(): + on_output(pending.strip()) + + process.wait() + drain.join(timeout=5) + return Result(argv, process.returncode, "".join(collected), errors.get("stderr", ""), host) + + +def _emit_lines(pending, on_output): + """:return: the unterminated tail; every complete line goes to ``on_output``.""" + pending = pending.replace("\r\n", "\n") + while True: + cut = min((i for i in (pending.find("\n"), pending.find("\r")) if i >= 0), default=-1) + if cut < 0: + return pending + line, pending = pending[:cut], pending[cut + 1:] + if line.strip(): + on_output(line.strip()) + + +def _drain(pipe, key, into): + """:return: a started daemon thread reading ``pipe`` to EOF into ``into[key]``.""" + def read(): + try: + into[key] = _decode(pipe.read()) + except (OSError, ValueError): + into[key] = "" + thread = threading.Thread(target=read, daemon=True) + thread.start() + return thread + + +def _stream_to_stdin(argv, local_path, on_bytes, *, host): + """ + Feed a file into ``argv``'s stdin in chunks, reporting the running total. + + Both output pipes are drained by threads for the whole transfer rather than collected + at the end by ``communicate``. Two reasons, and the second one is why this function + does its own bookkeeping instead of writing four lines: + + * Nothing would read them while a multi-gigabyte payload is being written, and an ssh + that says anything at all - a login banner, a host-key notice - fills its pipe and + wedges the write part way through, with no error and no end. + * ``communicate`` flushes stdin before closing it, and guards only ``BrokenPipeError`` + while doing so. A handle closed here would make that flush raise + ``ValueError('flush of closed file')`` - after the payload is already on the far + end, so a transfer that fully succeeded is reported as a failure. + """ + try: + # pylint: disable=consider-using-with + process = subprocess.Popen( # noqa: S603 - argv is always a list, never a string + argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except FileNotFoundError as ex: + raise TransportError("%s: %s" % (argv[0], ex)) from ex + + output = {} + readers = [_drain(process.stdout, "stdout", output), + _drain(process.stderr, "stderr", output)] + + sent = 0 + try: + with open(local_path, "rb") as handle: + for block in iter(lambda: handle.read(CHUNK), b""): + process.stdin.write(block) + sent += len(block) + on_bytes(sent) + except (BrokenPipeError, OSError): + # The far end died mid-transfer; its stderr says why, so fall through to wait(). + pass + finally: + # The close is what tells the remote `cat` that the file is over, so it has to + # happen even when the write above failed. + try: + process.stdin.close() + except (BrokenPipeError, OSError): + pass + + process.wait() + for reader in readers: + reader.join(timeout=5) + result = Result(argv, process.returncode, output.get("stdout", ""), + output.get("stderr", ""), host) + return result.check() + + +def build_transport(host: str, *, user=None, port=22, identity_file=None, connect_timeout=15, + dry_run=False, verbose=False, printer=None) -> Transport: + """:return: a :class:`LocalTransport` for ``local``, otherwise an :class:`SshTransport`.""" + if host in (None, "", "local", "localhost"): + return LocalTransport(dry_run=dry_run, verbose=verbose, printer=printer) + return SshTransport(name=host, user=user, port=int(port or 22), identity_file=identity_file, + connect_timeout=connect_timeout, dry_run=dry_run, verbose=verbose, + printer=printer) + + +def quote_all(values: Sequence[Union[str, Path]]) -> str: + """:return: the values shell-quoted and space joined.""" + return " ".join(shlex.quote(str(v)) for v in values) diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py index 06d8799077657..2c3ded900f311 100644 --- a/modules/ducktests/tests/setup.py +++ b/modules/ducktests/tests/setup.py @@ -31,4 +31,11 @@ packages=find_packages(exclude=["ignitetest.tests", "ignitetest.tests.*"]), include_package_data=True, install_requires=open('docker/requirements.txt').read(), + entry_points={ + "console_scripts": [ + # Coordinator-side CLI for running ducktests on a real VM cluster. + # It never imports ducktape; see ducktests_remote/README.md. + "ducktests-remote = ducktests_remote.cli:main", + ], + }, tests_require=["pytest==6.2.5"])