diff --git a/CHANGELOG.md b/CHANGELOG.md index e473826b8..acc668c91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +* **role:postgresql_server**: The daily dump runs with low CPU and IO priority, so it no longer slows down PostgreSQL. +* **role:mongodb**: The daily dump runs with low CPU and IO priority, so it no longer slows down MongoDB. +* **role:mariadb_server**: The daily dump runs with low CPU and IO priority, so it no longer slows down MariaDB. +* **role:influxdb**: The daily dump runs with low CPU and IO priority, so it no longer slows down InfluxDB. +* **role:gitlab_ce**: The daily dump runs with low CPU and IO priority, so it no longer slows down GitLab. * **role:redis, role:valkey**: Redis and Valkey also listen on a Unix socket that only the members of their group may use, on RHEL at the path the package ships (`/run/redis/redis.sock`, `/run/valkey/valkey.sock`). * **role:apache_httpd**: Apache answers requests for PHP files that do not exist with 404 itself instead of passing them to PHP-FPM, so scanners no longer tie up pool workers or fill the error log. * **playbook:setup_basic**: Checks the variables of all its roles, the compiled firewall in the Firewall Builder repository and the reachability of the Icinga2 master before it changes anything on the host, so a missing prerequisite no longer aborts the run halfway through, after SSH was already hardened. @@ -125,6 +130,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +* **role:mariadb_server**: The daily dump works with the mydumper that Debian and Ubuntu ship. * **role:lvm**: `growpart: true` grows the right partition when the PV is partition 10 or higher, such as `/dev/vda12` or `/dev/nvme0n1p12`. Until now it addressed partition 2 there, or failed. * **role:lvm**: The argument specs name the `fstype` subkey correctly and accept `lvm__lvs__dependent_var` and `lvm__vgs__dependent_var`, so the `setup_*` playbooks can inject LVs and VGs. * **role:repo_mongodb**: On RHEL, the repository key always matches `repo_mongodb__version`, instead of reusing the key of an earlier run with another version, which made the installation fail. diff --git a/extensions/molecule/setup_nextcloud/inventory/group_vars/systems_under_test.yml b/extensions/molecule/setup_nextcloud/inventory/group_vars/systems_under_test.yml index aaccef6e0..5915dd03b 100644 --- a/extensions/molecule/setup_nextcloud/inventory/group_vars/systems_under_test.yml +++ b/extensions/molecule/setup_nextcloud/inventory/group_vars/systems_under_test.yml @@ -15,6 +15,11 @@ icinga2_master__downtime_api_user: mariadb_server__admin_user: username: 'mariadb-admin' password: 'linuxfabrik' +# setting the dump user enables the daily dump (mariadb-dump.service and .timer) +mariadb_server__dump_user: + username: 'mariadb-backup' + password: 'linuxfabrik' + state: 'present' nextcloud__fqdn: 'nextcloud.example.com' nextcloud__skip_notify_push: true diff --git a/extensions/molecule/setup_nextcloud/verify.yml b/extensions/molecule/setup_nextcloud/verify.yml index 7735e45cd..54ed7b941 100644 --- a/extensions/molecule/setup_nextcloud/verify.yml +++ b/extensions/molecule/setup_nextcloud/verify.yml @@ -73,3 +73,34 @@ fail_msg: 'mariadbd runs as "{{ __molecule__mariadbd_label_result["stdout"] | trim }}" instead of mysqld_t, so the mysqld_exec_t label on the binary is missing.' when: - '__molecule__mariadbd_label_result is not skipped' + + +# The dump must not slow down MariaDB. Ask systemd for the effective values instead of reading the +# unit file, so a unit that fails to load or a setting it rejects shows up here. +- name: 'Verify mariadb-dump.service runs with low CPU and IO priority' + hosts: 'systems_under_test' + gather_facts: false + become: true + tasks: + + - name: 'systemctl show mariadb-dump.service' + ansible.builtin.command: >- + systemctl show mariadb-dump.service + --property=CPUWeight,IOSchedulingClass,IOSchedulingPriority,IOWeight,Nice + register: '__molecule__mariadb_dump_show_result' + changed_when: false + + - name: 'Assert the dump runs niced, best-effort and with a low CPU and IO weight' + ansible.builtin.assert: + that: + - '"CPUWeight=20" in __molecule__mariadb_dump_show_result["stdout_lines"]' + - '"IOSchedulingClass=2" in __molecule__mariadb_dump_show_result["stdout_lines"]' # 2 = best-effort + - '"IOSchedulingPriority=7" in __molecule__mariadb_dump_show_result["stdout_lines"]' + - '"IOWeight=20" in __molecule__mariadb_dump_show_result["stdout_lines"]' + - '"Nice=19" in __molecule__mariadb_dump_show_result["stdout_lines"]' + fail_msg: 'mariadb-dump.service has unexpected priorities: {{ __molecule__mariadb_dump_show_result["stdout_lines"] }}' + + # the dump itself has to work with the mydumper of each platform, not just carry the settings + - name: 'systemctl start mariadb-dump.service' + ansible.builtin.command: 'systemctl start mariadb-dump.service' + changed_when: false diff --git a/roles/gitlab_ce/README.md b/roles/gitlab_ce/README.md index fb1d4d7c8..a2a74ed64 100644 --- a/roles/gitlab_ce/README.md +++ b/roles/gitlab_ce/README.md @@ -9,6 +9,11 @@ This role installs and configures [GitLab CE](https://about.gitlab.com/), includ *Available since LFOps `2.0.0`.* +## How the Role Behaves + +* `gitlab-dump.service` runs with `Nice=19`, best-effort IO priority 7 and `CPUWeight=` / `IOWeight=` of `20` (systemd default: `100`), so the dump does not slow down the application on a busy host. The weights only take effect under contention. `IOWeight=` only has an effect on block devices that use the BFQ IO scheduler; with `none` or `mq-deadline`, as on a RHEL 9 VM with virtio disks, it is ignored. + + ## Dependent Roles Any [LFOps playbook](https://github.com/Linuxfabrik/lfops/blob/main/playbooks/README.md) that installs this role runs these for you. Optional ones can be disabled via the playbook's skip variables. diff --git a/roles/gitlab_ce/templates/etc/systemd/system/gitlab-dump.service.j2 b/roles/gitlab_ce/templates/etc/systemd/system/gitlab-dump.service.j2 index 6cea073f0..25786b972 100644 --- a/roles/gitlab_ce/templates/etc/systemd/system/gitlab-dump.service.j2 +++ b/roles/gitlab_ce/templates/etc/systemd/system/gitlab-dump.service.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -# 2022091501 +# 2026101301 [Unit] Description=Dumps (Backups) for GitLab @@ -10,3 +10,10 @@ Description=Dumps (Backups) for GitLab ExecStart=-/usr/bin/gitlab-backup create Type=oneshot User=root +# the dump must not slow down GitLab and the applications behind it. Nice= and the CPU / IO +# weights (default 100) only matter under contention, so an idle host dumps at full speed. +Nice=19 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +CPUWeight=20 +IOWeight=20 diff --git a/roles/influxdb/README.md b/roles/influxdb/README.md index 3a7e3b11a..8189ced8a 100644 --- a/roles/influxdb/README.md +++ b/roles/influxdb/README.md @@ -6,6 +6,11 @@ This role installs and configures [InfluxDB](https://www.influxdata.com/products *Available since LFOps `2.0.0`.* +## How the Role Behaves + +* `influxdb-dump.service` runs with `Nice=19`, best-effort IO priority 7 and `CPUWeight=` / `IOWeight=` of `20` (systemd default: `100`), so the dump does not slow down the application on a busy host. The weights only take effect under contention. `IOWeight=` only has an effect on block devices that use the BFQ IO scheduler; with `none` or `mq-deadline`, as on a RHEL 9 VM with virtio disks, it is ignored. + + ## Dependent Roles Any [LFOps playbook](https://github.com/Linuxfabrik/lfops/blob/main/playbooks/README.md) that installs this role runs these for you. Optional ones can be disabled via the playbook's skip variables. diff --git a/roles/influxdb/templates/etc/systemd/system/influxdb-dump.service.j2 b/roles/influxdb/templates/etc/systemd/system/influxdb-dump.service.j2 index fc6760e7a..149a0d423 100644 --- a/roles/influxdb/templates/etc/systemd/system/influxdb-dump.service.j2 +++ b/roles/influxdb/templates/etc/systemd/system/influxdb-dump.service.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -# 2022050901 +# 2026101301 [Unit] Description=Dumps (Backups) for InfluxDB @@ -8,3 +8,10 @@ Description=Dumps (Backups) for InfluxDB ExecStart=/usr/local/bin/influxdb-dump Type=oneshot User=root +# the dump must not slow down InfluxDB and the applications behind it. Nice= and the CPU / IO +# weights (default 100) only matter under contention, so an idle host dumps at full speed. +Nice=19 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +CPUWeight=20 +IOWeight=20 diff --git a/roles/mariadb_server/README.md b/roles/mariadb_server/README.md index 906128726..4a7471f0e 100644 --- a/roles/mariadb_server/README.md +++ b/roles/mariadb_server/README.md @@ -25,6 +25,11 @@ By default, Ansible runs each task on all hosts affected by a play before starti *Available since LFOps `2.0.0`.* +## How the Role Behaves + +* `mariadb-dump.service` runs with `Nice=19`, best-effort IO priority 7 and `CPUWeight=` / `IOWeight=` of `20` (systemd default: `100`), so the dump does not slow down the application on a busy host. The weights only take effect under contention. `IOWeight=` only has an effect on block devices that use the BFQ IO scheduler; with `none` or `mq-deadline`, as on a RHEL 9 VM with virtio disks, it is ignored. The IO class is deliberately not `idle`: `mydumper` holds a read lock on every non-InnoDB table (MyISAM, Aria, most of `mysql.*`) until it has dumped that table, and an idle dump could stretch that lock and keep writes waiting. + + ## Dependent Roles Any [LFOps playbook](https://github.com/Linuxfabrik/lfops/blob/main/playbooks/README.md) that installs this role runs these for you. Optional ones can be disabled via the playbook's skip variables. diff --git a/roles/mariadb_server/templates/etc/systemd/system/mariadb-dump.service.j2 b/roles/mariadb_server/templates/etc/systemd/system/mariadb-dump.service.j2 index 0fccb86bb..daa261ed5 100644 --- a/roles/mariadb_server/templates/etc/systemd/system/mariadb-dump.service.j2 +++ b/roles/mariadb_server/templates/etc/systemd/system/mariadb-dump.service.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -# 2020112301 +# 2026101301 [Unit] Description=mariadbdump Service @@ -8,6 +8,16 @@ Description=mariadbdump Service ExecStart=/usr/local/bin/mariadb-dump Type=oneshot User=root +# the dump must not slow down MariaDB and the applications behind it. Nice= and the CPU / IO +# weights (default 100) only matter under contention, so an idle host dumps at full speed. +# best-effort instead of idle: mydumper holds `LOCK TABLES ... READ LOCAL` on every non-InnoDB +# table (MyISAM, Aria, most of `mysql.*`) until that table is dumped, so a starved dump keeps +# writes to these tables waiting. verified against mydumper 1.0.11 and MariaDB 11.8 on Rocky 9. +Nice=19 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +CPUWeight=20 +IOWeight=20 [Install] WantedBy=basic.target diff --git a/roles/mariadb_server/templates/usr/local/bin/mariadb-dump.j2 b/roles/mariadb_server/templates/usr/local/bin/mariadb-dump.j2 index 8354457e8..e430c131b 100644 --- a/roles/mariadb_server/templates/usr/local/bin/mariadb-dump.j2 +++ b/roles/mariadb_server/templates/usr/local/bin/mariadb-dump.j2 @@ -6,7 +6,7 @@ # License: The Unlicense, see LICENSE file. # {{ ansible_managed }} -# 2026022701 +# 2026101302 # no `set -e` because we change some things at the beginning of the script @@ -20,6 +20,12 @@ function exit_with_first_failed_retc() { } +# true if version $1 is the same as or newer than version $2 (plain `X.Y.Z[-N]`, no leading "v") +function version_ge() { + [ "$(printf '%s\n%s\n' "$1" "$2" | sort --version-sort | tail -n 1)" == "$1" ] +} + + # make sure the backup is not readable by others old_umask=$(umask) umask 027 @@ -46,8 +52,10 @@ mysql_table_lock_retc=$? # https://github.com/mydumper/mydumper -# get installed mydumper version ("mydumper v0.15.1-3, built against MySQL 8.0.33-25 with SSL support") -mydumper_version=$(mydumper --version | awk '{print $2}' | tr --delete ',') +# get installed mydumper version ("mydumper v0.15.1-3, built against MySQL 8.0.33-25 with SSL support" +# on RHEL, "mydumper 0.10.0, built against MySQL 8.0.36" in the Debian and Ubuntu repositories). +# strip the comma and the leading "v", so that version_ge() compares the same way for both. +mydumper_version=$(mydumper --version | awk '{print $2}' | tr --delete ',' | sed 's/^v//') # build the cmdline parameters for mydumper mydumper_params="--host=$HOST " @@ -57,9 +65,8 @@ mydumper_params+="--build-empty-files " if [ -n "$COMPRESS" ]; then mydumper_params+="--compress=$COMPRESS " fi -if [[ "$(echo -e "$mydumper_version\nv0.15.2-7" | sort --version-sort | tail -n1)" == "$mydumper_version" ]]; then - # if $(mydumper --version) > 0.15.2-7: - # parameter available since mydumper v0.15.2-8+ +if version_ge "$mydumper_version" 0.15.2-8; then + # parameter available since mydumper v0.15.2-8 mydumper_params+="--dirty " fi mydumper_params+="--events " @@ -67,9 +74,18 @@ mydumper_params+="--logfile=$BACKUP_DIR/mydumper.log " mydumper_params+="--long-query-guard=$LONG_QUERY_GUARD " mydumper_params+="--outputdir=$BACKUP_DIR " mydumper_params+="--routines " +if [ "$THREADS" -eq 0 ] && ! version_ge "$mydumper_version" 0.15.3-1; then + # 0 means "number of CPUs" only since mydumper v0.15.3-1; older versions (0.10.x in the + # Debian and Ubuntu repositories) wait forever without a single worker + THREADS=$(nproc) +fi mydumper_params+="--threads=$THREADS " mydumper_params+="--triggers " -mydumper_params+="--trx-tables=0 " +if version_ge "$mydumper_version" 0.20.1-1; then + # `--trx-tables=0` takes an argument since mydumper v0.20.1-1, where it also turned on by + # default. older versions do not know the argument and behave like "0" anyway. + mydumper_params+="--trx-tables=0 " +fi mydumper_params+="--verbose=3 " mydumper_params+="$RAW " diff --git a/roles/mongodb/README.md b/roles/mongodb/README.md index d9a96dc5d..7c5417111 100644 --- a/roles/mongodb/README.md +++ b/roles/mongodb/README.md @@ -16,6 +16,11 @@ This role is only compatible with the following MongoDB versions: *Available since LFOps `2.0.0`.* +## How the Role Behaves + +* `mongodb-dump.service` runs with `Nice=19`, best-effort IO priority 7 and `CPUWeight=` / `IOWeight=` of `20` (systemd default: `100`), so the dump does not slow down the application on a busy host. The weights only take effect under contention. `IOWeight=` only has an effect on block devices that use the BFQ IO scheduler; with `none` or `mq-deadline`, as on a RHEL 9 VM with virtio disks, it is ignored. The IO class is deliberately not `idle`: the file-based method keeps the instance write-locked while it copies `/var/lib/mongo`, and an idle copy could stretch that lock. + + ## Known Limitations * MongoDB publishes no server packages for Debian 13 and Ubuntu 26.04, so the role cannot install MongoDB there. diff --git a/roles/mongodb/templates/etc/systemd/system/mongodb-dump.service.j2 b/roles/mongodb/templates/etc/systemd/system/mongodb-dump.service.j2 index 5341f0e03..581618390 100644 --- a/roles/mongodb/templates/etc/systemd/system/mongodb-dump.service.j2 +++ b/roles/mongodb/templates/etc/systemd/system/mongodb-dump.service.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -# 2022083101 +# 2026101301 [Unit] Description=MongoDB Dump Service @@ -8,6 +8,16 @@ Description=MongoDB Dump Service ExecStart=/usr/local/sbin/mongodb-dump Type=oneshot User=root +# the dump must not slow down mongod and the applications behind it. Nice= and the CPU / IO +# weights (default 100) only matter under contention, so an idle host dumps at full speed. +# best-effort instead of idle: with `mongodb__dump_method_file_based` the instance stays +# write-locked (fsyncLock) while /var/lib/mongo is copied, and an idle IO class could starve +# the copy and stretch that lock. +Nice=19 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +CPUWeight=20 +IOWeight=20 [Install] WantedBy=basic.target diff --git a/roles/postgresql_server/README.md b/roles/postgresql_server/README.md index 9cec62940..a085af59a 100644 --- a/roles/postgresql_server/README.md +++ b/roles/postgresql_server/README.md @@ -6,6 +6,11 @@ This role installs and configures a [PostgreSQL](https://www.postgresql.org/) se *Available since LFOps `2.0.0`.* +## How the Role Behaves + +* `postgresql-dump.service` runs with `Nice=19`, best-effort IO priority 7 and `CPUWeight=` / `IOWeight=` of `20` (systemd default: `100`), so the dump does not slow down the application on a busy host. The weights only take effect under contention. `IOWeight=` only has an effect on block devices that use the BFQ IO scheduler; with `none` or `mq-deadline`, as on a RHEL 9 VM with virtio disks, it is ignored. + + ## Dependent Roles Any [LFOps playbook](https://github.com/Linuxfabrik/lfops/blob/main/playbooks/README.md) that installs this role runs these for you. Optional ones can be disabled via the playbook's skip variables. diff --git a/roles/postgresql_server/templates/etc/systemd/system/postgresql-dump.service.j2 b/roles/postgresql_server/templates/etc/systemd/system/postgresql-dump.service.j2 index 0e3c793ff..d71743e86 100644 --- a/roles/postgresql_server/templates/etc/systemd/system/postgresql-dump.service.j2 +++ b/roles/postgresql_server/templates/etc/systemd/system/postgresql-dump.service.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -# 2022111801 +# 2026101301 [Unit] Description=PostgreSQL Dump Service @@ -8,6 +8,13 @@ Description=PostgreSQL Dump Service ExecStart=/usr/local/sbin/postgresql-dump Type=oneshot User=root +# the dump must not slow down PostgreSQL and the applications behind it. Nice= and the CPU / IO +# weights (default 100) only matter under contention, so an idle host dumps at full speed. +Nice=19 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +CPUWeight=20 +IOWeight=20 [Install] WantedBy=basic.target