fix(roles): keep the Icinga API credentials away from local users

CHANGELOG.md

@@ -122,6 +122,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+* **role:borg_local**: The Icinga downtime for `clamd@scan` during a backup is set, where Icinga had rejected the request as invalid JSON.
 * **role:acme_sh**: A changed `acme-sh.service` or `acme-sh.timer`, for example a new schedule, takes effect in the same run. Until now systemd kept the old unit until the next reload.
 * **role:clamav**: A changed `clamav-clamdscan.service` takes effect in the same run. Until now only a changed timer reloaded systemd.
 * **role:gitlab_ce**: A changed `gitlab-dump.service` takes effect in the same run. Until now only a changed timer reloaded systemd.
@@ -237,6 +238,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security
 
+* **role:borg_local, role:schedule_reboot, role:tools**: Neither the password of the Icinga API user nor the Rocket.Chat webhook of `schedule_reboot` shows up in the process list or in a file that every local user can read, which moves `schedule-icinga-downtime` from a shell function in `/etc/profile.d/alias.sh` to a command in `/usr/local/sbin` that only root may run.
 * **role:openvpn_server**: The server negotiates TLS 1.2 only with forward secrecy and checks client certificates against OpenSSL security level 2 ([#358](https://github.com/Linuxfabrik/lfops/issues/358)).
 * **role:apache_httpd**: The OWASP Core Rule Set archive is checked against a SHA-256 checksum before it is deployed, so a tampered or corrupted download aborts the run.
 * **role:monitoring_plugins**: The source install checks every pinned Python dependency against the checksums in the lockfile instead of installing whatever PyPI serves, and puts the sudoers drop-ins in place only once `visudo` accepts them.

extensions/molecule/schedule_reboot/no_reboot/verify.yml

@@ -37,6 +37,20 @@
           - '__molecule__spool_stat_result["stat"]["isdir"]'
           - '__molecule__spool_stat_result["stat"]["mode"] == "0755"'
 
+    # do-reboot carries the credentials of the Icinga API user and the Rocket.Chat webhook, and
+    # only schedule-reboot.service runs it.
+    - name: 'stat /usr/local/libexec/do-reboot'
+      ansible.builtin.stat:
+        path: '/usr/local/libexec/do-reboot'
+      register: '__molecule__do_reboot_stat_result'
+
+    - name: 'Assert that do-reboot is readable by root only'
+      ansible.builtin.assert:
+        that:
+          - '__molecule__do_reboot_stat_result["stat"]["mode"] == "0700"'
+          - '__molecule__do_reboot_stat_result["stat"]["pw_name"] == "root"'
+        fail_msg: '/usr/local/libexec/do-reboot is {{ __molecule__do_reboot_stat_result["stat"] | d("missing") | to_json }}'
+
     # Run the actor against the empty post-converge spool. do-reboot must exit 0
     # without rebooting; that it runs at all proves the usr_t libexec script is
     # executable under systemd/SELinux (the FHS-placement concern).

extensions/molecule/setup_basic/inventory/group_vars/systems_under_test.yml

@@ -29,3 +29,10 @@ postfix__sender_canonicals__host_var:
 setup_basic__skip_duplicity: true
 setup_basic__skip_repo_icinga: true
 setup_basic__skip_icinga2_agent: true
+
+# Deploys schedule-icinga-downtime. Nothing listens on the URL, the script only has to exist and
+# keep the credentials to root. The password is unique so verify.yml can search for it.
+tools__icinga2_api_url: 'https://127.0.0.1:5665'
+tools__icinga2_api_user_login:
+  username: 'downtime-api'
+  password: 'linuxfabrik-downtime-api'

extensions/molecule/setup_basic/verify.yml

@@ -289,3 +289,47 @@
       ansible.builtin.assert:
         that: '__molecule__psi_cpu_result["rc"] == 0'
         fail_msg: '{{ __molecule__psi_cpu_result["stdout"] }}'
+
+# The Icinga API credentials live in /usr/local/sbin/schedule-icinga-downtime, which only root may
+# read. A login shell of an unprivileged user sources /etc/profile.d/alias.sh, where they used to
+# be, so it must neither see them nor be able to run the script.
+- name: 'Verify the Icinga API credentials of the tools role are readable by root only'
+  hosts: 'systems_under_test'
+  gather_facts: false
+  become: true
+  tasks:
+
+    - name: 'stat /usr/local/sbin/schedule-icinga-downtime'
+      ansible.builtin.stat:
+        path: '/usr/local/sbin/schedule-icinga-downtime'
+      register: '__molecule__schedule_icinga_downtime_stat_result'
+
+    - name: 'Assert that schedule-icinga-downtime is readable by root only'
+      ansible.builtin.assert:
+        that:
+          - '__molecule__schedule_icinga_downtime_stat_result["stat"]["mode"] == "0700"'
+          - '__molecule__schedule_icinga_downtime_stat_result["stat"]["pw_name"] == "root"'
+        fail_msg: '/usr/local/sbin/schedule-icinga-downtime is {{ __molecule__schedule_icinga_downtime_stat_result["stat"] | d("missing") | to_json }}'
+
+    - name: 'runuser --user nobody -- bash --login -c "alias; declare -f; /usr/local/sbin/schedule-icinga-downtime 60"'
+      ansible.builtin.command:
+        argv:
+          - 'runuser'
+          - '--user'
+          - 'nobody'
+          - '--'
+          - 'bash'
+          - '--login'
+          - '-c'
+          - 'alias; declare -f; /usr/local/sbin/schedule-icinga-downtime 60'
+      register: '__molecule__nobody_login_shell_result'
+      changed_when: false
+      failed_when: false
+
+    - name: 'Assert that an unprivileged login shell neither sees nor uses the credentials'
+      ansible.builtin.assert:
+        that:
+          - '"alias reboot=" in __molecule__nobody_login_shell_result["stdout"]'
+          - '"linuxfabrik-downtime-api" not in __molecule__nobody_login_shell_result["stdout"]'
+          - '"Permission denied" in __molecule__nobody_login_shell_result["stderr"]'
+        fail_msg: '{{ __molecule__nobody_login_shell_result | to_json }}'

roles/borg_local/tasks/main.yml

@@ -60,7 +60,7 @@
       dest: '/usr/local/bin/borg-backup'
       owner: 'root'
       group: 'root'
-      mode: 0o755
+      mode: 0o700 # carries the credentials of the Icinga API user
 
   tags:
     - 'borg_local'

roles/borg_local/templates/usr/local/bin/borg-backup.j2

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # {{ ansible_managed }}
-# 2025020301
+# 2026092201
 
 # no `set -e` because we want to finish writing the log file even if a borg command fails
 function exit_with_first_failed_retc() {
@@ -17,14 +17,16 @@ if [[ $? -eq 0 ]]; then
     # needed for Icinga to set clamd@scan service downtime (max. 90 minutes downtime)
     START_TIME=$(date +%s)
     END_TIME=$(( START_TIME + 5400 ))
-    curl --connect-timeout 5 --insecure --silent --user '{{ borg_local__icinga2_api_user_login["username"] }}:{{ borg_local__icinga2_api_user_login["password"] }}' --header 'Accept: application/json' --request POST '{{ borg_local__icinga2_api_url }}/v1/actions/schedule-downtime' --data-binary @- 1> /dev/null << EOF
+    # The credentials reach curl as a header read from a pipe that the printf builtin fills, so
+    # they never show up in the process list the way `curl --user` would put them there.
+    curl --connect-timeout 5 --insecure --silent --header @<(printf 'Authorization: Basic %s\n' '{{ (borg_local__icinga2_api_user_login["username"] ~ ":" ~ borg_local__icinga2_api_user_login["password"]) | b64encode }}') --header 'Accept: application/json' --request POST '{{ borg_local__icinga2_api_url }}/v1/actions/schedule-downtime' --data-binary @- 1> /dev/null << EOF
     {
         "type": "Service",
-        "filter": "match(\"{{ borg_local__icinga2_hostname }}\", host.name) && "match(\"*Systemd Unit - clamd@scan*\", service.name)",
+        "filter": "match(\"{{ borg_local__icinga2_hostname }}\", host.name) && match(\"*Systemd Unit - clamd@scan*\", service.name)",
         "start_time": "$START_TIME",
         "end_time": "$END_TIME",
         "author": "{{ borg_local__icinga2_hostname }}",
-        "comment": "Disabling clamd@scan during borg backup.",
+        "comment": "Disabling clamd@scan during borg backup."
     }
 EOF
 {% endif %}
@@ -90,7 +92,7 @@ if [[ $start_clamd -eq 1 ]]; then
 
     # remove Icinga downtime
     DATA="{ \"type\": \"Service\", \"filter\": \"match(\\\"*{{ borg_local__icinga2_hostname }}*\\\", host.name) && match(\\\"*Systemd Unit - clamd@scan*\\\", service.name)\" }"
-    curl --insecure --silent --user {{ borg_local__icinga2_api_user_login["username"] }}:{{ borg_local__icinga2_api_user_login["password"] }} --header 'Accept: application/json' --request POST '{{ borg_local__icinga2_api_url }}/v1/actions/remove-downtime' --data "$DATA" 1> /dev/null
+    curl --insecure --silent --header @<(printf 'Authorization: Basic %s\n' '{{ (borg_local__icinga2_api_user_login["username"] ~ ":" ~ borg_local__icinga2_api_user_login["password"]) | b64encode }}') --header 'Accept: application/json' --request POST '{{ borg_local__icinga2_api_url }}/v1/actions/remove-downtime' --data "$DATA" 1> /dev/null
 {% endif %}
 fi
 

roles/schedule_reboot/tasks/main.yml

@@ -20,7 +20,7 @@
       dest: '/usr/local/libexec/do-reboot'
       owner: 'root'
       group: 'root'
-      mode: 0o744
+      mode: 0o700 # carries the credentials of the Icinga API user and the Rocket.Chat webhook
 
   - name: 'Deploy /usr/local/sbin/schedule-reboot'
     ansible.builtin.template:

roles/schedule_reboot/templates/usr/local/libexec/do-reboot.j2

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # {{ ansible_managed }}
-# 2026082501
+# 2026092201
 
 # The single reboot actor. Run by schedule-reboot.service (a timer at the reboot
 # window, ordered After= the update lanes) and by `schedule-reboot --now`. Reboots
@@ -26,13 +26,15 @@ send_msg () {
         printf '\n%s\n' "$body"
     } | sendmail -oi -t -f "$MAIL_FROM"
 {% if schedule_reboot__rocketchat_url is defined and schedule_reboot__rocketchat_url | length %}
+    # The webhook URL carries its token, so it reaches curl through a pipe rather than the command
+    # line, which every local user can read in the process list.
     /usr/bin/curl --silent --output /dev/null --data-urlencode \
         "text=${subject}
 
         ${body}
         {{ schedule_reboot__rocketchat_msg_suffix }}" \
         --data-urlencode "parse_mode=HTML" --data-urlencode "disable_web_page_preview=true" \
-        "{{ schedule_reboot__rocketchat_url }}"
+        --config <(printf 'url = "%s"\n' '{{ schedule_reboot__rocketchat_url }}')
 {% endif %}
 }
 
@@ -67,7 +69,9 @@ send_msg "$SUBJECT_PREFIX - Automatic reboot due to ${reasons%, }" "$MSGBODY"
 # needed for Icinga to set downtime (max. 5 minutes downtime)
 START_TIME=$(date +%s)
 END_TIME=$(( START_TIME + 300 ))
-curl --connect-timeout 5 --insecure --silent --user '{{ schedule_reboot__icinga2_api_user_login["username"] }}:{{ schedule_reboot__icinga2_api_user_login["password"] }}' --header 'Accept: application/json' --request POST '{{ schedule_reboot__icinga2_api_url }}/v1/actions/schedule-downtime' --data-binary @- 1> /dev/null << EOF
+# The credentials reach curl as a header read from a pipe that the printf builtin fills, so they
+# never show up in the process list the way `curl --user` would put them there.
+curl --connect-timeout 5 --insecure --silent --header @<(printf 'Authorization: Basic %s\n' '{{ (schedule_reboot__icinga2_api_user_login["username"] ~ ":" ~ schedule_reboot__icinga2_api_user_login["password"]) | b64encode }}') --header 'Accept: application/json' --request POST '{{ schedule_reboot__icinga2_api_url }}/v1/actions/schedule-downtime' --data-binary @- 1> /dev/null << EOF
 {
     "type": "Host",
     "filter": "match(\"{{ schedule_reboot__icinga2_hostname }}\", host.name)",

roles/tools/README.md

@@ -63,7 +63,7 @@ Any [LFOps playbook](https://github.com/Linuxfabrik/lfops/blob/main/playbooks/RE
 
 `tools__icinga2_api_user_login`
 
-* The Icinga2 API User to set the downtime for the corresponding host and all its services in the `reboot` alias. Defaults to the downtime API user the `icinga2_master` role creates. When neither is set, the alias sets no downtime.
+* The Icinga2 API User to set the downtime for the corresponding host and all its services in the `reboot` alias. The role writes it into `/usr/local/sbin/schedule-icinga-downtime`, which only root may read and run, and which the alias calls. Defaults to the downtime API user the `icinga2_master` role creates. When neither is set, the alias sets no downtime.
 * Type: Dictionary.
 * Default: `'{{ icinga2_master__downtime_api_user | d({}) }}'`
 

roles/tools/tasks/main.yml

@@ -58,5 +58,23 @@
       - 'history.sh'
       - 'prompt.sh'
 
+  - name: 'Deploy /usr/local/sbin/schedule-icinga-downtime'
+    ansible.builtin.template:
+      backup: true
+      src: 'usr/local/sbin/schedule-icinga-downtime.j2'
+      dest: '/usr/local/sbin/schedule-icinga-downtime'
+      owner: 'root'
+      group: 'root'
+      mode: 0o700
+    when:
+      - 'tools__icinga2_api_user_login is defined and tools__icinga2_api_user_login | length > 0'
+
+  - name: 'rm -f /usr/local/sbin/schedule-icinga-downtime'
+    ansible.builtin.file:
+      path: '/usr/local/sbin/schedule-icinga-downtime'
+      state: 'absent'
+    when:
+      - 'not (tools__icinga2_api_user_login is defined and tools__icinga2_api_user_login | length > 0)'
+
   tags:
     - 'tools'

roles/tools/templates/etc/profile.d/alias.sh.j2

@@ -1,5 +1,5 @@
 # {{ ansible_managed }}
-# 2023062801
+# 2026092201
 
 alias whatismyip="curl ipecho.net/plain"
 alias dmesg='dmesg --level=emerg,alert,crit,err --reltime'
@@ -18,34 +18,7 @@ function lf-confirm() {
 }
 
 {% if tools__icinga2_api_user_login is defined and tools__icinga2_api_user_login | length %}
-function schedule-icinga-downtime() {
-    if [ -z "$1" ]; then
-        echo 'arg 1 required! (downtime duration in s).'
-        exit 1
-    fi
-
-    if [ -z "$2" ]; then
-        comment="Downtime set per schedule-icinga-downtime."
-    else
-        comment="$2"
-    fi
-
-    START_TIME=$(date +%s)
-    END_TIME=$(( START_TIME + $1 ))
-    curl --connect-timeout 5 --insecure --silent --user '{{ tools__icinga2_api_user_login["username"] }}:{{ tools__icinga2_api_user_login["password"] }}' --header 'Accept: application/json' --request POST '{{ tools__icinga2_api_url }}/v1/actions/schedule-downtime' --data-binary @- 1> /dev/null << EOF
-    {
-        "type": "Host",
-        "filter": "match(\"{{ tools__icinga2_hostname }}\", host.name)",
-        "start_time": "$START_TIME",
-        "end_time": "$END_TIME",
-        "author": "{{ tools__icinga2_hostname }}",
-        "comment": "$comment",
-        "all_services": true
-    }
-EOF
-
-}
-
+# /usr/local/sbin/schedule-icinga-downtime is readable and executable by root only.
 alias reboot="lf-confirm && schedule-icinga-downtime 300 'Manual reboot' && /usr/sbin/reboot"
 {% else %}
 alias reboot="lf-confirm && /usr/sbin/reboot"

roles/tools/templates/usr/local/sbin/schedule-icinga-downtime.j2

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# {{ ansible_managed }}
+# 2026092201
+
+# Sets an Icinga downtime for this host and all its services. Readable by root only, since it
+# carries the credentials of the Icinga API user; /etc/profile.d/alias.sh, which every login
+# shell reads, only calls it.
+
+set -o nounset
+
+if [ -z "${1:-}" ]; then
+    echo 'Usage: schedule-icinga-downtime SECONDS [COMMENT]' >&2
+    exit 1
+fi
+comment="${2:-Downtime set per schedule-icinga-downtime.}"
+
+START_TIME=$(date +%s)
+END_TIME=$(( START_TIME + $1 ))
+
+# The credentials reach curl as a header read from a pipe that the printf builtin fills, so they
+# never show up in the process list the way `curl --user` would put them there.
+curl \
+    --connect-timeout 5 \
+    --insecure \
+    --silent \
+    --header @<(printf 'Authorization: Basic %s\n' '{{ (tools__icinga2_api_user_login["username"] ~ ":" ~ tools__icinga2_api_user_login["password"]) | b64encode }}') \
+    --header 'Accept: application/json' \
+    --request POST '{{ tools__icinga2_api_url }}/v1/actions/schedule-downtime' \
+    --data-binary @- 1> /dev/null << EOT
+{
+    "type": "Host",
+    "filter": "match(\"{{ tools__icinga2_hostname }}\", host.name)",
+    "start_time": "$START_TIME",
+    "end_time": "$END_TIME",
+    "author": "{{ tools__icinga2_hostname }}",
+    "comment": "$comment",
+    "all_services": true
+}
+EOT