document test stack

testdata/README.md

@@ -0,0 +1,87 @@
+# Prometheus Alertmanager / Icinga Bridge
+
+Test stack of components required to bring Prometheus Alertmanager alerts into
+Icinga2. This particular setup uses Prometheus as alert source
+
+## Components
+
+The alerting landscape consists of user facing- and support services.
+The former are accessible via web browser locally.
+
+### Alertmanager Prometheus Bridge
+
+Binding glue between Prometheus Alertmanager and Icinga2.
+
+### Icinga2
+
+Monitoring system; actively schedules host/service checks and accepts results
+via HTTP API from Alertmanager Prometheus Bridge.
+
+### IcingaDB
+
+Icinga2 worker queue to process monitoring events.
+
+### Postgres
+
+RDBMS used by IcingaDB and Icingaweb2 as permanent storage.
+
+### Valkey
+
+Key-value store used by IcingaDB as intermediate storage.
+
+### Prometheus
+
+Metrics collector and storage. The web interface is
+accessible at [http://localhost:9090](http://localhost:9090)
+
+### Prometheus Alertmanager
+
+Prometheus Alertmanager is receiving alerts either from Prometheus or locally
+via helper script (`./fire.sh`). The web interface is accessible
+at [http://localhost:9093](http://localhost:9093)
+
+### Icingaweb2
+
+Web interface for Icinga2 accessible at
+[http://localhost:8080](http://localhost:8080), with credentials
+for the administrator account being `icingaadmin`/`icingaadmin`.
+
+## Alerts
+
+The alerts are for demonstration purposes only and do not necessarily
+represent real monitoring scenarios.
+
+### PrometheusDiscoveryHighRequestRate
+
+Visit the discovery endpoints in the Prometheus web interface to
+trigger this synthetic alert; refresh the page to resolve the alert:
+
+* http://localhost:9090/alertmanager-discovery
+    * http://localhost:8080/icingadb/service?name=PrometheusDiscoveryHighRequestRate_a274e53dc7c35811&host.name=prometheus.localhost
+* http://localhost:9090/service-discovery
+    * http://localhost:8080/icingadb/service?name=PrometheusDiscoveryHighRequestRate_a5b0cfc1ec22ef1c&host.name=prometheus.localhost
+
+### Watchdog
+
+An alert to report on the state of the alerting pipeline from back to front.
+It constantly fires in Prometheus to act as heartbeat. Stopping the Prometheus
+container or otherwise interrupting the alert dispatching will eventually
+result in the service state to reflect this:
+
+* http://localhost:8080/dashboard#!/icingadb/service?name=Watchdog_664753abe3262835&host.name=bridge.localhost
+
+### AlertmanagerUp
+
+Example alert to showcase mapping alerts to host objects in Icinga2.
+Visit the Prometheus Alertmanager status page to trigger this
+synthetic alert; refresh the page to resolve the alert:
+
+* http://localhost:9093/#/status
+    * http://localhost:8080/icingadb/host?name=alertmanager.localhost
+
+### External Connectivity (heartbeat)
+
+Not an alert in the Prometheus configuration but a heartbeat signal created
+and managed exclusively by the Alertmanager Prometheus Bridge:
+
+* http://localhost:8080/icingadb/service?name=heartbeat&host.name=bridge.localhost

testdata/bridge/config.env

@@ -7,3 +7,4 @@ ALERTMANAGER_ICINGA_BRIDGE_ICINGA_PASSWORD=alertmanager-icinga-bridge
 ALERTMANAGER_ICINGA_BRIDGE_HEARTBEAT_SERVICE=heartbeat
 ALERTMANAGER_ICINGA_BRIDGE_LISTEN_ADDR=0.0.0.0:8888
 ALERTMANAGER_ICINGA_BRIDGE_BEARER_TOKEN=prometheus-alertmanager-bearer-token
+ALERTMANAGER_ICINGA_BRIDGE_SERVICE_TEMPLATE=external-service

testdata/docker-compose.yml

@@ -47,7 +47,9 @@ services:
       - bridge
   bridge:
     restart: unless-stopped
-    image: ghcr.io/netways/alertmanager-icinga-bridge:1.0.3
+    image: ghcr.io/netways/alertmanager-icinga-bridge:unstable
+    build:
+      context: ..
     env_file:
       - ./bridge/config.env
     depends_on:

testdata/icinga/objects/api-users.conf

@@ -1,8 +1,23 @@
+object ApiUser "icingaweb"  {
+
+  password = "icingaweb"
+
+  permissions = [
+    "actions/*",
+    "objects/query/*",
+    "objects/modify/*",
+    "status/query",
+  ]
+}
+
 object ApiUser "alertmanager-icinga-bridge"  {
 
   password = "alertmanager-icinga-bridge"
 
   permissions = [
+    {
+      permission = "filter-expression"
+    },
     {
       permission = "objects/query/*"
       filter = {{ "external-hosts" in host.groups }}

testdata/icinga/objects/applies.conf

@@ -1,11 +1,8 @@
 apply Service "heartbeat" to Host {
-  import "generic-service"
+  import "external-service"
 
   display_name = "External connectivity"
 
-  check_interval = 2m
-  check_command = "dummy"
-
   vars.dummy_state = 2
   vars.dummy_text = {{
     var service = get_service(macro("$host.name$"), macro("$service.name$"))

testdata/icinga/objects/service-groups.conf

@@ -0,0 +1,5 @@
+object ServiceGroup "external-services" {
+  display_name = "Externally managed services"
+
+  assign where "external-service" in service.templates
+}

testdata/icinga/objects/templates.conf

@@ -1,20 +1,18 @@
-template Host "generic-host" {
-  max_check_attempts = 3
-  check_interval = 1m
-  retry_interval = 30s
+template Host "external-host" {
+  enable_perfdata = false
+  check_command = "dummy"
+  check_interval = 3m
+
+  vars.dummy_state = 0
+  vars.dummy_text = "Host is operational"
 
-  check_command = "hostalive"
+  vars.managed_by = "bridge.localhost"
 }
 
-template Host "external-host" {
-  enable_passive_checks = false
+template Service "external-service" {
   enable_perfdata = false
-
   check_command = "dummy"
-}
+  check_interval = 3m
 
-template Service "generic-service" {
-  max_check_attempts = 5
-  check_interval = 1m
-  retry_interval = 30s
+  vars.managed_by = "bridge.localhost"
 }

testdata/icingaweb/config.env

@@ -25,8 +25,8 @@ icingaweb.modules.icingadb.redis.redis1.password=icingadb
 
 icingaweb.modules.icingadb.commandtransports.icinga2.transport=api
 icingaweb.modules.icingadb.commandtransports.icinga2.host=icinga
-icingaweb.modules.icingadb.commandtransports.icinga2.username=icingadb
-icingaweb.modules.icingadb.commandtransports.icinga2.password=icingadb
+icingaweb.modules.icingadb.commandtransports.icinga2.username=icingaweb
+icingaweb.modules.icingadb.commandtransports.icinga2.password=icingaweb
 
 icingaweb.resources.icingaweb_compose.type=db
 icingaweb.resources.icingaweb_compose.db=pgsql

testdata/prometheus/alerts/prometheus.yml

@@ -19,4 +19,42 @@ groups:
           description: >-
             Requests against the discovery endpoint {{ $labels.handler }}
             are higher than the expected average.
-          runbook_url: https://runbooks.example.com/prometheus/{{ $labels.alertname }}
+          runbook_url: https://runbooks.example.com/prometheus/PrometheusDiscoveryHighRequestRate
+      - alert: Watchdog
+        expr: vector(1)
+        for: 0m
+        labels:
+          # alertmanager-icinga-bridge will treat this as an unknown severity,
+          # causing the Icinga service to flip into UNKNOWN once the heartbeat
+          # stops
+          severity: none
+          # alertmanager re-sends alerts every 2 minutes and waits another minute
+          # to group alerts in batches; use that value as heartbeat baseline
+          heartbeat: "3m"
+        annotations:
+          message: DeadMansSwitch to ensure the alerting pipeline is fully operational.
+          description: >-
+            This is an alert meant to ensure that the entire alerting pipeline is functional.
+            This alert is always firing, therefore it should always be firing in Alertmanager
+            and always fire against a receiver.
+          runbook_url: https://runbooks.example.com/prometheus/Watchdog
+  - name: Alertmanager
+    rules:
+      - alert: AlertmanagerUp
+        # this expression does not represent an actual availability check
+        # it simply serves as an easy trigger for alerts:
+        # visit the status endpoint in the Alertmanager Web UI to trigger the alert
+        # (/#/status); refresh the page to cause the alert to resolve
+        expr: >-
+          alertmanager_http_request_duration_seconds_count{handler="/api/v2/status"} % 2 == 1
+        for: 0m
+        labels:
+          severity: critical
+          icinga_use_host: alertmanager.localhost
+          icinga_use_host_check: "true"
+        annotations:
+          message: Status endpoint {{ $labels.handler }} is in a degraded state
+          description: >-
+            Health information from the API endpoint {{ $labels.handler }} suggest
+            the application is not working properly.
+          runbook_url: https://runbooks.example.com/alertmanager/AlertmanagerUp

testdata/prometheus/prometheus.yml

@@ -12,6 +12,10 @@ scrape_configs:
     static_configs:
       - targets:
           - 127.0.0.1:9090
+  - job_name: Alertmanager
+    static_configs:
+      - targets:
+          - alertmanager:9093
 
 alerting:
   alertmanagers: