diff --git a/testdata/README.md b/testdata/README.md new file mode 100644 index 0000000..8e3ba03 --- /dev/null +++ b/testdata/README.md @@ -0,0 +1,87 @@ +# Prometheus Alertmanager / Icinga Bridge + +Test stack of components required to bring Prometheus Alertmanager alerts into +Icinga2. This particular setup uses Prometheus as alert source + +## Components + +The alerting landscape consists of user facing- and support services. +The former are accessible via web browser locally. + +### Alertmanager Prometheus Bridge + +Binding glue between Prometheus Alertmanager and Icinga2. + +### Icinga2 + +Monitoring system; actively schedules host/service checks and accepts results +via HTTP API from Alertmanager Prometheus Bridge. + +### IcingaDB + +Icinga2 worker queue to process monitoring events. + +### Postgres + +RDBMS used by IcingaDB and Icingaweb2 as permanent storage. + +### Valkey + +Key-value store used by IcingaDB as intermediate storage. + +### Prometheus + +Metrics collector and storage. The web interface is +accessible at [http://localhost:9090](http://localhost:9090) + +### Prometheus Alertmanager + +Prometheus Alertmanager is receiving alerts either from Prometheus or locally +via helper script (`./fire.sh`). The web interface is accessible +at [http://localhost:9093](http://localhost:9093) + +### Icingaweb2 + +Web interface for Icinga2 accessible at +[http://localhost:8080](http://localhost:8080), with credentials +for the administrator account being `icingaadmin`/`icingaadmin`. + +## Alerts + +The alerts are for demonstration purposes only and do not necessarily +represent real monitoring scenarios. + +### PrometheusDiscoveryHighRequestRate + +Visit the discovery endpoints in the Prometheus web interface to +trigger this synthetic alert; refresh the page to resolve the alert: + +* http://localhost:9090/alertmanager-discovery + * http://localhost:8080/icingadb/service?name=PrometheusDiscoveryHighRequestRate_a274e53dc7c35811&host.name=prometheus.localhost +* http://localhost:9090/service-discovery + * http://localhost:8080/icingadb/service?name=PrometheusDiscoveryHighRequestRate_a5b0cfc1ec22ef1c&host.name=prometheus.localhost + +### Watchdog + +An alert to report on the state of the alerting pipeline from back to front. +It constantly fires in Prometheus to act as heartbeat. Stopping the Prometheus +container or otherwise interrupting the alert dispatching will eventually +result in the service state to reflect this: + +* http://localhost:8080/dashboard#!/icingadb/service?name=Watchdog_664753abe3262835&host.name=bridge.localhost + +### AlertmanagerUp + +Example alert to showcase mapping alerts to host objects in Icinga2. +Visit the Prometheus Alertmanager status page to trigger this +synthetic alert; refresh the page to resolve the alert: + +* http://localhost:9093/#/status + * http://localhost:8080/icingadb/host?name=alertmanager.localhost + +### External Connectivity (heartbeat) + +Not an alert in the Prometheus configuration but a heartbeat signal created +and managed exclusively by the Alertmanager Prometheus Bridge: + +* http://localhost:8080/icingadb/service?name=heartbeat&host.name=bridge.localhost diff --git a/testdata/bridge/config.env b/testdata/bridge/config.env index f2a5527..ba14c6a 100644 --- a/testdata/bridge/config.env +++ b/testdata/bridge/config.env @@ -7,3 +7,4 @@ ALERTMANAGER_ICINGA_BRIDGE_ICINGA_PASSWORD=alertmanager-icinga-bridge ALERTMANAGER_ICINGA_BRIDGE_HEARTBEAT_SERVICE=heartbeat ALERTMANAGER_ICINGA_BRIDGE_LISTEN_ADDR=0.0.0.0:8888 ALERTMANAGER_ICINGA_BRIDGE_BEARER_TOKEN=prometheus-alertmanager-bearer-token +ALERTMANAGER_ICINGA_BRIDGE_SERVICE_TEMPLATE=external-service diff --git a/testdata/docker-compose.yml b/testdata/docker-compose.yml index fe0abd3..823165e 100644 --- a/testdata/docker-compose.yml +++ b/testdata/docker-compose.yml @@ -47,7 +47,9 @@ services: - bridge bridge: restart: unless-stopped - image: ghcr.io/netways/alertmanager-icinga-bridge:1.0.3 + image: ghcr.io/netways/alertmanager-icinga-bridge:unstable + build: + context: .. env_file: - ./bridge/config.env depends_on: diff --git a/testdata/icinga/objects/api-users.conf b/testdata/icinga/objects/api-users.conf index 71aa019..c71487c 100644 --- a/testdata/icinga/objects/api-users.conf +++ b/testdata/icinga/objects/api-users.conf @@ -1,8 +1,23 @@ +object ApiUser "icingaweb" { + + password = "icingaweb" + + permissions = [ + "actions/*", + "objects/query/*", + "objects/modify/*", + "status/query", + ] +} + object ApiUser "alertmanager-icinga-bridge" { password = "alertmanager-icinga-bridge" permissions = [ + { + permission = "filter-expression" + }, { permission = "objects/query/*" filter = {{ "external-hosts" in host.groups }} diff --git a/testdata/icinga/objects/applies.conf b/testdata/icinga/objects/applies.conf index a44922d..11f3476 100644 --- a/testdata/icinga/objects/applies.conf +++ b/testdata/icinga/objects/applies.conf @@ -1,11 +1,8 @@ apply Service "heartbeat" to Host { - import "generic-service" + import "external-service" display_name = "External connectivity" - check_interval = 2m - check_command = "dummy" - vars.dummy_state = 2 vars.dummy_text = {{ var service = get_service(macro("$host.name$"), macro("$service.name$")) diff --git a/testdata/icinga/objects/service-groups.conf b/testdata/icinga/objects/service-groups.conf new file mode 100644 index 0000000..bcbe1cf --- /dev/null +++ b/testdata/icinga/objects/service-groups.conf @@ -0,0 +1,5 @@ +object ServiceGroup "external-services" { + display_name = "Externally managed services" + + assign where "external-service" in service.templates +} diff --git a/testdata/icinga/objects/templates.conf b/testdata/icinga/objects/templates.conf index b0ccaa3..8022843 100644 --- a/testdata/icinga/objects/templates.conf +++ b/testdata/icinga/objects/templates.conf @@ -1,20 +1,18 @@ -template Host "generic-host" { - max_check_attempts = 3 - check_interval = 1m - retry_interval = 30s +template Host "external-host" { + enable_perfdata = false + check_command = "dummy" + check_interval = 3m + + vars.dummy_state = 0 + vars.dummy_text = "Host is operational" - check_command = "hostalive" + vars.managed_by = "bridge.localhost" } -template Host "external-host" { - enable_passive_checks = false +template Service "external-service" { enable_perfdata = false - check_command = "dummy" -} + check_interval = 3m -template Service "generic-service" { - max_check_attempts = 5 - check_interval = 1m - retry_interval = 30s + vars.managed_by = "bridge.localhost" } diff --git a/testdata/icingaweb/config.env b/testdata/icingaweb/config.env index f64e779..c833b69 100644 --- a/testdata/icingaweb/config.env +++ b/testdata/icingaweb/config.env @@ -25,8 +25,8 @@ icingaweb.modules.icingadb.redis.redis1.password=icingadb icingaweb.modules.icingadb.commandtransports.icinga2.transport=api icingaweb.modules.icingadb.commandtransports.icinga2.host=icinga -icingaweb.modules.icingadb.commandtransports.icinga2.username=icingadb -icingaweb.modules.icingadb.commandtransports.icinga2.password=icingadb +icingaweb.modules.icingadb.commandtransports.icinga2.username=icingaweb +icingaweb.modules.icingadb.commandtransports.icinga2.password=icingaweb icingaweb.resources.icingaweb_compose.type=db icingaweb.resources.icingaweb_compose.db=pgsql diff --git a/testdata/prometheus/alerts/prometheus.yml b/testdata/prometheus/alerts/prometheus.yml index 742872a..3660792 100644 --- a/testdata/prometheus/alerts/prometheus.yml +++ b/testdata/prometheus/alerts/prometheus.yml @@ -19,4 +19,42 @@ groups: description: >- Requests against the discovery endpoint {{ $labels.handler }} are higher than the expected average. - runbook_url: https://runbooks.example.com/prometheus/{{ $labels.alertname }} + runbook_url: https://runbooks.example.com/prometheus/PrometheusDiscoveryHighRequestRate + - alert: Watchdog + expr: vector(1) + for: 0m + labels: + # alertmanager-icinga-bridge will treat this as an unknown severity, + # causing the Icinga service to flip into UNKNOWN once the heartbeat + # stops + severity: none + # alertmanager re-sends alerts every 2 minutes and waits another minute + # to group alerts in batches; use that value as heartbeat baseline + heartbeat: "3m" + annotations: + message: DeadMansSwitch to ensure the alerting pipeline is fully operational. + description: >- + This is an alert meant to ensure that the entire alerting pipeline is functional. + This alert is always firing, therefore it should always be firing in Alertmanager + and always fire against a receiver. + runbook_url: https://runbooks.example.com/prometheus/Watchdog + - name: Alertmanager + rules: + - alert: AlertmanagerUp + # this expression does not represent an actual availability check + # it simply serves as an easy trigger for alerts: + # visit the status endpoint in the Alertmanager Web UI to trigger the alert + # (/#/status); refresh the page to cause the alert to resolve + expr: >- + alertmanager_http_request_duration_seconds_count{handler="/api/v2/status"} % 2 == 1 + for: 0m + labels: + severity: critical + icinga_use_host: alertmanager.localhost + icinga_use_host_check: "true" + annotations: + message: Status endpoint {{ $labels.handler }} is in a degraded state + description: >- + Health information from the API endpoint {{ $labels.handler }} suggest + the application is not working properly. + runbook_url: https://runbooks.example.com/alertmanager/AlertmanagerUp diff --git a/testdata/prometheus/prometheus.yml b/testdata/prometheus/prometheus.yml index 6eb4a1b..48e826d 100644 --- a/testdata/prometheus/prometheus.yml +++ b/testdata/prometheus/prometheus.yml @@ -12,6 +12,10 @@ scrape_configs: static_configs: - targets: - 127.0.0.1:9090 + - job_name: Alertmanager + static_configs: + - targets: + - alertmanager:9093 alerting: alertmanagers: