Add option to set host check results

README.md

@@ -89,6 +89,7 @@ Flags:
 --max-check-attempts=1                                    The maximum number of checks which are executed before changing to a hard state ($ALERTMANAGER_ICINGA_BRIDGE_SERVICE_MAX_CHECK_ATTEMPTS)
 --templates=generic-service,...                           Create Icinga services with the given template (can be repeated) ($ALERTMANAGER_ICINGA_BRIDGE_SERVICE_TEMPLATE)
 --icinga-host-object-labels=icinga_use_host,...           List of Labels depicting the Icinga Host object to associate the alert service with (uses --icinga-hostname if no label is found on alerts) ($ALERTMANAGER_BRIDGE_ICINGA_HOST_OBJECT_LABELS)
+--icinga-host-check-labels=icinga_use_host_check,...      List of Labels to be used to associate the incoming alert with the Icinga Host object instead of a Service object ($ALERTMANAGER_BRIDGE_ICINGA_HOST_CHECK_LABELS)
 --icinga-host-zone-labels=icinga_use_zone,...             List of Labels to be used to set the zone attribute of the Icinga service object ($ALERTMANAGER_BRIDGE_ICINGA_HOST_ZONE_LABELS)
 --icinga-host-template-labels=icinga_use_template,...     List of Labels to be used to set the included template for the Icinga service object (in addition to --templates) ($ALERTMANAGER_BRIDGE_ICINGA_HOST_TEMPLATE_LABELS)
 --plugin-output-annotations=message,...                   List of Annotation names to be used to set the plugin output for the Icinga service ($ALERTMANAGER_ICINGA_BRIDGE_PLUGINOUTPUT_ANNOTATIONS)
@@ -299,10 +300,12 @@ This can be overridden by the following labels (unless configured otherwise):
 | Alert      | Icinga      |
 | ---------- | ----------- |
 | Label: `icinga_use_host: MyHost` | If present, use given host for the new service. The host must exist beforehand |
+| Label: `icinga_use_host_check: "true"` | If this and `icinga_use_host` are present, set the host check result for the given host |
 | Label: `icinga_use_zone: MyZone` | If present, use given zone for the new service The zone must exist beforehand |
 | Label: `icinga_use_template: MyTemplate` | If present, use given template for the new service. The template must exist beforehand |
 
-Note that this requires the Alertmanager-Icinga-Bridge user to have the necessary permissions on the host.
+Note that, `icinga_use_host` requires the Alertmanager-Icinga-Bridge user to have the necessary permissions on the given host.
+The `icinga_use_host_check` will map "firing" alerts to the Icinga Host state "down" and "resolved" to "up".
 
 ## Heartbeat Services
 

internal/api/listener.go

@@ -48,6 +48,7 @@ type Listener struct {
 	notesURLGetter       AlertFieldGetter
 	pluginOutputGetter   AlertFieldGetter
 	hostNameGetter       AlertFieldGetter
+	hostCheckGetter      AlertFieldGetter
 	zoneGetter           AlertFieldGetter
 	templatesGetter      AlertFieldGetter
 }
@@ -62,6 +63,7 @@ func NewListener(config *config.Config, logger *slog.Logger, icingaClient *icing
 		notesTextGetter:      AnnotationsGetter(config.NotesTextAnnotations),
 		notesURLGetter:       AnnotationsGetter(config.NotesURLAnnotations),
 		hostNameGetter:       LabelsGetter(config.IcingaHostObjectLabels),
+		hostCheckGetter:      LabelsGetter(config.IcingaHostCheckLabels),
 		zoneGetter:           LabelsGetter(config.IcingaHostZoneLabels),
 		templatesGetter:      LabelsGetter(config.IcingaHostTemplateLabels),
 	}
@@ -206,11 +208,11 @@ func (l *Listener) handleIncomingAlert(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	errManage := l.manageIcingaService(r.Context(), payload)
+	errManage := l.manageIcingaResults(r.Context(), payload)
 
 	if errManage != nil {
-		l.logger.Error("Could not manage Icinga service for incoming alert", "component", "listener", "error", errManage.Error())
-		http.Error(w, "Could not manage Icinga service for incoming alert", http.StatusInternalServerError)
+		l.logger.Error("Could not manage Icinga results for incoming alert", "component", "listener", "error", errManage.Error())
+		http.Error(w, "Could not manage Icinga results for incoming alert", http.StatusInternalServerError)
 
 		return
 	}
@@ -221,14 +223,15 @@ func (l *Listener) handleIncomingAlert(w http.ResponseWriter, r *http.Request) {
 	l.logger.Debug("Handled incoming alert", "component", "listener")
 }
 
-// manageIcingaService talks to the Icinga API to manage the service for the incoming alert
-func (l *Listener) manageIcingaService(ctx context.Context, payload WebhookPayload) error {
-	l.logger.Debug("Managing Icinga service", "component", "listener")
+// manageIcingaResults talks to the Icinga API to manage the results for the incoming alert
+func (l *Listener) manageIcingaResults(ctx context.Context, payload WebhookPayload) error {
+	l.logger.Debug("Managing Icinga results", "component", "listener")
 
 	ctxIcinga, cancel := context.WithTimeout(ctx, 20*time.Second)
 	defer cancel()
 
-	_, errHost := l.icingaClient.GetHost(ctxIcinga, l.config.IcingaHostname)
+	// We always check for the configured alertmanager-bridge host, since we use it as backup
+	bridgeHost, errHost := l.icingaClient.GetHost(ctxIcinga, l.config.IcingaHostname)
 
 	if errHost != nil {
 		return errHost
@@ -243,55 +246,79 @@ func (l *Listener) manageIcingaService(ctx context.Context, payload WebhookPaylo
 			l.logger.Warn("alert does not have label 'alertname'", "alert", alert)
 		}
 
-		serviceName := generateServiceName(l.config.ID, alert, l.fingerprintExcludes)
-
-		if !l.serviceNameValidator.MatchString(serviceName) {
-			return fmt.Errorf("service name '%v' does not match Icinga constraints", serviceName)
+		// Do we use a host check result for this alert or not
+		useHostCheck := false
+		// The default host is the alertmanager-bridge host as default
+		hostName := bridgeHost.Name
+		// When a different hostname is used then we check for the icinga_use_host_check label
+		if value := l.hostNameGetter.GetField(&alert, 0); value != "" {
+			hostName = value
+
+			useHostCheckLabelValue := l.hostCheckGetter.GetField(&alert, 0)
+			// This alert will be attached to a host check result
+			useHostCheck, _ = strconv.ParseBool(useHostCheckLabelValue)
 		}
 
-		if l.config.DisplayNameAsServiceName {
-			displayName = serviceName
+		// Get the exitcode of the alert
+		exitCode := severityToExitCode(alert.Status, alert.Labels["severity"], l.severityLevels, useHostCheck)
+
+		// Get the Plugin Output from the first Annotation we find that has some data
+		pluginOutput := l.pluginOutputGetter.GetField(&alert, exitCode)
+
+		icingaAction := icinga2.Action{
+			ExitStatus:   exitCode,
+			PluginOutput: pluginOutput,
 		}
 
-		exitCode := severityToExitCode(alert.Status, alert.Labels["severity"], l.severityLevels)
+		if useHostCheck { // nolint: nestif
+			l.logger.Debug("Using host check for process-check-result", "component", "listener", "alert", alert)
 
-		svc, errUpsert := l.updateOrCreateService(ctxIcinga, serviceName, displayName, exitCode, alert)
+			icingaAction.Filter = fmt.Sprintf("host.name==%q", hostName)
+			icingaAction.Type = "Host"
+		} else {
+			l.logger.Debug("Using service check for process-check-result", "component", "listener", "alert", alert)
 
-		if errUpsert != nil {
-			return errUpsert
-		}
+			serviceName := generateServiceName(l.config.ID, alert, l.fingerprintExcludes)
 
-		// If we got an empty service object, the service was not
-		// created, don't try to call process-check-result
-		if svc.Name == "" {
-			l.logger.Warn("Got empty service object for alert: " + displayName)
+			if !l.serviceNameValidator.MatchString(serviceName) {
+				return fmt.Errorf("service name '%v' does not match Icinga constraints", serviceName)
+			}
 
-			continue
-		}
+			if l.config.DisplayNameAsServiceName {
+				displayName = serviceName
+			}
 
-		// Get the Plugin Output from the first Annotation we find that has some data
-		pluginOutput := l.pluginOutputGetter.GetField(&alert, exitCode)
+			svc, errUpsert := l.updateOrCreateService(ctxIcinga, serviceName, displayName, exitCode, alert)
 
-		// heartbeat alerts will use exit code OK since they always fire and the active check will set it to not OK
-		if _, ok := alert.Labels["heartbeat"]; ok {
-			exitCode = icinga2.ExitStatusOK
-		}
+			if errUpsert != nil {
+				return errUpsert
+			}
 
-		action := icinga2.Action{
-			ExitStatus:   exitCode,
-			Filter:       fmt.Sprintf("host.name==\"%s\" && service.name==\"%s\"", svc.HostName, svc.Name),
-			Type:         "Service",
-			PluginOutput: pluginOutput,
+			// If we got an empty service object, the service was not
+			// created, don't try to call process-check-result
+			if svc.Name == "" {
+				l.logger.Warn("Got empty service object for alert: " + displayName)
+
+				continue
+			}
+
+			// heartbeat alerts will use exit code OK since they always fire and the active check will set it to not OK
+			if _, ok := alert.Labels["heartbeat"]; ok {
+				icingaAction.ExitStatus = icinga2.ExitStatusOK
+			}
+
+			icingaAction.Filter = fmt.Sprintf("host.name==%q && service.name==%q", svc.HostName, svc.Name)
+			icingaAction.Type = "Service"
 		}
 
-		errProcess := l.icingaClient.ProcessCheckResult(ctxIcinga, action)
+		errProcess := l.icingaClient.ProcessCheckResult(ctxIcinga, icingaAction)
 
 		if errProcess != nil {
 			return errProcess
 		}
 	}
 
-	l.logger.Debug("Managed Icinga service", "component", "listener")
+	l.logger.Debug("Managed Icinga results", "component", "listener")
 
 	return nil
 }
@@ -452,7 +479,20 @@ func mapToStableString(data map[string]string, excludes map[string]struct{}) str
 
 // severityToExitStatus computes an exit code which Icinga understands from
 // an alert's status and severity label
-func severityToExitCode(status string, severity string, severityLevels map[string]int) icinga2.ExitStatus {
+func severityToExitCode(status string, severity string, severityLevels map[string]int, isHostCheck bool) icinga2.ExitStatus {
+	// If it's a host check we only map firing to "down (1)" and resolved to "up (0)".
+	// We're reusing the Warning/OK semantic here to keep things simple for now
+	if isHostCheck {
+		switch status {
+		case alertStatusFiring:
+			return icinga2.ExitStatusWarning
+		case alertStatusResolved:
+			return icinga2.ExitStatusOK
+		default:
+			return icinga2.ExitStatusWarning
+		}
+	}
+	// Services are mapped depending on severity labels when the alert is firing
 	switch status {
 	case alertStatusFiring:
 		code, ok := severityLevels[strings.ToLower(severity)]

internal/api/listener_test.go

@@ -35,6 +35,7 @@ func testConfig(url string) *config.Config {
 		IcingaHostname:           "unittest",
 		PluginOutputAnnotations:  []string{"message"},
 		IcingaHostObjectLabels:   []string{"icinga_use_host"},
+		IcingaHostCheckLabels:    []string{"icinga_use_host_check"},
 		IcingaHostZoneLabels:     []string{"icinga_use_zone"},
 		IcingaHostTemplateLabels: []string{"icinga_use_template"},
 		NotesTextAnnotations:     []string{"description"},
@@ -211,7 +212,7 @@ func TestGenerateServiceName(t *testing.T) {
 	}
 }
 
-func TestManageIcingaService(t *testing.T) {
+func TestManageIcingaResult_WithService(t *testing.T) {
 	datasetHost := "testdata/host.json"
 	datasetService := "testdata/service.json"
 
@@ -265,14 +266,84 @@ func TestManageIcingaService(t *testing.T) {
 		ExternalURL: "http://alertmanager.internal",
 		Alerts:      []Alert{alert},
 	}
-	err := l.manageIcingaService(context.Background(), payload)
+	err := l.manageIcingaResults(context.Background(), payload)
 
 	if err != nil {
 		t.Errorf("expected no error got %v", err)
 	}
 
 	actual := buf.String()
-	expected := "Managed Icinga service"
+	expected := "Using service check for process-check-result"
+
+	if !strings.Contains(actual, expected) {
+		t.Fatalf("expected %v, got %v", expected, actual)
+	}
+}
+
+func TestManageIcingaResult_WithHost(t *testing.T) {
+	datasetHost := "testdata/host.json"
+	datasetService := "testdata/service.json"
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v1/objects/hosts/unittest":
+			w.WriteHeader(http.StatusOK)
+			w.Write(loadTestdata(datasetHost))
+		case "/v1/objects/hosts/unittest!heartbeat":
+			w.WriteHeader(http.StatusOK)
+			w.Write(loadTestdata(datasetService))
+		case "/v1/actions/process-check-result/":
+			w.WriteHeader(http.StatusOK)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+
+	defer ts.Close()
+
+	var buf bytes.Buffer
+	logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
+	config := testConfig(ts.URL)
+	icingaClient := icinga2.NewClient(config, logger)
+	l := NewListener(config, logger, icingaClient)
+
+	alert := Alert{
+		Status: "firing",
+		Labels: map[string]string{
+			"alertname":             "Down",
+			"instance":              "myhost-01",
+			"icinga_use_host":       "MyHost",
+			"icinga_use_host_check": "true",
+			"severity":              "critical",
+		},
+		Annotations: map[string]string{
+			"summary":     "MyHost is down",
+			"description": "Unittest",
+		},
+		GeneratorURL: "http://prometheus.example.com/",
+		Fingerprint:  "a1b2c3d4e5f6g7h8i9j0",
+	}
+
+	payload := WebhookPayload{
+		Status:   "firing",
+		Receiver: "webhook",
+		GroupLabels: map[string]string{
+			"alertname": "HighCPUUsage",
+		},
+		CommonLabels: map[string]string{
+			"team": "ops",
+		},
+		ExternalURL: "http://alertmanager.internal",
+		Alerts:      []Alert{alert},
+	}
+	err := l.manageIcingaResults(context.Background(), payload)
+
+	if err != nil {
+		t.Errorf("expected no error got %v", err)
+	}
+
+	actual := buf.String()
+	expected := "Using host check for process-check-result"
 
 	if !strings.Contains(actual, expected) {
 		t.Fatalf("expected %v, got %v", expected, actual)
@@ -429,22 +500,26 @@ func TestSeverityToExitCode(t *testing.T) {
 		name     string
 		status   string
 		severity string
+		isHost   bool
 		want     icinga2.ExitStatus
 	}{
-		{"firing-critical", "firing", "critical", icinga2.ExitStatusCritical},
-		{"firing-warning", "firing", "warning", icinga2.ExitStatusWarning},
-		{"firing-info", "firing", "info", icinga2.ExitStatusOK},
-		{"firing-unknown-severity", "firing", "unknown", icinga2.ExitStatusUnknown},
-		{"firing-mixed-case", "firing", "CrItIcAl", icinga2.ExitStatusCritical},
-		{"resolved-any-severity", "resolved", "critical", icinga2.ExitStatusOK},
-		{"resolved-empty-severity", "resolved", "", icinga2.ExitStatusOK},
-		{"other-status", "pending", "critical", icinga2.ExitStatusUnknown},
-		{"empty-status", "", "critical", icinga2.ExitStatusUnknown},
+		{"firing-critical", "firing", "critical", false, icinga2.ExitStatusCritical},
+		{"firing-warning", "firing", "warning", false, icinga2.ExitStatusWarning},
+		{"firing-info", "firing", "info", false, icinga2.ExitStatusOK},
+		{"firing-unknown-severity", "firing", "unknown", false, icinga2.ExitStatusUnknown},
+		{"firing-mixed-case", "firing", "CrItIcAl", false, icinga2.ExitStatusCritical},
+		{"resolved-any-severity", "resolved", "critical", false, icinga2.ExitStatusOK},
+		{"resolved-empty-severity", "resolved", "", false, icinga2.ExitStatusOK},
+		{"other-status", "pending", "critical", false, icinga2.ExitStatusUnknown},
+		{"empty-status", "", "critical", false, icinga2.ExitStatusUnknown},
+		{"host-firing-down", "firing", "warning", true, icinga2.ExitStatusWarning},
+		{"host-resolved-up", "resolved", "warning", true, icinga2.ExitStatusOK},
+		{"host-firing-empty-status", "", "warning", true, icinga2.ExitStatusWarning},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := severityToExitCode(tt.status, tt.severity, severityLevels); got != tt.want {
+			if got := severityToExitCode(tt.status, tt.severity, severityLevels, tt.isHost); got != tt.want {
 				t.Errorf("severityToExitCode(%q, %q) = %d, want %d",
 					tt.status, tt.severity, got, tt.want)
 			}

internal/config/cli.go

@@ -113,6 +113,7 @@ type CLI struct {
 	Templates            []string `kong:"default='generic-service',env='ALERTMANAGER_ICINGA_BRIDGE_SERVICE_TEMPLATE',help='Create Icinga services with the given template (can be repeated)'"`
 
 	IcingaHostObjectLabels   []string `kong:"default='icinga_use_host',env='ALERTMANAGER_BRIDGE_ICINGA_HOST_OBJECT_LABELS',help='List of Labels depicting the Icinga Host object to associate the alert service with (uses --icinga-hostname if no label is found on alerts)'"`
+	IcingaHostCheckLabels    []string `kong:"default='icinga_use_host_check',env='ALERTMANAGER_BRIDGE_ICINGA_HOST_CHECK_LABELS',help='List of Labels to be used to associate the incoming alert with the Icinga Host object instead of a Service object'"`
 	IcingaHostZoneLabels     []string `kong:"default='icinga_use_zone',env='ALERTMANAGER_BRIDGE_ICINGA_HOST_ZONE_LABELS',help='List of Labels to be used to set the zone attribute of the Icinga service object'"`
 	IcingaHostTemplateLabels []string `kong:"default='icinga_use_template',env='ALERTMANAGER_BRIDGE_ICINGA_HOST_TEMPLATE_LABELS',help='List of Labels to be used to set the included template for the Icinga service object (in addition to --templates)'"`
 	PluginOutputAnnotations  []string `kong:"default='message',env='ALERTMANAGER_ICINGA_BRIDGE_PLUGINOUTPUT_ANNOTATIONS',help='List of Annotation names to be used to set the plugin output for the Icinga service'"`

internal/config/config.go

@@ -37,6 +37,7 @@ type Config struct {
 
 	PluginOutputAnnotations  []string
 	IcingaHostObjectLabels   []string
+	IcingaHostCheckLabels    []string
 	IcingaHostZoneLabels     []string
 	IcingaHostTemplateLabels []string
 	NotesTextAnnotations     []string
@@ -75,6 +76,7 @@ func NewConfigFromCLI(cli *CLI) (*Config, error) {
 		PluginOutputByStates:     cli.PluginOutputByStates,
 		PluginOutputAnnotations:  cli.PluginOutputAnnotations,
 		IcingaHostObjectLabels:   cli.IcingaHostObjectLabels,
+		IcingaHostCheckLabels:    cli.IcingaHostCheckLabels,
 		IcingaHostZoneLabels:     cli.IcingaHostZoneLabels,
 		IcingaHostTemplateLabels: cli.IcingaHostTemplateLabels,
 		NotesTextAnnotations:     cli.NotesTextAnnotations,

internal/icinga2/icinga.go

@@ -75,7 +75,7 @@ func (c *Client) Do(req *http.Request, path string) (*http.Response, error) {
 	var endpointErrors strings.Builder
 
 	for _, base := range c.IcingaURL {
-		// Since we determine what the path ist, this probably won't fail
+		// Since we determine what the path is this probably won't fail
 		req.URL, _ = req.URL.Parse(base + path)
 
 		// Reset the body for the attempt, http.Client.Do always drains/closes