From cec550a24945875b9c640939f5f4ce246f4711ff Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Mon, 19 Oct 2026 00:58:10 +0200 Subject: [PATCH] refactor: notification metrics come from spans, and NotificationMetrics is gone Each delivery is a notification.deliver span with category, channel, severity, result, error and delivery seconds, and the throttle, each retry and each subscription update leave a notification.throttled, notification.retry or notification.subscription_change span. The collector's signaltometrics connector makes the same ten series from them. Labels take the span attribute names, so the notifications dashboard moves with them. Part of #389. --- backend/app/core/metrics/__init__.py | 2 - backend/app/core/metrics/notifications.py | 83 -------------- backend/app/core/providers.py | 9 +- backend/app/services/notification_service.py | 107 +++++++++--------- .../dashboards/notifications.json | 20 ++-- backend/otel-collector-config.yaml | 107 ++++++++++++++++++ backend/tests/unit/conftest.py | 6 - .../test_notification_delivery_metrics.py | 88 ++++++++++---- docs/operations/metrics-reference.md | 28 ++--- 9 files changed, 257 insertions(+), 193 deletions(-) delete mode 100644 backend/app/core/metrics/notifications.py diff --git a/backend/app/core/metrics/__init__.py b/backend/app/core/metrics/__init__.py index ee4176cb..86ac4f77 100644 --- a/backend/app/core/metrics/__init__.py +++ b/backend/app/core/metrics/__init__.py @@ -2,7 +2,6 @@ from app.core.metrics.connections import ConnectionMetrics from app.core.metrics.database import IdempotencyMetrics from app.core.metrics.events import EventMetrics -from app.core.metrics.notifications import NotificationMetrics from app.core.metrics.rate_limit import RateLimitMetrics from app.core.metrics.replay import ReplayMetrics from app.core.metrics.security import SecurityMetrics @@ -12,7 +11,6 @@ "ConnectionMetrics", "IdempotencyMetrics", "EventMetrics", - "NotificationMetrics", "RateLimitMetrics", "ReplayMetrics", "SecurityMetrics", diff --git a/backend/app/core/metrics/notifications.py b/backend/app/core/metrics/notifications.py deleted file mode 100644 index f41acd41..00000000 --- a/backend/app/core/metrics/notifications.py +++ /dev/null @@ -1,83 +0,0 @@ -from app.core.metrics.base import BaseMetrics - - -class NotificationMetrics(BaseMetrics): - """Metrics for notifications.""" - - def _create_instruments(self) -> None: - self.notifications_sent = self._meter.create_counter( - name="notifications.sent.total", description="Total number of notifications sent", unit="1" - ) - - self.notifications_failed = self._meter.create_counter( - name="notifications.failed.total", description="Total number of failed notifications", unit="1" - ) - - self.notification_delivery_time = self._meter.create_histogram( - name="notification.delivery.time", description="Time taken to deliver notifications in seconds", unit="s" - ) - - self.notifications_by_channel = self._meter.create_counter( - name="notifications.by.channel.total", description="Total notifications sent by channel", unit="1" - ) - - self.channel_delivery_time = self._meter.create_histogram( - name="notification.channel.delivery.time", description="Delivery time by channel in seconds", unit="s" - ) - - self.channel_failures = self._meter.create_counter( - name="notification.channel.failures.total", description="Total failures by channel", unit="1" - ) - - self.notifications_by_severity = self._meter.create_counter( - name="notifications.by.severity.total", description="Total notifications by severity level", unit="1" - ) - - self.notifications_throttled = self._meter.create_counter( - name="notifications.throttled.total", description="Total notifications throttled", unit="1" - ) - - self.notification_retries = self._meter.create_counter( - name="notification.retries.total", - description="Delivery attempts that failed and were retried", - unit="1", - ) - - self.subscription_changes = self._meter.create_counter( - name="notification.subscription.changes.total", description="Total subscription changes", unit="1" - ) - - def record_notification_sent( - self, notification_type: str, channel: str = "in_app", severity: str = "medium" - ) -> None: - self.notifications_sent.add(1, attributes={"category": notification_type}) - self.notifications_by_channel.add(1, attributes={"channel": channel, "category": notification_type}) - self.notifications_by_severity.add(1, attributes={"severity": severity, "category": notification_type}) - - def record_notification_failed(self, notification_type: str, error: str, channel: str = "in_app") -> None: - self.notifications_failed.add(1, attributes={"category": notification_type, "error": error}) - self.channel_failures.add(1, attributes={"channel": channel, "error": error}) - - def record_notification_delivery_time( - self, duration_seconds: float, notification_type: str, channel: str = "in_app" - ) -> None: - self.notification_delivery_time.record(duration_seconds, attributes={"category": notification_type}) - self.channel_delivery_time.record( - duration_seconds, attributes={"channel": channel, "category": notification_type} - ) - - def record_notification_throttled(self, notification_type: str) -> None: - self.notifications_throttled.add(1, attributes={"category": notification_type}) - - def record_notification_retry(self, notification_type: str, channel: str) -> None: - self.notification_retries.add(1, attributes={"category": notification_type, "channel": channel}) - - def record_subscription_change(self, channel: str, enabled: bool | None) -> None: - action = "enabled" if enabled is True else "disabled" if enabled is False else "updated" - self.subscription_changes.add( - 1, - attributes={ - "channel": channel, - "action": action, - }, - ) diff --git a/backend/app/core/providers.py b/backend/app/core/providers.py index 27e649c3..530d5727 100644 --- a/backend/app/core/providers.py +++ b/backend/app/core/providers.py @@ -16,7 +16,6 @@ ConnectionMetrics, EventMetrics, IdempotencyMetrics, - NotificationMetrics, RateLimitMetrics, ReplayMetrics, SecurityMetrics, @@ -267,10 +266,6 @@ def get_rate_limit_metrics(self, settings: Settings) -> RateLimitMetrics: def get_idempotency_metrics(self, settings: Settings) -> IdempotencyMetrics: return IdempotencyMetrics(settings) - @provide - def get_notification_metrics(self, settings: Settings) -> NotificationMetrics: - return NotificationMetrics(settings) - @provide def get_replay_metrics(self, settings: Settings) -> ReplayMetrics: return ReplayMetrics(settings) @@ -464,7 +459,7 @@ def get_notification_service( sse_redis_bus: SSERedisBus, settings: Settings, logger: structlog.stdlib.BoundLogger, - notification_metrics: NotificationMetrics, + tracer: trace.Tracer, ) -> NotificationService: return NotificationService( notification_repository=notification_repository, @@ -472,7 +467,7 @@ def get_notification_service( sse_bus=sse_redis_bus, settings=settings, logger=logger, - notification_metrics=notification_metrics, + tracer=tracer, ) @provide diff --git a/backend/app/services/notification_service.py b/backend/app/services/notification_service.py index 5013e983..7228a66f 100644 --- a/backend/app/services/notification_service.py +++ b/backend/app/services/notification_service.py @@ -11,7 +11,6 @@ from tenacity import AsyncRetrying, RetryCallState, stop_after_attempt, wait_exponential from tenacity.wait import wait_base -from app.core.metrics import NotificationMetrics from app.db import NotificationRepository from app.domain.enums import NotificationChannel, NotificationSeverity, NotificationStatus, UserRole from app.domain.events import ( @@ -108,13 +107,13 @@ def __init__( sse_bus: SSERedisBus, settings: Settings, logger: structlog.stdlib.BoundLogger, - notification_metrics: NotificationMetrics, + tracer: trace.Tracer, retry_wait: wait_base = DELIVERY_RETRY_WAIT, ) -> None: self.repository = notification_repository self._retry_wait = retry_wait self._producer = producer - self.metrics = notification_metrics + self._tracer = tracer self.settings = settings self.sse_bus = sse_bus self.logger = logger @@ -172,7 +171,7 @@ async def create_notification( max_per_window=self.settings.NOTIF_THROTTLE_MAX_PER_HOUR, window_hours=self.settings.NOTIF_THROTTLE_WINDOW_HOURS, ) - self.metrics.record_notification_throttled("general") + self._tracer.start_span("notification.throttled", attributes={"notification.category": "general"}).end() raise NotificationThrottledError( user_id, self.settings.NOTIF_THROTTLE_MAX_PER_HOUR, @@ -302,7 +301,9 @@ async def _create_system_for_user( max_per_hour=self.settings.NOTIF_THROTTLE_MAX_PER_HOUR, ) if throttled: - self.metrics.record_notification_throttled("system") + self._tracer.start_span( + "notification.throttled", attributes={"notification.category": "system"} + ).end() return "throttled" await self.create_notification( @@ -553,7 +554,9 @@ async def update_subscription( raise NotificationValidationError("slack_webhook is required when enabling SLACK") result = await self.repository.upsert_subscription(user_id, channel, update_data) - self.metrics.record_subscription_change(channel, update_data.enabled) + action = "enabled" if update_data.enabled is True else "disabled" if update_data.enabled is False else "updated" + attributes = {"notification.channel": channel, "notification.action": action} + self._tracer.start_span("notification.subscription_change", attributes=attributes).end() return result async def mark_all_as_read(self, user_id: str) -> int: @@ -622,9 +625,11 @@ def _log_delivery_retry(self, state: RetryCallState) -> None: max_attempts=notification.max_retries, error=str(state.outcome.exception() if state.outcome else None), ) - self.metrics.record_notification_retry( - notification.tags[0] if notification.tags else "unknown", notification.channel - ) + attributes = { + "notification.category": notification.tags[0] if notification.tags else "unknown", + "notification.channel": notification.channel, + } + self._tracer.start_span("notification.retry", attributes=attributes).end() async def _deliver_notification(self, notification: DomainNotification) -> bool: """Deliver notification through configured channel with inline retry. @@ -674,7 +679,11 @@ async def _deliver_notification(self, notification: DomainNotification) -> bool: return False start_time = asyncio.get_running_loop().time() - notification_type = notification.tags[0] if notification.tags else "unknown" + attributes = { + "notification.category": notification.tags[0] if notification.tags else "unknown", + "notification.channel": notification.channel, + "notification.severity": notification.severity, + } retrying = AsyncRetrying( stop=stop_after_attempt(notification.max_retries), @@ -683,44 +692,40 @@ async def _deliver_notification(self, notification: DomainNotification) -> bool: reraise=True, ) - try: - await retrying(handler, notification, subscription) - delivery_time = asyncio.get_running_loop().time() - start_time - await self.repository.update_notification( - notification.notification_id, - notification.user_id, - DomainNotificationUpdate(status=NotificationStatus.DELIVERED, delivered_at=datetime.now(UTC)), - ) - self.logger.info( - "Delivered notification", - notification_id=str(notification.notification_id), - channel=notification.channel, - delivery_time_ms=int(delivery_time * 1000), - ) - self.metrics.record_notification_sent( - notification_type, channel=notification.channel, severity=notification.severity - ) - self.metrics.record_notification_delivery_time( - delivery_time, notification_type, channel=notification.channel - ) - return True - except Exception as last_error: - await self.repository.update_notification( - notification.notification_id, - notification.user_id, - DomainNotificationUpdate( - status=NotificationStatus.FAILED, - failed_at=datetime.now(UTC), - error_message=f"Delivery failed via {notification.channel}: {last_error}", - retry_count=notification.max_retries, - ), - ) - self.metrics.record_notification_failed( - notification_type, type(last_error).__name__, channel=notification.channel - ) - self.logger.error( - "All delivery attempts exhausted", - notification_id=str(notification.notification_id), - exc_info=last_error, - ) - return False + with self._tracer.start_as_current_span("notification.deliver", attributes=attributes) as span: + try: + await retrying(handler, notification, subscription) + delivery_time = asyncio.get_running_loop().time() - start_time + span.set_attributes( + {"notification.result": "delivered", "notification.delivery_seconds": delivery_time} + ) + await self.repository.update_notification( + notification.notification_id, + notification.user_id, + DomainNotificationUpdate(status=NotificationStatus.DELIVERED, delivered_at=datetime.now(UTC)), + ) + self.logger.info( + "Delivered notification", + notification_id=str(notification.notification_id), + channel=notification.channel, + delivery_time_ms=int(delivery_time * 1000), + ) + return True + except Exception as last_error: + span.set_attributes({"notification.result": "failed", "notification.error": type(last_error).__name__}) + await self.repository.update_notification( + notification.notification_id, + notification.user_id, + DomainNotificationUpdate( + status=NotificationStatus.FAILED, + failed_at=datetime.now(UTC), + error_message=f"Delivery failed via {notification.channel}: {last_error}", + retry_count=notification.max_retries, + ), + ) + self.logger.error( + "All delivery attempts exhausted", + notification_id=str(notification.notification_id), + exc_info=last_error, + ) + return False diff --git a/backend/grafana/provisioning/dashboards/notifications.json b/backend/grafana/provisioning/dashboards/notifications.json index c526bb12..1c25cb4f 100644 --- a/backend/grafana/provisioning/dashboards/notifications.json +++ b/backend/grafana/provisioning/dashboards/notifications.json @@ -96,7 +96,7 @@ "targets": [ { "expr": "rate(notifications_by_channel_total[5m])", - "legendFormat": "{{channel}}", + "legendFormat": "{{notification_channel}}", "refId": "A", "datasource": "Victoria Metrics" } @@ -121,7 +121,7 @@ "targets": [ { "expr": "rate(notifications_by_severity_total[5m])", - "legendFormat": "{{severity}}", + "legendFormat": "{{notification_severity}}", "refId": "A", "datasource": "Victoria Metrics" } @@ -145,8 +145,8 @@ "id": 7, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(notification_channel_delivery_time_seconds_bucket[5m])) by (le, channel))", - "legendFormat": "{{channel}} p95", + "expr": "histogram_quantile(0.95, sum(rate(notification_channel_delivery_time_seconds_bucket[5m])) by (le, notification_channel))", + "legendFormat": "{{notification_channel}} p95", "refId": "A", "datasource": "Victoria Metrics" } @@ -170,8 +170,8 @@ "id": 8, "targets": [ { - "expr": "sum by (channel, error) (rate(notification_channel_failures_total[5m]))", - "legendFormat": "{{channel}} {{error}}", + "expr": "sum by (notification_channel, notification_error) (rate(notification_channel_failures_total[5m]))", + "legendFormat": "{{notification_channel}} {{notification_error}}", "refId": "A", "datasource": "Victoria Metrics" } @@ -246,8 +246,8 @@ "id": 11, "targets": [ { - "expr": "sum by (category) (rate(notifications_throttled_total[5m]))", - "legendFormat": "{{category}}", + "expr": "sum by (notification_category) (rate(notifications_throttled_total[5m]))", + "legendFormat": "{{notification_category}}", "refId": "A", "datasource": "Victoria Metrics" } @@ -271,8 +271,8 @@ "id": 12, "targets": [ { - "expr": "sum by (channel) (rate(notification_retries_total[5m]))", - "legendFormat": "{{channel}}", + "expr": "sum by (notification_channel) (rate(notification_retries_total[5m]))", + "legendFormat": "{{notification_channel}}", "refId": "A", "datasource": "Victoria Metrics" } diff --git a/backend/otel-collector-config.yaml b/backend/otel-collector-config.yaml index b676389b..09ed405f 100644 --- a/backend/otel-collector-config.yaml +++ b/backend/otel-collector-config.yaml @@ -373,6 +373,113 @@ connectors: sum: value: "1" monotonic: true + - name: notifications.sent + description: Notifications delivered + unit: "{notification}" + attributes: + - key: notification.category + conditions: + - span.name == "notification.deliver" and span.attributes["notification.result"] == "delivered" + sum: + value: "1" + monotonic: true + - name: notifications.by.channel + description: Notifications delivered, by channel + unit: "{notification}" + attributes: + - key: notification.channel + - key: notification.category + conditions: + - span.name == "notification.deliver" and span.attributes["notification.result"] == "delivered" + sum: + value: "1" + monotonic: true + - name: notifications.by.severity + description: Notifications delivered, by severity + unit: "{notification}" + attributes: + - key: notification.severity + - key: notification.category + conditions: + - span.name == "notification.deliver" and span.attributes["notification.result"] == "delivered" + sum: + value: "1" + monotonic: true + - name: notifications.failed + description: Notifications that failed every delivery attempt + unit: "{notification}" + attributes: + - key: notification.category + - key: notification.error + conditions: + - span.name == "notification.deliver" and span.attributes["notification.result"] == "failed" + sum: + value: "1" + monotonic: true + - name: notification.channel.failures + description: Notifications that failed every delivery attempt, by channel + unit: "{notification}" + attributes: + - key: notification.channel + - key: notification.error + conditions: + - span.name == "notification.deliver" and span.attributes["notification.result"] == "failed" + sum: + value: "1" + monotonic: true + - name: notification.delivery.time + description: Time to deliver a notification, retries included + unit: s + attributes: + - key: notification.category + conditions: + - span.name == "notification.deliver" and span.attributes["notification.delivery_seconds"] != nil + histogram: + buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30] + value: Double(span.attributes["notification.delivery_seconds"]) + - name: notification.channel.delivery.time + description: Time to deliver a notification, retries included, by channel + unit: s + attributes: + - key: notification.channel + - key: notification.category + conditions: + - span.name == "notification.deliver" and span.attributes["notification.delivery_seconds"] != nil + histogram: + buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30] + value: Double(span.attributes["notification.delivery_seconds"]) + - name: notifications.throttled + description: Notifications held back by the per-user throttle + unit: "{notification}" + attributes: + - key: notification.category + conditions: + - span.name == "notification.throttled" + sum: + value: "1" + monotonic: true + - name: notification.retries + description: Delivery attempts that failed and were retried + unit: "{retry}" + attributes: + - key: notification.category + - key: notification.channel + conditions: + - span.name == "notification.retry" + sum: + value: "1" + monotonic: true + - name: notification.subscription.changes + description: Subscription updates, by channel and action + unit: "{change}" + attributes: + - key: notification.channel + - key: notification.action + conditions: + - span.name == "notification.subscription_change" + sum: + value: "1" + monotonic: true processors: batch: diff --git a/backend/tests/unit/conftest.py b/backend/tests/unit/conftest.py index a4263e0d..4124ab0e 100644 --- a/backend/tests/unit/conftest.py +++ b/backend/tests/unit/conftest.py @@ -26,7 +26,6 @@ ConnectionMetrics, EventMetrics, IdempotencyMetrics, - NotificationMetrics, RateLimitMetrics, ReplayMetrics, SecurityMetrics, @@ -239,11 +238,6 @@ def event_metrics(test_settings: Settings) -> EventMetrics: return EventMetrics(test_settings) -@pytest.fixture -def notification_metrics(test_settings: Settings) -> NotificationMetrics: - return NotificationMetrics(test_settings) - - @pytest.fixture def rate_limit_metrics(test_settings: Settings) -> RateLimitMetrics: return RateLimitMetrics(test_settings) diff --git a/backend/tests/unit/services/test_notification_delivery_metrics.py b/backend/tests/unit/services/test_notification_delivery_metrics.py index 79f248bc..9123e931 100644 --- a/backend/tests/unit/services/test_notification_delivery_metrics.py +++ b/backend/tests/unit/services/test_notification_delivery_metrics.py @@ -1,22 +1,24 @@ -from unittest.mock import AsyncMock, MagicMock +from typing import Any +from unittest.mock import AsyncMock import pytest import structlog +from opentelemetry import trace +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from tenacity import wait_none -from app.core.metrics import NotificationMetrics from app.db import NotificationRepository -from app.domain.enums import NotificationChannel -from app.domain.notification import DomainNotification, DomainNotificationSubscription +from app.domain.enums import NotificationChannel, NotificationSeverity +from app.domain.notification import DomainNotification, DomainNotificationSubscription, DomainSubscriptionUpdate from app.events import UnifiedProducer -from app.services.notification_service import NotificationService +from app.services.notification_service import NotificationService, SystemConfig, ThrottleCache from app.services.sse import SSERedisBus from app.settings import Settings pytestmark = pytest.mark.unit -def _service(test_settings: Settings, metrics: NotificationMetrics, bus: SSERedisBus) -> NotificationService: +def _service(test_settings: Settings, tracer: trace.Tracer, bus: SSERedisBus) -> NotificationService: repo = AsyncMock(spec=NotificationRepository) repo.try_claim_pending.return_value = True repo.get_subscription.return_value = DomainNotificationSubscription( @@ -28,35 +30,81 @@ def _service(test_settings: Settings, metrics: NotificationMetrics, bus: SSERedi sse_bus=bus, settings=test_settings, logger=structlog.get_logger("test.notification_delivery_metrics"), - notification_metrics=metrics, + tracer=tracer, retry_wait=wait_none(), ) -async def test_retried_delivery_counts_the_retry(test_settings: Settings) -> None: - metrics = MagicMock(spec=NotificationMetrics) +def _spans(exporter: InMemorySpanExporter) -> list[tuple[str, dict[str, Any]]]: + return [(span.name, dict(span.attributes or {})) for span in exporter.get_finished_spans()] + + +async def test_retried_delivery_counts_the_retry( + test_settings: Settings, tracer: trace.Tracer, span_exporter: InMemorySpanExporter +) -> None: bus = AsyncMock(spec=SSERedisBus) bus.publish_notification.side_effect = [ConnectionError("redis down"), None] - svc = _service(test_settings, metrics, bus) + svc = _service(test_settings, tracer, bus) delivered = await svc._deliver_notification(DomainNotification(user_id="u1", tags=["execution"], max_retries=2)) assert delivered is True - metrics.record_notification_retry.assert_called_once_with("execution", NotificationChannel.IN_APP) - metrics.record_notification_sent.assert_called_once() - metrics.record_notification_failed.assert_not_called() + (retry_name, retry), (deliver_name, deliver) = _spans(span_exporter) + assert (retry_name, retry) == ( + "notification.retry", + {"notification.category": "execution", "notification.channel": "in_app"}, + ) + assert deliver_name == "notification.deliver" + assert deliver.pop("notification.delivery_seconds") >= 0 + assert deliver == { + "notification.category": "execution", + "notification.channel": "in_app", + "notification.severity": "medium", + "notification.result": "delivered", + } -async def test_failed_delivery_labels_the_error_by_type(test_settings: Settings) -> None: - metrics = MagicMock(spec=NotificationMetrics) +async def test_failed_delivery_labels_the_error_by_type( + test_settings: Settings, tracer: trace.Tracer, span_exporter: InMemorySpanExporter +) -> None: bus = AsyncMock(spec=SSERedisBus) bus.publish_notification.side_effect = ConnectionError("redis down at 10.0.0.1:6379") - svc = _service(test_settings, metrics, bus) + svc = _service(test_settings, tracer, bus) delivered = await svc._deliver_notification(DomainNotification(user_id="u1", tags=["execution"], max_retries=1)) assert delivered is False - metrics.record_notification_retry.assert_not_called() - metrics.record_notification_failed.assert_called_once_with( - "execution", "ConnectionError", channel=NotificationChannel.IN_APP - ) + [(name, attributes)] = _spans(span_exporter) + assert name == "notification.deliver" + assert (attributes["notification.result"], attributes["notification.error"]) == ("failed", "ConnectionError") + + +@pytest.mark.parametrize(("enabled", "action"), [(True, "enabled"), (False, "disabled"), (None, "updated")]) +async def test_subscription_change_reports_its_action( + test_settings: Settings, + tracer: trace.Tracer, + span_exporter: InMemorySpanExporter, + enabled: bool | None, + action: str, +) -> None: + svc = _service(test_settings, tracer, AsyncMock(spec=SSERedisBus)) + + await svc.update_subscription("u1", NotificationChannel.IN_APP, DomainSubscriptionUpdate(enabled=enabled)) + + assert _spans(span_exporter) == [ + ("notification.subscription_change", {"notification.channel": "in_app", "notification.action": action}) + ] + + +async def test_throttled_system_notification_reports_the_throttle( + test_settings: Settings, tracer: trace.Tracer, span_exporter: InMemorySpanExporter +) -> None: + svc = _service(test_settings, tracer, AsyncMock(spec=SSERedisBus)) + svc._throttle_cache = AsyncMock(spec=ThrottleCache) + svc._throttle_cache.check_throttle.return_value = True + cfg = SystemConfig(severity=NotificationSeverity.LOW, throttle_exempt=False) + + outcome = await svc._create_system_for_user("u1", cfg, "Maintenance", {"message": "soon"}, ["system"]) + + assert outcome == "throttled" + assert _spans(span_exporter) == [("notification.throttled", {"notification.category": "system"})] diff --git a/docs/operations/metrics-reference.md b/docs/operations/metrics-reference.md index 6e97fe99..320e9786 100644 --- a/docs/operations/metrics-reference.md +++ b/docs/operations/metrics-reference.md @@ -24,7 +24,7 @@ The collector's metrics pipeline (`backend/otel-collector-config.yaml`) reads th ## Metrics made from spans -Some metrics have no instrument in the code. A service puts the facts on a span, and the collector's `signaltometrics` connector (alpha in collector v0.162.0) turns them into counters, histograms and gauges, defined under `connectors` in `backend/otel-collector-config.yaml`. The SDK sends every span (`TRACING_SAMPLING_RATE = 1.0`), and the traces pipeline gives each span to the connector before `tail_sampling` decides what Jaeger keeps, so the counts don't depend on sampling. The connector emits a point only when a matching span arrives. Sent straight to VictoriaMetrics, a counter that sees one event every half hour would get one point every half hour, and `increase` and `rate` stop finding the previous point once two points are more than about 8 to 10 minutes apart (measured on v1.153.0): `increase` then counts the whole total again, and `rate` returns nothing. So counters and histograms go another way. `deltatocumulative` adds up the connector's deltas and keeps each total through 24 hours without spans (its default is 5 minutes, after which an idle counter would start again from zero). The collector's `prometheus` exporter serves the totals on `otel-collector:8889`, and VictoriaMetrics scrapes it every 15 seconds (`backend/victoria-metrics/scrape.yaml`, with `honor_labels` so each service's `job` and `instance` stay). A series the exporter hasn't updated for 24 hours drops out until its next span. Gauges still go by remote write, so a gauge goes stale when the job that reports it on a timer stops. Panels wrap counter rates in `or vector(0)` so a series that hasn't started yet reads 0. The execution, pod, queue and dead letter queue metrics work this way so far ([#389](https://github.com/HardMax71/Integr8sCode/issues/389)). +Some metrics have no instrument in the code. A service puts the facts on a span, and the collector's `signaltometrics` connector (alpha in collector v0.162.0) turns them into counters, histograms and gauges, defined under `connectors` in `backend/otel-collector-config.yaml`. The SDK sends every span (`TRACING_SAMPLING_RATE = 1.0`), and the traces pipeline gives each span to the connector before `tail_sampling` decides what Jaeger keeps, so the counts don't depend on sampling. The connector emits a point only when a matching span arrives. Sent straight to VictoriaMetrics, a counter that sees one event every half hour would get one point every half hour, and `increase` and `rate` stop finding the previous point once two points are more than about 8 to 10 minutes apart (measured on v1.153.0): `increase` then counts the whole total again, and `rate` returns nothing. So counters and histograms go another way. `deltatocumulative` adds up the connector's deltas and keeps each total through 24 hours without spans (its default is 5 minutes, after which an idle counter would start again from zero). The collector's `prometheus` exporter serves the totals on `otel-collector:8889`, and VictoriaMetrics scrapes it every 15 seconds (`backend/victoria-metrics/scrape.yaml`, with `honor_labels` so each service's `job` and `instance` stay). A series the exporter hasn't updated for 24 hours drops out until its next span. Gauges still go by remote write, so a gauge goes stale when the job that reports it on a timer stops. Panels wrap counter rates in `or vector(0)` so a series that hasn't started yet reads 0. The execution, pod, queue, dead letter queue and notification metrics work this way so far ([#389](https://github.com/HardMax71/Integr8sCode/issues/389)). ## Names and labels in VictoriaMetrics @@ -118,20 +118,20 @@ The status snapshot a stream sends first comes from the database, not Redis, and ### Notifications (backend) -`NotificationService`. Direct and scheduled delivery both run in backend. `category` is the notification's first tag, such as `execution` or `system`, or `unknown` without tags. +`NotificationService` in backend, where direct and scheduled delivery both run. Each delivery is a `notification.deliver` span, and the throttle, each retry and each subscription update leave a span named `notification.throttled`, `notification.retry` or `notification.subscription_change`. `notification_category` is the notification's first tag, such as `execution` or `system`, or `unknown` without tags. -| Metric | Type | Labels | Notes | -|--------|------|--------|-------| -| `notifications_sent_total` | counter | category | delivered | -| `notifications_by_channel_total` | counter | channel, category | delivered, by channel | -| `notifications_by_severity_total` | counter | severity, category | delivered, by severity | -| `notification_delivery_time_seconds` | histogram | category | delivery attempts including backoff waits | -| `notification_channel_delivery_time_seconds` | histogram | channel, category | the same value split by channel | -| `notifications_failed_total` | counter | category, error | `error` is the exception type of the last attempt | -| `notification_channel_failures_total` | counter | channel, error | | -| `notification_retries_total` | counter | category, channel | each failed attempt that backoff retries | -| `notifications_throttled_total` | counter | category: general, system | per-user throttle; `system` for system broadcasts | -| `notification_subscription_changes_total` | counter | channel, action: enabled, disabled, updated | | +| Metric | Type | Labels | Recorded from | +|--------|------|--------|---------------| +| `notifications_sent_total` | counter | notification_category | `notification.deliver` with `notification.result` `delivered` | +| `notifications_by_channel_total` | counter | notification_channel, notification_category | the same spans, by channel | +| `notifications_by_severity_total` | counter | notification_severity, notification_category | the same spans, by severity | +| `notification_delivery_time_seconds` | histogram | notification_category | `notification.delivery_seconds` on those spans: delivery attempts including backoff waits | +| `notification_channel_delivery_time_seconds` | histogram | notification_channel, notification_category | the same value split by channel | +| `notifications_failed_total` | counter | notification_category, notification_error | `notification.deliver` with `notification.result` `failed`; the error is the exception type of the last attempt | +| `notification_channel_failures_total` | counter | notification_channel, notification_error | the same spans, by channel | +| `notification_retries_total` | counter | notification_category, notification_channel | `notification.retry`: each failed attempt that backoff retries | +| `notifications_throttled_total` | counter | notification_category: general, system | `notification.throttled`: the per-user throttle; `system` for system broadcasts | +| `notification_subscription_changes_total` | counter | notification_channel, notification_action: enabled, disabled, updated | `notification.subscription_change` | ### Event replay (backend)