From 6ccc02e861b441d9614f27ae6b3741c3f65a1765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Nicklisch-Franken?= Date: Wed, 22 Jul 2026 09:10:30 +0200 Subject: [PATCH 1/8] Initial alarm system sketch --- .gitignore | 2 + cardano-tracer/bench/cardano-tracer-bench.hs | 1 + cardano-tracer/cardano-tracer.cabal | 11 +- cardano-tracer/docs/alarm-system-concept.md | 493 ++++++++++++++++++ .../src/Cardano/Tracer/Configuration.hs | 214 +++++++- .../src/Cardano/Tracer/Environment.hs | 2 + .../Cardano/Tracer/Handlers/Alarms/Auth.hs | 94 ++++ .../Tracer/Handlers/Alarms/Consumers.hs | 40 ++ .../Tracer/Handlers/Alarms/Registry.hs | 105 ++++ .../Cardano/Tracer/Handlers/Alarms/Server.hs | 221 ++++++++ .../Cardano/Tracer/Handlers/Alarms/Store.hs | 160 ++++++ .../Tracer/Handlers/Alarms/TraceRules.hs | 85 +++ .../Cardano/Tracer/Handlers/Alarms/Types.hs | 204 ++++++++ .../Tracer/Handlers/Logs/TraceObjects.hs | 5 + .../src/Cardano/Tracer/MetaTrace.hs | 56 ++ cardano-tracer/src/Cardano/Tracer/Run.hs | 5 + .../test/Cardano/Tracer/Test/Acceptor.hs | 2 + .../test/Cardano/Tracer/Test/Alarms/Tests.hs | 156 ++++++ .../Cardano/Tracer/Test/DataPoint/Tests.hs | 1 + .../test/Cardano/Tracer/Test/Logs/Tests.hs | 3 + .../test/Cardano/Tracer/Test/Restart/Tests.hs | 1 + cardano-tracer/test/cardano-tracer-test.hs | 2 + 22 files changed, 1858 insertions(+), 5 deletions(-) create mode 100644 cardano-tracer/docs/alarm-system-concept.md create mode 100644 cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs create mode 100644 cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs create mode 100644 cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs create mode 100644 cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs create mode 100644 cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Store.hs create mode 100644 cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TraceRules.hs create mode 100644 cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Types.hs create mode 100644 cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs diff --git a/.gitignore b/.gitignore index 57bd14a96ff..6f9d80034fb 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,8 @@ bench/trace-schemas/messages/ # Test artifacts cardano-tracer/cardano-tracer-test +/cardano-tracer-test/ +/kernel-resource-summary.json # IntellIJ project folder .idea/ diff --git a/cardano-tracer/bench/cardano-tracer-bench.hs b/cardano-tracer/bench/cardano-tracer-bench.hs index 388db5dfd03..29290cba4e4 100644 --- a/cardano-tracer/bench/cardano-tracer-bench.hs +++ b/cardano-tracer/bench/cardano-tracer-bench.hs @@ -118,6 +118,7 @@ main = do , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , tlsCertificate = Nothing , logging = NE.fromList [LoggingParams root FileMode format] , rotation = Nothing diff --git a/cardano-tracer/cardano-tracer.cabal b/cardano-tracer/cardano-tracer.cabal index 49a528944d6..7a2239b06ff 100644 --- a/cardano-tracer/cardano-tracer.cabal +++ b/cardano-tracer/cardano-tracer.cabal @@ -61,6 +61,14 @@ library Cardano.Tracer.Acceptors.Server Cardano.Tracer.Acceptors.Utils + Cardano.Tracer.Handlers.Alarms.Auth + Cardano.Tracer.Handlers.Alarms.Consumers + Cardano.Tracer.Handlers.Alarms.Registry + Cardano.Tracer.Handlers.Alarms.Server + Cardano.Tracer.Handlers.Alarms.Store + Cardano.Tracer.Handlers.Alarms.TraceRules + Cardano.Tracer.Handlers.Alarms.Types + Cardano.Tracer.Handlers.Logs.File Cardano.Tracer.Handlers.Logs.Journal Cardano.Tracer.Handlers.Logs.Rotator @@ -278,7 +286,8 @@ test-suite cardano-tracer-test main-is: cardano-tracer-test.hs - other-modules: Cardano.Tracer.Test.Forwarder + other-modules: Cardano.Tracer.Test.Alarms.Tests + Cardano.Tracer.Test.Forwarder Cardano.Tracer.Test.DataPoint.Tests Cardano.Tracer.Test.Logs.Tests Cardano.Tracer.Test.Restart.Tests diff --git a/cardano-tracer/docs/alarm-system-concept.md b/cardano-tracer/docs/alarm-system-concept.md new file mode 100644 index 00000000000..0824d60c891 --- /dev/null +++ b/cardano-tracer/docs/alarm-system-concept.md @@ -0,0 +1,493 @@ +# Alarm system concept for `cardano-tracer` + +Status: proposal + +## Summary + +`cardano-tracer` should provide a central alarm service for alarms detected by: + +1. `hermod-recon`, running as a separate process; +2. rules evaluated against the timeseries store in `cardano-tracer`; and +3. trace messages received by `cardano-tracer` whose severity is at or above a + configured threshold. + +Both sources publish the same immutable `AlarmEvent` representation. The alarm +service stores accepted events, fans them out to statically configured consumers, +and makes them available to authorised clients through a REST API. + +An alarm is a one-shot event. There is no acknowledgement, ownership, or +active/resolved lifecycle in this proposal. + +## Goals + +- Give Hermod and timeseries rules one consistent way to raise alarms. +- Decouple alarm detection from alarm delivery. +- Make alarm consumers and their filters configurable. +- Preserve enough evidence to understand why an alarm was raised. +- Prevent a continuously true timeseries rule or a flood of repeated trace + messages from producing an event on every occurrence. +- Keep the set of delivery mechanisms open until the initial consumers are + selected. + +## Non-goals + +- Incident management, acknowledgement, assignment, or resolution. +- Running `hermod-recon` inside the `cardano-tracer` process. +- Defining a new rule language. Hermod continues to use its LTL language and + timeseries rules use the existing `cardano-timeseries-io` query language. +- Letting remote clients create arbitrary timeseries rules through the API. + +## Terminology + +- **Producer**: a component that detects a condition and submits an alarm. +- **Rule**: configured detection logic, identified by a stable `ruleId`. +- **Alarm event**: the immutable record produced by a rule occurrence. +- **Consumer**: a statically configured destination to which the service pushes + matching alarm events. + +## Architecture + +```text + node trace files ---> hermod-recon ---- HTTP POST ----+ + | + node metrics ---> timeseries store ---> rule engine +--> alarm service + | | + node trace messages ---> trace severity rules ------+ +--> alarm store + | +--> configured consumers + | `--> REST history + | + `--> common AlarmEvent +``` + +The alarm service is a component of `cardano-tracer`, but should be kept separate +from the existing legacy email-notification code. The latter groups trace messages +by severity and has a different data model. Its email implementation may later be +reused by an email alarm consumer. + +### Internal components + +1. **Ingress API** validates, authenticates, normalises, and deduplicates alarms + submitted by external producers such as Hermod. +2. **Timeseries rule evaluator** periodically executes configured queries using + the existing in-process `TimeseriesHandle`. +3. **Trace severity matcher** checks every trace message received from a node + against the configured trace severity rules. +4. **Alarm store** assigns an event ID and retains accepted events for history. +5. **Dispatcher** matches accepted events against configured consumers and hands + them to consumer-specific workers. +6. **History API** serves filtered history to authorised clients. + +An event is visible to consumers and readers only after it has been accepted +by the alarm store. + +## Alarm event + +All producers are normalised to a versioned envelope. A representative JSON +event is: + +```json +{ + "schemaVersion": 1, + "eventId": "0198b80a-cad7-7b2d-95cb-37ee0dd3ee81", + "sourceEventId": "hermod-mainnet-42-1731529958123", + "raisedAt": "2026-07-13T12:34:56.123Z", + "receivedAt": "2026-07-13T12:34:56.184Z", + "source": "hermod-recon", + "ruleId": "chain-growth-42", + "severity": "critical", + "summary": "Chain did not grow within the expected interval", + "scope": { + "network": "mainnet", + "nodeId": "relay-1" + }, + "labels": { + "team": "node-operations", + "site": "eu-central" + }, + "details": { + "formulaIndex": 42, + "formula": "...", + "relevance": [] + } +} +``` + +Required producer fields are `sourceEventId`, `raisedAt`, `ruleId`, `severity`, +and `summary`. `cardano-tracer` supplies `eventId`, `receivedAt`, and the trusted +`source` associated with the producer credential. It must not trust a caller to +choose its own source identity. + +`severity` should use the existing Cardano severity vocabulary: +`debug`, `info`, `notice`, `warning`, `error`, `critical`, `alert`, and +`emergency`. Alarm configurations should normally use `warning` or higher. + +`scope` contains well-known routing dimensions. `labels` is an extensible map for +consumer filtering. `details` is source-specific JSON and must have configured +size and nesting limits. + +The pair `(source, sourceEventId)` is the idempotency key. Re-submitting the same +event returns the already assigned `eventId` and does not dispatch it twice. + +## Producers + +### Hermod ReCon + +`hermod-recon` remains a separate process. On a `FormulaNegativeOutcome`, it +submits one alarm to the ingress API. The mapping is: + +| Hermod value | Alarm field | +| --- | --- | +| configured rule name, or formula index as fallback | `ruleId` | +| formula index | `details.formulaIndex` | +| formula | `details.formula` | +| relevant trace events | `details.relevance` | +| configured rule severity | `severity` | +| configured rule summary | `summary` | + +Hermod should use a structured alarm output/sink. Human-readable output and the +current `--grep` output are not a stable integration contract. As an incremental +migration, a small adapter process may translate Hermod's machine-formatted +`FormulaNegativeOutcome` trace into the ingress request, but the preferred result +is a native Hermod HTTP alarm sink. + +The Hermod worker owns retry with exponential backoff for transient failures. +It uses the same `sourceEventId` on every retry so retries are safe. A permanent +validation or authorisation error is logged locally and is not retried forever. + +### Timeseries rules + +Timeseries rules are configured in `cardano-tracer` and evaluated in process. +A rule consists of: + +- a stable identifier, summary, severity, labels, and optional node scope; +- an existing timeseries query that evaluates to a boolean vector; +- an evaluation interval; +- an optional `for` duration for which the expression must remain true; and +- an optional repeat interval. + +For every output series, the evaluator maintains a small state machine: + +```text +false/missing -- true --> pending -- true for configured duration --> publish + ^ | | + `------ false/missing --'---- false/missing <------------------' +``` + +The default is edge-triggered: one event is published when the rule first meets +its `for` duration. It does not publish a resolution event. After the expression +becomes false or missing, a later false-to-true transition may publish a new +alarm. When `repeatEvery` is configured, a still-true rule may publish reminders +at that interval. + +Each output series must generate a distinct deterministic series key from its +labels. The series key is included in `sourceEventId` and in the alarm labels. +Evaluation errors and missing data are traced as health information, not silently +interpreted as alarms. A future rule option may explicitly define missing data as +alarming. + +The evaluator must put limits on query execution time, result cardinality, and +concurrent evaluations so alarm rules cannot starve normal timeseries ingestion +and queries. + +### Trace severity rules + +`cardano-tracer` already receives every trace message forwarded by the connected +nodes. A trace severity rule raises an alarm when a received message's severity +is at or above a configured threshold. A rule consists of: + +- a stable `ruleId` and an optional summary; +- a `threshold` severity (using the same lowercase severity vocabulary); and +- an optional `suppressForSecs` window (default 300 seconds) and optional labels. + +The mapping from a matching trace message to the alarm event is: + +| Trace message value | Alarm field | +| --- | --- | +| message severity | `severity` | +| message timestamp | `raisedAt` | +| node name | `scope.nodeId` | +| namespace (dot-joined) | `labels.namespace` | +| machine-readable message, hostname, thread ID | `details` | + +The trusted `source` is the constant `trace`: like the future timeseries rules, +this is an internal producer that never passes HTTP authentication, so its +identity is fixed rather than derived from a credential. + +Flood prevention reuses the idempotency key instead of a separate rate limiter: +the `sourceEventId` is `ruleId:node:namespace:windowIndex`, where the window +index is the message timestamp divided by `suppressForSecs`. All matches from +the same node and namespace within one window therefore share the +`(source, sourceEventId)` key and collapse into a single alarm; a match in the +next window raises a fresh one. + +## Consumer model + +A consumer implementation has the following conceptual interface: + +```text +initialise(config) -> worker +matches(filter, AlarmEvent) -> Bool +deliver(worker, AlarmEvent) -> delivered | retryable-error | permanent-error +shutdown(worker) +``` + +Each configured consumer has: + +- a unique name and consumer type; +- type-specific destination settings and credentials; +- a filter; +- queue capacity, retry policy, and timeout; and +- an enabled flag. + +Filters can select `source`, `ruleId`, minimum severity, scope fields, and labels. +The first implementation should support conjunction only. More complex boolean +filter expressions can be added later without changing `AlarmEvent`. + +Consumer types are intentionally not fixed by this concept. Likely initial +implementations are `webhook`, `email`, and `log`. A webhook is a useful generic +first delivery mechanism because downstream systems can adapt it to chat or +incident-management products. + +Consumer workers use bounded, independent queues. A slow or broken destination +must not block alarm ingestion or other consumers. Delivery is at least once for +consumers that enable retries; a destination should therefore deduplicate using +`eventId`. Exhausted delivery attempts are retained as operational failures and +traced by `cardano-tracer`. They do not create alarm events recursively. + +## API + +The alarm API should use a separately configurable endpoint, even if its server +implementation later shares code with the timeseries server. + +### Producer ingress + +```text +POST /alarms/v1/events +Authorization: Bearer +Content-Type: application/json +``` + +- `201 Created` for a newly accepted event; +- `200 OK` for an idempotent replay; +- `400 Bad Request` for invalid input; +- `401 Unauthorized` or `403 Forbidden` for rejected credentials; and +- `429 Too Many Requests` when producer limits are exceeded. + +The response contains `eventId`, `receivedAt`, and whether the event was newly +created. + +### History + +```text +GET /alarms/v1/events?after=&limit=100&source=hermod-recon&minSeverity=warning +Authorization: Bearer +``` + +Results are ordered by the store cursor and use cursor pagination. Supported +filters mirror the simple consumer filters. Retention is configurable. + +## Configuration sketch + +The precise Haskell representation can be refined during implementation. This +YAML illustrates the intended operator-facing configuration: + +```yaml +alarms: + endpoint: + epHost: "127.0.0.1" + epPort: 3210 + epForceSSL: true + retention: + maxAge: 7d + maxEvents: 100000 + limits: + maxEventBytes: 262144 + ingressQueue: 1000 + authentication: + producers: + - name: hermod-mainnet + tokenFile: "/run/secrets/hermod-alarm-token" + source: hermod-recon + readers: + - name: operations-dashboard + tokenFile: "/run/secrets/dashboard-alarm-token" + allowHistory: true + filter: + minSeverity: warning + labels: + team: node-operations + timeseriesRules: + - ruleId: high-ping-latency + summary: "Average node ping latency is above 500 ms" + severity: warning + query: "avg_over_time (netdata_statsd_cardano_node_ping_latency_ms_gauge_value_average[now - 5m; now]) > 500" + evaluateEvery: 30s + for: 2m + labels: + team: node-operations + traceRules: + - ruleId: error-traces + summary: "A trace message with severity error or above was received" + threshold: error + suppressForSecs: 300 + labels: + team: node-operations + consumers: + - name: alarm-audit-log + type: log + enabled: true + filter: + minSeverity: warning +``` + +Secrets should be read from protected files or an equivalent secret provider, +not written directly into the main configuration. TLS uses the existing +`tlsCertificate` configuration initially. Deployments that terminate TLS at a +trusted reverse proxy must restrict direct access to the alarm listener. + +## Storage and delivery guarantees + +The store is append-only from the alarm domain's perspective because alarm events +are immutable. It must support: + +- atomic insert with idempotency-key uniqueness; +- cursor-ordered reads and filtered history; +- retention by age and maximum count; and +- recovery across `cardano-tracer` restarts. + +The concrete backend should sit behind a small `AlarmStore` interface. An embedded +transactional database is preferable to ad-hoc JSON files once retry state and +cursor pagination are implemented. Storage failure rejects ingress rather than +publishing an event that cannot subsequently be replayed. + +The core guarantee is: after the service reports an event as accepted, the event +is durable until retention removes it. Configured consumers receive at-least-once +delivery when retries are enabled. + +## Security + +- Deny producer and history access by default. +- Give producers, readers, and administrators distinct credentials and roles. +- Derive the producer source from credentials. +- Apply per-credential request and connection limits. +- Bound event size, label count, label lengths, JSON depth, and query result + cardinality. +- Do not include credentials in traces or alarm details. +- Treat Hermod relevance evidence as potentially sensitive operational data; + consumer and reader filters are an authorisation boundary. +- Refuse to start an externally reachable clear-text endpoint unless an explicit + insecure setting is enabled. + +The first version does not require a remote administration API. Rules, consumers, +and credentials are managed through configuration and take effect on restart. +Hot reload can be considered separately. + +## Observability + +`cardano-tracer` should expose metrics and structured traces for: + +- accepted, rejected, duplicate, and rate-limited ingress events; +- timeseries rule evaluations, evaluation failures, and evaluation duration; +- alarms published by source, rule, and severity, with bounded label cardinality; +- per-consumer queue depth, attempts, successes, retries, permanent failures, + and drops; and +- alarm-store size, retention removals, and failures. + +Alarm infrastructure failures must not create alarms through the same pipeline, +which could cause a feedback loop. They are emitted as normal internal traces and +metrics. + +## Failure behaviour + +- If Hermod cannot reach `cardano-tracer`, Hermod retries with its stable + idempotency key and bounded local buffering. +- If the store is unavailable, external ingress fails and timeseries alarm + publication is retried without advancing the rule's published state. +- If one consumer is unavailable, its worker retries independently while other + consumers continue. +- If a timeseries query fails, the rule reports an evaluation failure and keeps + its previous state until a configured stale-state timeout; it does not publish + a false alarm. +- If `cardano-tracer` restarts, persisted rule state avoids duplicate alarm edges. + When state is unavailable, the evaluator observes one complete `for` duration + before publishing. + +## Suggested implementation phases + +### Phase 1: alarm core and Hermod ingress + +- Introduce `AlarmEvent`, validation, store, and idempotent ingress. +- Add the authenticated history endpoint. +- Add a `log` consumer to exercise filtering and dispatch. +- Add either the native Hermod HTTP sink or the temporary machine-trace adapter. + +### Phase 2: timeseries evaluation + +- Add configuration and validation for timeseries rules. +- Implement edge/`for` state and persistence. +- Add evaluator safety limits and metrics. + +### Phase 3: production consumers + +- Select and implement the required consumer types. +- Add independent retry queues and persisted delivery attempts where required. +- Add operational dashboards and retention tooling. + +## Possible future features + +The features in this section are not part of the current design. They are +recorded here so a later revision can pick them up without redesigning the +core. + +### Live subscription (SSE) + +A **subscriber** is an authorised API client that receives a live stream of +alarm events. Subscribers are deliberately different from consumers: consumers +are controlled by the operator's configuration and can implement reliable +delivery, while subscribers are dynamic clients and receive a best-effort live +stream. Guaranteeing delivery to a live client while it is disconnected is +explicitly out of scope. + +```text +GET /alarms/v1/stream?source=timeseries&minSeverity=warning +Authorization: Bearer +Accept: text/event-stream +Last-Event-ID: +``` + +Server-Sent Events (SSE) is recommended over WebSockets because the data flow +is one-way, it works with ordinary HTTP infrastructure, and it has a standard +reconnection model. Events use `event: alarm`, an event ID/cursor, and the JSON +`AlarmEvent` as data. Periodic comments act as heartbeats. + +Subscription is disabled unless `allowSubscriptions` is true. A reader +credential gains an `allowSubscription` flag; its allowed filter acts as a +ceiling that a requested filter can only narrow, never broaden. If retained +history contains the `Last-Event-ID`, the server replays later events before +switching to live delivery. Otherwise it reports that the cursor is outside +retention and the client must resume from the oldest available cursor. The +alarm store's cursor doubles as the SSE resumption cursor, so history and +stream resumption share the same pagination mechanism. + +Each subscriber gets a bounded queue (a `subscriberQueue` size under `limits` +in the configuration). SSE delivery is best effort: if a client is slow and +its queue fills, only that client is disconnected, and it reconnects using its +last processed event ID. Subscription access is denied by default, and the SSE +queue size joins the list of bounded resources. + +Observability for this feature adds connected SSE subscribers and slow-client +disconnects to the exposed metrics and traces. + +## Decisions still open + +The following choices do not block the architecture: + +1. Which push consumers are required first (`webhook`, `email`, a message broker, + or another integration). +2. Which embedded storage backend should implement `AlarmStore`. +3. Whether Hermod gains a native HTTP alarm sink immediately or initially uses an + adapter. +4. Whether configuration hot reload is needed. +5. Whether consumer retry queues must survive process restart in the first + release. + diff --git a/cardano-tracer/src/Cardano/Tracer/Configuration.hs b/cardano-tracer/src/Cardano/Tracer/Configuration.hs index 1803bc00968..69eab453304 100644 --- a/cardano-tracer/src/Cardano/Tracer/Configuration.hs +++ b/cardano-tracer/src/Cardano/Tracer/Configuration.hs @@ -11,6 +11,13 @@ module Cardano.Tracer.Configuration ( Address + , AlarmFilterConfig (..) + , AlarmsAuthConfig (..) + , AlarmsConfig (..) + , AlarmsConsumerConfig (..) + , AlarmsLimitsConfig (..) + , AlarmsRetentionConfig (..) + , AlarmsTraceRuleConfig (..) , Certificate (..) , Net.HowToConnect (..) , Endpoint (..) @@ -20,18 +27,22 @@ module Cardano.Tracer.Configuration , LogMode (..) , LoggingParams (..) , Network (..) + , ProducerCredentialConfig (..) + , ReaderCredentialConfig (..) , RotationParams (..) , TracerConfig (..) , Verbosity (..) + , alarmSeverityToText + , parseAlarmSeverityText , readTracerConfig ) where -import Cardano.Logging.Types (HowToConnect) +import Cardano.Logging.Types (HowToConnect, SeverityS (..)) import qualified Cardano.Logging.Types as Log import qualified Cardano.Logging.Types as Net import Control.Applicative ((<|>)) -import Data.Aeson (FromJSON (..), ToJSON (..), withObject, (.:)) +import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, (.:), (.:?), (.=)) import Data.Fixed (Pico) import Data.Function ((&)) import Data.Functor ((<&>)) @@ -41,7 +52,7 @@ import Data.List.Extra (notNull) import Data.List.NonEmpty (NonEmpty) import qualified Data.List.NonEmpty as NE import Data.Map.Strict (Map) -import Data.Maybe (catMaybes) +import Data.Maybe (catMaybes, fromMaybe) import Data.String (fromString) import Data.Text (Text) import qualified Data.Text as Text @@ -106,6 +117,179 @@ instance FromJSON RotationParams where rpKeepFilesNum <- o .: "rpKeepFilesNum" pure RotationParams{..} +-- | A conjunctive selection over alarm events, used both for a static +-- consumer's configuration and for a reader credential's allowed +-- (ceiling) or requested filter. +data AlarmFilterConfig = AlarmFilterConfig + { afcSource :: !(Maybe Text) + , afcRuleId :: !(Maybe Text) + , afcMinSeverity :: !(Maybe SeverityS) + , afcScope :: !(Maybe (Map Text Text)) + , afcLabels :: !(Maybe (Map Text Text)) + } + deriving stock (Eq, Show, Generic) + +instance FromJSON AlarmFilterConfig where + parseJSON = withObject "AlarmFilterConfig" \o -> do + afcSource <- o .:? "source" + afcRuleId <- o .:? "ruleId" + afcMinSeverityText <- o .:? "minSeverity" + afcMinSeverity <- traverse parseSeverityOrFail afcMinSeverityText + afcScope <- o .:? "scope" + afcLabels <- o .:? "labels" + pure AlarmFilterConfig{..} + where + parseSeverityOrFail t = + maybe (fail ("unknown severity: " <> Text.unpack t)) pure (parseAlarmSeverityText t) + +instance ToJSON AlarmFilterConfig where + toJSON AlarmFilterConfig{..} = object $ catMaybes + [ ("source" .=) <$> afcSource + , ("ruleId" .=) <$> afcRuleId + , ("minSeverity" .=) . alarmSeverityToText <$> afcMinSeverity + , ("scope" .=) <$> afcScope + , ("labels" .=) <$> afcLabels + ] + +-- | A statically configured alarm dispatch target. The only variant +-- implemented so far is @log@; @webhook@/@email@ are the obvious follow-up +-- extension (see @Cardano.Tracer.Handlers.Alarms.Consumers@). +data AlarmsConsumerConfig = AlarmsConsumerLog + { aclName :: !Text + , aclEnabled :: !Bool + , aclFilter :: !(Maybe AlarmFilterConfig) + } + deriving stock (Eq, Show, Generic) + +instance FromJSON AlarmsConsumerConfig where + parseJSON = withObject "AlarmsConsumerConfig" \o -> do + consumerType :: Text <- o .: "type" + case consumerType of + "log" -> AlarmsConsumerLog + <$> o .: "name" + <*> (fromMaybe True <$> o .:? "enabled") + <*> o .:? "filter" + other -> fail ("unknown alarm consumer type: " <> Text.unpack other) + +instance ToJSON AlarmsConsumerConfig where + toJSON AlarmsConsumerLog{..} = object + [ "type" .= ("log" :: Text) + , "name" .= aclName + , "enabled" .= aclEnabled + , "filter" .= aclFilter + ] + +-- | A producer credential: a bearer token (read once from @pcTokenFile@) that +-- authenticates alarm ingress requests as coming from @pcSource@. The +-- source is never taken from the request body itself. +data ProducerCredentialConfig = ProducerCredentialConfig + { pcName :: !Text + , pcTokenFile :: !FilePath + , pcSource :: !Text + } + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) + +-- | A reader credential: a bearer token that grants history access, capped +-- by an allowed (ceiling) filter that a caller's requested filter may +-- only narrow. +data ReaderCredentialConfig = ReaderCredentialConfig + { rcName :: !Text + , rcTokenFile :: !FilePath + , rcAllowHistory :: !(Maybe Bool) + , rcFilter :: !(Maybe AlarmFilterConfig) + } + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) + +data AlarmsAuthConfig = AlarmsAuthConfig + { aacProducers :: ![ProducerCredentialConfig] + , aacReaders :: ![ReaderCredentialConfig] + } + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) + +data AlarmsRetentionConfig = AlarmsRetentionConfig + { arcMaxAgeSeconds :: !(Maybe Word64) + , arcMaxEvents :: !(Maybe Word64) + } + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) + +data AlarmsLimitsConfig = AlarmsLimitsConfig + { alcMaxEventBytes :: !(Maybe Word64) + } + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) + +-- | A rule that raises an alarm for every received trace message whose +-- severity is at or above @threshold@. @suppressForSecs@ (default 300) +-- bounds the alarm frequency: within one window, repeated matches from the +-- same node and namespace collapse into a single alarm via the store's +-- idempotency key (see @Cardano.Tracer.Handlers.Alarms.TraceRules@). +data AlarmsTraceRuleConfig = AlarmsTraceRuleConfig + { atrRuleId :: !Text + , atrSummary :: !(Maybe Text) + , atrThreshold :: !SeverityS + , atrSuppressForSecs :: !(Maybe Word64) + , atrLabels :: !(Maybe (Map Text Text)) + } + deriving stock (Eq, Show, Generic) + +instance FromJSON AlarmsTraceRuleConfig where + parseJSON = withObject "AlarmsTraceRuleConfig" \o -> do + atrRuleId <- o .: "ruleId" + atrSummary <- o .:? "summary" + thresholdText <- o .: "threshold" + atrThreshold <- parseSeverityOrFail thresholdText + atrSuppressForSecs <- o .:? "suppressForSecs" + atrLabels <- o .:? "labels" + pure AlarmsTraceRuleConfig{..} + where + parseSeverityOrFail t = + maybe (fail ("unknown severity: " <> Text.unpack t)) pure (parseAlarmSeverityText t) + +instance ToJSON AlarmsTraceRuleConfig where + toJSON AlarmsTraceRuleConfig{..} = object $ + [ "ruleId" .= atrRuleId + , "threshold" .= alarmSeverityToText atrThreshold + ] <> catMaybes + [ ("summary" .=) <$> atrSummary + , ("suppressForSecs" .=) <$> atrSuppressForSecs + , ("labels" .=) <$> atrLabels + ] + +-- | Configuration for the alarm subsystem (see +-- @cardano-tracer/docs/alarm-system-concept.md@). @Nothing@ for the +-- enclosing 'Maybe' in 'TracerConfig' is the on/off switch for the whole +-- subsystem; every field below is only meaningful once it's turned on. +data AlarmsConfig = AlarmsConfig + { alEndpoint :: !Endpoint + , alAllowInsecure :: !(Maybe Bool) + , alRetention :: !(Maybe AlarmsRetentionConfig) + , alLimits :: !(Maybe AlarmsLimitsConfig) + , alAuthentication :: !AlarmsAuthConfig + , alConsumers :: ![AlarmsConsumerConfig] + , alTraceRules :: !(Maybe [AlarmsTraceRuleConfig]) + } + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) + +-- | The alarm envelope's wire form uses lowercase severity names (e.g. +-- @"critical"@), unlike 'SeverityS'\'s own derived 'FromJSON'\/'ToJSON' +-- instances (from @trace-dispatcher@), which serialise the capitalised +-- constructor name (@"Critical"@). Every place a severity crosses the +-- alarm system's wire boundary -- the 'AlarmEvent' envelope, YAML +-- 'AlarmFilterConfig', and the @minSeverity=@ query parameter -- must go +-- through these two functions instead of 'SeverityS'\'s own instances, or +-- producers/consumers will silently disagree on casing. +alarmSeverityToText :: SeverityS -> Text +alarmSeverityToText = Text.toLower . Text.pack . show + +parseAlarmSeverityText :: Text -> Maybe SeverityS +parseAlarmSeverityText t = + lookup (Text.toLower t) [(alarmSeverityToText s, s) | s <- [minBound .. maxBound]] + -- | Logging mode. data LogMode = FileMode -- ^ Store items in log file. @@ -164,6 +348,7 @@ data TracerConfig = TracerConfig , hasEKG :: !(Maybe Endpoint) -- ^ Endpoint for EKG web-page. , hasPrometheus :: !(Maybe Endpoint) -- ^ Endpoint for Prometheus web-page. , hasTimeseries :: !(Maybe Endpoint) + , alarms :: !(Maybe AlarmsConfig) -- ^ Alarm subsystem configuration; 'Nothing' disables it. , tlsCertificate :: !(Maybe Certificate) -- | Socket for tracer's to reforward on. Second member of the triplet is the list of prefixes to reforward. -- Third member of the triplet is the forwarder config. @@ -205,6 +390,8 @@ wellFormed TracerConfig { network , hasEKG , hasPrometheus + , hasTimeseries + , alarms , logging } = if null problems @@ -220,15 +407,34 @@ wellFormed TracerConfig , check "duplicate ports in config" $ hasDuplicates ports , check "no host(s) in hasEKG" . nullEndpoint =<< hasEKG , check "no host in hasPrometheus" . nullEndpoint =<< hasPrometheus + , check "no host in hasTimeseries" . nullEndpoint =<< hasTimeseries + , check "no host in alarms endpoint" . nullEndpoint . alEndpoint =<< alarms + , check "alarms: no producer or reader credentials configured" . noCredentials =<< alarms + , check "alarms: duplicate consumer names" . hasDuplicateConsumerNames =<< alarms ] + -- NB. every internal service's endpoint port is included here, including + -- 'hasTimeseries' and 'alarms' (the former was previously missing from + -- this check). ports :: [Port] - ports = epPort <$> catMaybes [hasEKG, hasPrometheus] + ports = epPort <$> catMaybes + [hasEKG, hasPrometheus, hasTimeseries, alEndpoint <$> alarms] check :: String -> Bool -> Maybe String check msg True = Just msg check _ False = Nothing + noCredentials :: AlarmsConfig -> Bool + noCredentials AlarmsConfig{alAuthentication = AlarmsAuthConfig{aacProducers, aacReaders}} = + null aacProducers && null aacReaders + + hasDuplicateConsumerNames :: AlarmsConfig -> Bool + hasDuplicateConsumerNames AlarmsConfig{alConsumers} = + hasDuplicates (map consumerName alConsumers) + + consumerName :: AlarmsConsumerConfig -> Text + consumerName AlarmsConsumerLog{aclName} = aclName + nullAddress :: Address -> Bool nullAddress (Net.LocalPipe address) = null address nullAddress (Net.RemoteSocket host _port) = Text.null host diff --git a/cardano-tracer/src/Cardano/Tracer/Environment.hs b/cardano-tracer/src/Cardano/Tracer/Environment.hs index 5ea9ef3fecb..e6708a72a66 100644 --- a/cardano-tracer/src/Cardano/Tracer/Environment.hs +++ b/cardano-tracer/src/Cardano/Tracer/Environment.hs @@ -5,6 +5,7 @@ module Cardano.Tracer.Environment import Cardano.Logging.Types import Cardano.Timeseries.Component (TimeseriesHandle) import Cardano.Tracer.Configuration +import Cardano.Tracer.Handlers.Alarms.Registry (AlarmRegistry) import Cardano.Tracer.MetaTrace import Cardano.Tracer.Types @@ -29,4 +30,5 @@ data TracerEnv = TracerEnv , teStateDir :: !(Maybe FilePath) , teMetricsHelp :: ![(Text, Builder)] , teTimeseriesHandle :: !(Maybe TimeseriesHandle) + , teAlarmRegistry :: !(Maybe AlarmRegistry) } diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs new file mode 100644 index 00000000000..d178afbd7c5 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs @@ -0,0 +1,94 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | Minimal bearer-token authentication for the alarm HTTP surface. There is +-- no existing auth pattern anywhere in @cardano-tracer@ to follow (none of +-- the existing Prometheus\/EKG\/Timeseries servers check any credential at +-- all), so this is deliberately simple: static token tables, read once from +-- config-supplied files. No hashing, no constant-time comparison, no +-- rotation, no rate limiting. +module Cardano.Tracer.Handlers.Alarms.Auth + ( ProducerCredential (..) + , ReaderCredential (..) + , AuthTables (..) + , loadCredentials + , bearerToken + , lookupProducer + , lookupReader + ) where + +import Cardano.Tracer.Configuration (AlarmsAuthConfig (..), ProducerCredentialConfig (..), + ReaderCredentialConfig (..)) +import Cardano.Tracer.Handlers.Alarms.Types + +import qualified Data.ByteString.Char8 as BSC +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Maybe (fromMaybe) +import Data.Text (Text) +import qualified Data.Text as Text +import qualified Data.Text.Encoding as TE +import qualified Data.Text.Encoding.Error as TE +import Network.HTTP.Types (hAuthorization) +import Network.Wai (Request, requestHeaders) + +data ProducerCredential = ProducerCredential + { pcrSource :: !AlarmSource + } + deriving stock (Eq, Show) + +data ReaderCredential = ReaderCredential + { rcrAllowHistory :: !Bool + , rcrFilter :: !AlarmFilter + } + deriving stock (Eq, Show) + +data AuthTables = AuthTables + { atProducers :: !(Map Text ProducerCredential) + , atReaders :: !(Map Text ReaderCredential) + } + +-- | Reads every configured token file once, at registry-construction time +-- (per the design doc's Security section: "Secrets should be read from +-- protected files"). Not part of 'wellFormed', since that's pure and this +-- needs 'IO'. +loadCredentials :: AlarmsAuthConfig -> IO AuthTables +loadCredentials AlarmsAuthConfig{aacProducers, aacReaders} = do + producers <- traverse loadProducer aacProducers + readers <- traverse loadReader aacReaders + pure AuthTables + { atProducers = Map.fromList producers + , atReaders = Map.fromList readers + } + where + loadProducer ProducerCredentialConfig{pcTokenFile, pcSource} = do + token <- readTokenFile pcTokenFile + pure (token, ProducerCredential (AlarmSource pcSource)) + + loadReader ReaderCredentialConfig{rcTokenFile, rcAllowHistory, rcFilter} = do + token <- readTokenFile rcTokenFile + pure ( token + , ReaderCredential + { rcrAllowHistory = fromMaybe False rcAllowHistory + , rcrFilter = maybe emptyAlarmFilter filterFromConfig rcFilter + } + ) + +readTokenFile :: FilePath -> IO Text +readTokenFile path = Text.strip . Text.pack <$> readFile path + +-- | Extract the bearer token from a request's @Authorization@ header, if +-- present and well-formed. +bearerToken :: Request -> Maybe Text +bearerToken req = lookup hAuthorization (requestHeaders req) >>= extractBearer + where + extractBearer bs + | BSC.isPrefixOf "Bearer " bs = + Just (Text.strip (TE.decodeUtf8With TE.lenientDecode (BSC.drop 7 bs))) + | otherwise = Nothing + +lookupProducer :: AuthTables -> Text -> Maybe ProducerCredential +lookupProducer tables token = Map.lookup token (atProducers tables) + +lookupReader :: AuthTables -> Text -> Maybe ReaderCredential +lookupReader tables token = Map.lookup token (atReaders tables) diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs new file mode 100644 index 00000000000..9d1a59d2a60 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs @@ -0,0 +1,40 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | Statically configured alarm dispatch targets. Phase 1 (this sketch) +-- implements only 'ConsumerLog', matching the design doc's own staging +-- ("Add a log consumer to exercise filtering and dispatch"). Adding +-- @webhook@\/@email@ consumers later is one new constructor here and one new +-- case in 'dispatch' -- @email@ in particular can reuse the SMTP-sending +-- primitives in "Cardano.Tracer.Handlers.Notifications.Email" -- not +-- speculative machinery added now. +module Cardano.Tracer.Handlers.Alarms.Consumers + ( AlarmConsumer (..) + , consumerFromConfig + , dispatch + ) where + +import Cardano.Tracer.Configuration (AlarmsConsumerConfig (..)) +import Cardano.Tracer.Handlers.Alarms.Types +import Cardano.Tracer.MetaTrace (TracerTrace (..), Trace, traceWith) + +import Data.Text (Text) + +data AlarmConsumer = ConsumerLog + { clName :: !Text + , clEnabled :: !Bool + , clFilter :: !AlarmFilter + } + +consumerFromConfig :: AlarmsConsumerConfig -> AlarmConsumer +consumerFromConfig AlarmsConsumerLog{aclName, aclEnabled, aclFilter} = ConsumerLog + { clName = aclName + , clEnabled = aclEnabled + , clFilter = maybe emptyAlarmFilter filterFromConfig aclFilter + } + +dispatch :: Trace IO TracerTrace -> AlarmConsumer -> AlarmEvent -> IO () +dispatch tracer consumer@ConsumerLog{clEnabled, clFilter} ev + | clEnabled && matchesFilter clFilter ev = + traceWith tracer TracerAlarmDispatched { ttAlarmDispatchedConsumer = clName consumer } + | otherwise = pure () diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs new file mode 100644 index 00000000000..a7f72e60497 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs @@ -0,0 +1,105 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | The central alarm registry: static consumers (fixed at startup) sitting +-- on top of the append-only store. Mirrors the brainstorm design doc's +-- @AlarmRegistry@ sketch. Like +-- "Cardano.Tracer.Handlers.Alarms.Types"\/"...Store", this module never +-- imports "Cardano.Tracer.Environment". +module Cardano.Tracer.Handlers.Alarms.Registry + ( AlarmRegistry + , newAlarmRegistry + , acceptEvent + , rejectEvent + , checkTraceObjectsForAlarms + , readHistoryFiltered + , lookupProducerCredential + , lookupReaderCredential + ) where + +import Cardano.Logging.Types (TraceObject) +import Cardano.Tracer.Configuration (AlarmsConfig (..), AlarmsRetentionConfig (..)) +import Cardano.Tracer.Handlers.Alarms.Auth +import Cardano.Tracer.Handlers.Alarms.Consumers +import Cardano.Tracer.Handlers.Alarms.Store +import Cardano.Tracer.Handlers.Alarms.TraceRules +import Cardano.Tracer.Handlers.Alarms.Types +import Cardano.Tracer.MetaTrace (TracerTrace (..), Trace, traceWith) + +import Data.Foldable (for_) +import Data.Maybe (fromMaybe, mapMaybe) +import Data.Text (Text) +import Data.Time.Clock (getCurrentTime) + +data AlarmRegistry = AlarmRegistry + { arStore :: !AlarmStoreHandle + , arConsumers :: ![AlarmConsumer] -- ^ static, fixed at startup + , arTraceRules :: ![TraceAlarmRule] + , arAuth :: !AuthTables + , arTracer :: !(Trace IO TracerTrace) + } + +newAlarmRegistry :: Trace IO TracerTrace -> AlarmsConfig -> IO AlarmRegistry +newAlarmRegistry tracer AlarmsConfig{alRetention, alConsumers, alAuthentication, alTraceRules} = do + store <- newAlarmStore (fromMaybe emptyRetention alRetention) + authTable <- loadCredentials alAuthentication + pure AlarmRegistry + { arStore = store + , arConsumers = map consumerFromConfig alConsumers + , arTraceRules = map traceRuleFromConfig (fromMaybe [] alTraceRules) + , arAuth = authTable + , arTracer = tracer + } + where + emptyRetention = AlarmsRetentionConfig Nothing Nothing + +-- | Accept a (possibly duplicate) producer submission. On a genuinely new +-- event: insert into the store, then synchronously dispatch to every +-- static consumer. On a replay of an already-accepted +-- @(source, sourceEventId)@: skip dispatch -- "does not dispatch it +-- twice", per the design doc. +acceptEvent :: AlarmRegistry -> AlarmSource -> IngressRequest -> IO (AlarmEvent, Bool) +acceptEvent registry src req = do + receivedAt <- getCurrentTime + (_cursor, ev, wasCreated) <- insertOrGetExisting (arStore registry) src receivedAt req + if wasCreated + then do + traceWith (arTracer registry) TracerAlarmAccepted + { ttAlarmAcceptedSource = unAlarmSource src + , ttAlarmAcceptedRuleId = unRuleId (ruleId ev) + , ttAlarmAcceptedSeverity = severity ev + } + for_ (arConsumers registry) \consumer -> dispatch (arTracer registry) consumer ev + else + traceWith (arTracer registry) TracerAlarmDuplicate + { ttAlarmDuplicateSource = unAlarmSource src + , ttAlarmDuplicateSourceEventId = sourceEventId ev + } + pure (ev, wasCreated) + +-- | Trace a rejected (invalid, unauthenticated, or oversized) ingress +-- request. Never goes through 'acceptEvent' -- a rejected request never +-- reaches the store. +rejectEvent :: AlarmRegistry -> Text -> IO () +rejectEvent registry reason = + traceWith (arTracer registry) TracerAlarmRejected { ttAlarmRejectedReason = reason } + +-- | Check every received trace message against the configured trace +-- severity rules and submit each match through the normal 'acceptEvent' +-- path. Within a rule's suppression window, repeated matches share their +-- idempotency key, so 'acceptEvent' reports them as duplicates and +-- dispatches nothing. +checkTraceObjectsForAlarms :: AlarmRegistry -> Text -> [TraceObject] -> IO () +checkTraceObjectsForAlarms registry nodeName traceObjects = + for_ (arTraceRules registry) \rule -> + for_ (mapMaybe (traceRuleRequest rule nodeName) traceObjects) \req -> + acceptEvent registry traceAlarmSource req + +readHistoryFiltered :: AlarmRegistry -> Maybe AlarmCursor -> Int -> AlarmFilter -> IO [(AlarmCursor, AlarmEvent)] +readHistoryFiltered registry = readHistory (arStore registry) + +lookupProducerCredential :: AlarmRegistry -> Text -> Maybe ProducerCredential +lookupProducerCredential registry = lookupProducer (arAuth registry) + +lookupReaderCredential :: AlarmRegistry -> Text -> Maybe ReaderCredential +lookupReaderCredential registry = lookupReader (arAuth registry) diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs new file mode 100644 index 00000000000..0b982112c28 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs @@ -0,0 +1,221 @@ +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} + +-- | The alarm HTTP surface: producer ingress and history. A hand-rolled +-- 'wai' 'Application', modeled directly on +-- "Cardano.Tracer.Handlers.Metrics.TimeseriesServer" (no @servant@ +-- anywhere in this codebase). +module Cardano.Tracer.Handlers.Alarms.Server + ( runAlarms + ) where + +import Cardano.Tracer.Configuration +import Cardano.Tracer.Environment (TracerEnv (..)) +import Cardano.Tracer.Handlers.Alarms.Auth +import Cardano.Tracer.Handlers.Alarms.Registry +import Cardano.Tracer.Handlers.Alarms.Types +import Cardano.Tracer.Handlers.Metrics.Utils (contentHdrJSON) +import Cardano.Tracer.MetaTrace + +import Control.Monad (join) +import Data.Aeson (encode, object, (.=)) +import qualified Data.Aeson as Aeson +import qualified Data.ByteString as BS +import qualified Data.ByteString.Builder as BB +import qualified Data.ByteString.Lazy as BL +import Data.Foldable (for_) +import Data.Maybe (fromMaybe) +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Text.Read (decimal) +import Data.Word (Word64) +import Network.HTTP.Types +import Network.Wai +import Network.Wai.Handler.Warp hiding (run) +import Network.Wai.Handler.WarpTLS +import System.Time.Extra (sleep) + +-------------------------------------------------------------------------------- +-- Response helpers +-------------------------------------------------------------------------------- + +jsonResponse :: Status -> Aeson.Value -> Response +jsonResponse st v = responseLBS st contentHdrJSON (encode v) + +okResponse, createdResponse :: Aeson.Value -> Response +okResponse = jsonResponse status200 +createdResponse = jsonResponse status201 + +badRequest :: Text -> Response +badRequest msg = jsonResponse status400 (object ["error" .= msg]) + +unauthorized :: Response +unauthorized = jsonResponse status401 (object ["error" .= ("missing or invalid bearer token" :: Text)]) + +forbidden :: Text -> Response +forbidden msg = jsonResponse status403 (object ["error" .= msg]) + +notFound :: Response +notFound = responseLBS status404 [] "" + +-------------------------------------------------------------------------------- +-- Config helpers +-------------------------------------------------------------------------------- + +maxEventBytes :: AlarmsConfig -> Word64 +maxEventBytes cfg = fromMaybe 262144 (alLimits cfg >>= alcMaxEventBytes) -- 256 KiB default, matching the design doc's example config + +-------------------------------------------------------------------------------- +-- Query-string / header helpers +-------------------------------------------------------------------------------- + +lookupParam :: Text -> [(Text, Maybe Text)] -> Maybe Text +lookupParam key params = join (lookup key params) + +parseCursorText :: Text -> Maybe AlarmCursor +parseCursorText t = case decimal t of + Right (n, "") -> Just (AlarmCursor n) + _ -> Nothing + +parseIntText :: Text -> Maybe Int +parseIntText t = case decimal t of + Right (n, "") -> Just n + _ -> Nothing + +cursorToText :: AlarmCursor -> Text +cursorToText (AlarmCursor n) = Text.pack (show n) + +requestedFilterFromQuery :: [(Text, Maybe Text)] -> AlarmFilter +requestedFilterFromQuery params = AlarmFilter + { afSource = AlarmSource <$> lookupParam "source" params + , afRuleId = RuleId <$> lookupParam "ruleId" params + , afMinSeverity = lookupParam "minSeverity" params >>= parseAlarmSeverityText + -- Scope/label query-string filtering isn't implemented in this sketch; + -- only source/ruleId/minSeverity are selectable via GET requests. + , afScope = mempty + , afLabels = mempty + } + +-------------------------------------------------------------------------------- +-- Body-size-limited read (checked before decoding, per the design doc's +-- ingress limits -- never fully materialise an attacker-controlled oversized +-- payload) +-------------------------------------------------------------------------------- + +readLimitedBody :: Word64 -> Request -> IO (Maybe BL.ByteString) +readLimitedBody limit req = case requestBodyLength req of + KnownLength n | n > limit -> pure Nothing + _ -> go 0 mempty + where + go total acc = do + chunk <- getRequestBodyChunk req + let total' = total + fromIntegral (BS.length chunk) + if BS.null chunk + then pure (Just (BB.toLazyByteString acc)) + else if total' > limit + then pure Nothing + else go total' (acc <> BB.byteString chunk) + +-------------------------------------------------------------------------------- +-- Routes +-------------------------------------------------------------------------------- + +alarmsApp :: AlarmsConfig -> AlarmRegistry -> Application +alarmsApp cfg registry request send = case (requestMethod request, pathInfo request) of + ("POST", ["alarms", "v1", "events"]) -> handleIngress cfg registry request send + ("GET", ["alarms", "v1", "events"]) -> handleHistory registry request send + _ -> send notFound + +handleIngress :: AlarmsConfig -> AlarmRegistry -> Application +handleIngress cfg registry request send = + case bearerToken request >>= lookupProducerCredential registry of + Nothing -> send unauthorized + Just (ProducerCredential src) -> do + readLimitedBody (maxEventBytes cfg) request >>= \case + Nothing -> do + rejectEvent registry "request body missing or exceeds the configured size limit" + send (badRequest "request body missing or exceeds the configured size limit") + Just bs -> case Aeson.eitherDecode bs of + Left err -> do + rejectEvent registry (Text.pack ("invalid alarm event: " <> err)) + send (badRequest (Text.pack ("invalid alarm event: " <> err))) + Right ingressReq -> do + (ev, wasCreated) <- acceptEvent registry src ingressReq + send $ (if wasCreated then createdResponse else okResponse) $ object + [ "eventId" .= eventId ev + , "receivedAt" .= receivedAt ev + , "created" .= wasCreated + ] + +handleHistory :: AlarmRegistry -> Application +handleHistory registry request send = + case bearerToken request >>= lookupReaderCredential registry of + Nothing -> send unauthorized + Just credential + | not (rcrAllowHistory credential) -> send (forbidden "history access is not permitted for this credential") + | otherwise -> + let params = queryToQueryText (queryString request) + requested = requestedFilterFromQuery params + after = lookupParam "after" params >>= parseCursorText + limit = fromMaybe 100 (lookupParam "limit" params >>= parseIntText) + in if not (filterNarrows (rcrFilter credential) requested) + then send (forbidden "requested filter is not permitted by this credential") + else do + events <- readHistoryFiltered registry after limit requested + send $ okResponse $ object + [ "events" .= map (Aeson.toJSON . snd) events + , "nextCursor" .= case events of + [] -> Nothing + _ -> Just (cursorToText (fst (last events))) + ] + +-------------------------------------------------------------------------------- +-- Server startup +-------------------------------------------------------------------------------- + +runAlarms :: TracerEnv -> IO () +runAlarms tracerEnv = + for_ ((,) <$> alarms (teConfig tracerEnv) <*> teAlarmRegistry tracerEnv) \(cfg, registry) -> + runAlarmsServer tracerEnv cfg registry + +-- | Unlike the existing Prometheus\/EKG\/Timeseries servers -- which fall +-- back to plaintext with just a warning trace if TLS was requested but no +-- certificate is configured -- the alarm server refuses to start in that +-- situation unless 'alAllowInsecure' is explicitly set. Alarm requests +-- carry bearer tokens in the clear otherwise, and the design doc's +-- Security section calls for refusing an externally reachable clear-text +-- endpoint by default. +runAlarmsServer :: TracerEnv -> AlarmsConfig -> AlarmRegistry -> IO () +runAlarmsServer tracerEnv cfg registry = do + -- Pause to prevent collision between "Listening"-notifications from servers. + sleep 0.3 + case (wantsSSL, tlsCertificate (teConfig tracerEnv)) of + (True, Nothing) | not insecureOk -> + traceWith (teTracer tracerEnv) TracerAlarmRejected + { ttAlarmRejectedReason = + "alarms endpoint requested TLS but no certificate is configured; " <> + "refusing to start (set alAllowInsecure: true to override)" + } + (True, Nothing) -> do + traceWith (teTracer tracerEnv) TracerMissingCertificate { ttMissingCertificateEndpoint = endpoint } + startPlain + (True, Just cert) -> do + traceWith (teTracer tracerEnv) TracerStartedAlarms { ttAlarmsEndpoint = endpoint } + runTLS (tlsSettingsFor cert) settings application + (False, _) -> do + traceWith (teTracer tracerEnv) TracerStartedAlarms { ttAlarmsEndpoint = endpoint } + startPlain + where + endpoint = alEndpoint cfg + insecureOk = fromMaybe False (alAllowInsecure cfg) + wantsSSL = epForceSSL endpoint == Just True + + settings = setEndpoint endpoint defaultSettings + application = alarmsApp cfg registry + + startPlain = runSettings settings application + + tlsSettingsFor Certificate{..} = + tlsSettingsChain certificateFile (fromMaybe [] certificateChain) certificateKeyFile diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Store.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Store.hs new file mode 100644 index 00000000000..67dc7ce2187 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Store.hs @@ -0,0 +1,160 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +-- | An in-memory, append-only alarm store. Deliberately not persisted to +-- disk: the design doc (@cardano-tracer/docs/alarm-system-concept.md@) +-- explicitly leaves the storage backend as an open decision, so history is +-- lost across a @cardano-tracer@ restart in this sketch. The concrete +-- backend is meant to sit behind this small handle so a persistent +-- implementation can be swapped in later without changing callers. +module Cardano.Tracer.Handlers.Alarms.Store + ( AlarmStoreHandle + , newAlarmStore + , insertOrGetExisting + , readHistory + , pruneOnce + ) where + +import Cardano.Tracer.Configuration (AlarmsRetentionConfig (..)) +import Cardano.Tracer.Handlers.Alarms.Types +import Cardano.Tracer.Time (getTimeMs) + +import Control.Concurrent (threadDelay) +import Control.Concurrent.Async (async, link, race_) +import Control.Concurrent.MVar (MVar, newEmptyMVar, takeMVar) +import Control.Concurrent.STM (atomically) +import Control.Concurrent.STM.TVar (TVar, modifyTVar', newTVarIO, readTVar, readTVarIO, + stateTVar, writeTVar) +import Control.Monad (forever) +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Time.Clock (UTCTime, diffUTCTime, getCurrentTime) +import Data.Word (Word64) + +data AlarmStoreHandle = AlarmStoreHandle + { ashEvents :: !(TVar (Map AlarmCursor AlarmEvent)) + , ashIndex :: !(TVar (Map (AlarmSource, Text) AlarmCursor)) -- ^ (source, sourceEventId) -> cursor, for idempotency + , ashNextSeq :: !(TVar Word64) + , ashStartTag :: !Text -- ^ process-start timestamp (ms); prefixes generated eventIds + , ashRetention :: !AlarmsRetentionConfig + , ashWake :: !(MVar ()) -- ^ unused for now (no dynamic reconfiguration in this sketch); kept so the pruner loop matches Cardano.Timeseries.Component.create's shape + } + +-- | Prune every 60 seconds when any retention limit is configured; otherwise +-- the pruner thread just parks, mirroring +-- @Cardano.Timeseries.Component.create@'s pruner. +pruneIntervalMicros :: Int +pruneIntervalMicros = 60 * 1000 * 1000 + +newAlarmStore :: AlarmsRetentionConfig -> IO AlarmStoreHandle +newAlarmStore retention = do + events <- newTVarIO Map.empty + index <- newTVarIO Map.empty + nextSeq <- newTVarIO 0 + wake <- newEmptyMVar + startTag <- Text.pack . show <$> getTimeMs + let handle = AlarmStoreHandle events index nextSeq startTag retention wake + async (runPruner handle) >>= link + pure handle + where + runPruner :: AlarmStoreHandle -> IO () + runPruner handle = forever $ + case (arcMaxAgeSeconds (ashRetention handle), arcMaxEvents (ashRetention handle)) of + (Nothing, Nothing) -> takeMVar (ashWake handle) -- no retention configured; nothing to do, ever + _ -> do + now <- getCurrentTime + pruneOnce handle now + race_ (threadDelay pruneIntervalMicros) (takeMVar (ashWake handle)) + +-- | Insert a new event, or return the already-accepted event for a replayed +-- @(source, sourceEventId)@ pair without dispatching it again. One STM +-- transaction, so two concurrent submissions of the same key can never +-- both "win". +insertOrGetExisting + :: AlarmStoreHandle + -> AlarmSource + -> UTCTime -- ^ receivedAt + -> IngressRequest + -> IO (AlarmCursor, AlarmEvent, Bool) -- ^ (cursor, event, was newly created) +insertOrGetExisting handle src receivedAt req = + atomically do + idx <- readTVar (ashIndex handle) + case Map.lookup (src, irSourceEventId req) idx of + Just cursor -> do + evs <- readTVar (ashEvents handle) + case Map.lookup cursor evs of + Just ev -> pure (cursor, ev, False) + -- Indexed but pruned already: treat as a fresh submission. This + -- can only happen if pruning ever removed an event without also + -- removing its index entry -- 'pruneOnce' takes care to avoid + -- that, but resolving to a fresh insert here is a safe fallback + -- either way. + Nothing -> insertNew + Nothing -> insertNew + where + insertNew = do + seqNum <- stateTVar (ashNextSeq handle) (\n -> (n, n + 1)) + let cursor = AlarmCursor seqNum + eid = ashStartTag handle <> "-" <> Text.pack (show seqNum) + ev = AlarmEvent + { schemaVersion = 1 + , eventId = eid + , sourceEventId = irSourceEventId req + , raisedAt = irRaisedAt req + , receivedAt = receivedAt + , source = src + , ruleId = irRuleId req + , severity = irSeverity req + , summary = irSummary req + , scope = irScope req + , labels = irLabels req + , details = irDetails req + } + modifyTVar' (ashEvents handle) (Map.insert cursor ev) + modifyTVar' (ashIndex handle) (Map.insert (src, irSourceEventId req) cursor) + pure (cursor, ev, True) + +-- | Cursor-ordered, filtered history read, strictly after the given cursor +-- (exclusive), capped at @limit@ results. +readHistory :: AlarmStoreHandle -> Maybe AlarmCursor -> Int -> AlarmFilter -> IO [(AlarmCursor, AlarmEvent)] +readHistory handle after limit filt = do + evs <- readTVarIO (ashEvents handle) + let afterOk cursor = maybe True (cursor >) after + candidates = [ (c, ev) | (c, ev) <- Map.toAscList evs, afterOk c, matchesFilter filt ev ] + pure (take (max 0 limit) candidates) + +-- | Apply the configured age\/count retention limits. Removing an event from +-- 'ashEvents' always removes its @(source, sourceEventId)@ entry from +-- 'ashIndex' in the /same/ transaction -- if these two maps were ever +-- allowed to drift, a resubmission of a since-pruned key would return a +-- cursor that no longer resolves in 'readHistory'. +pruneOnce :: AlarmStoreHandle -> UTCTime -> IO () +pruneOnce handle now = atomically do + evs <- readTVar (ashEvents handle) + let kept = applyCount (applyAge evs) + dropped = Map.difference evs kept + writeTVar (ashEvents handle) kept + modifyTVar' (ashIndex handle) (removeDropped dropped) + where + retention = ashRetention handle + + applyAge :: Map AlarmCursor AlarmEvent -> Map AlarmCursor AlarmEvent + applyAge evs = case arcMaxAgeSeconds retention of + Nothing -> evs + Just maxAgeSecs -> Map.filter (\ev -> diffUTCTime now (receivedAt ev) <= fromIntegral maxAgeSecs) evs + + applyCount :: Map AlarmCursor AlarmEvent -> Map AlarmCursor AlarmEvent + applyCount evs = case arcMaxEvents retention of + Nothing -> evs + Just maxN -> Map.fromDistinctDescList (take (fromIntegral maxN) (Map.toDescList evs)) + + removeDropped + :: Map AlarmCursor AlarmEvent + -> Map (AlarmSource, Text) AlarmCursor + -> Map (AlarmSource, Text) AlarmCursor + removeDropped dropped idx = foldr Map.delete idx keysToRemove + where + keysToRemove = [ (source ev, sourceEventId ev) | ev <- Map.elems dropped ] diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TraceRules.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TraceRules.hs new file mode 100644 index 00000000000..544b715f0e0 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TraceRules.hs @@ -0,0 +1,85 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} + +-- | The trace-severity alarm producer: raises an alarm for every received +-- trace message whose severity is at or above a configured threshold. Like +-- "Cardano.Tracer.Handlers.Alarms.Consumers", this module holds the rule +-- type, its config translation, and the pure matching logic; the IO driving +-- lives in "Cardano.Tracer.Handlers.Alarms.Registry". +module Cardano.Tracer.Handlers.Alarms.TraceRules + ( TraceAlarmRule (..) + , traceRuleFromConfig + , traceAlarmSource + , traceRuleRequest + ) where + +import Cardano.Logging.Types (SeverityS, TraceObject (..)) +import Cardano.Tracer.Configuration (AlarmsTraceRuleConfig (..), alarmSeverityToText) +import Cardano.Tracer.Handlers.Alarms.Types + +import Data.Aeson (object, (.=)) +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Maybe (fromMaybe) +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) +import Data.Word (Word64) + +data TraceAlarmRule = TraceAlarmRule + { tarRuleId :: !RuleId + , tarSummary :: !Text + , tarThreshold :: !SeverityS + , tarSuppressSecs :: !Word64 + , tarLabels :: !(Map Text Text) + } + +traceRuleFromConfig :: AlarmsTraceRuleConfig -> TraceAlarmRule +traceRuleFromConfig AlarmsTraceRuleConfig{..} = TraceAlarmRule + { tarRuleId = RuleId atrRuleId + , tarSummary = fromMaybe defaultSummary atrSummary + , tarThreshold = atrThreshold + , tarSuppressSecs = max 1 (fromMaybe 300 atrSuppressForSecs) + , tarLabels = fromMaybe Map.empty atrLabels + } + where + defaultSummary = + "Trace message with severity at or above " + <> alarmSeverityToText atrThreshold <> " received" + +-- | The fixed trusted source identity for alarms raised by trace severity +-- rules. Internal producers never pass HTTP authentication, so the source +-- is a constant here, mirroring how external producers get their source +-- from their credential rather than choosing it themselves. +traceAlarmSource :: AlarmSource +traceAlarmSource = AlarmSource "trace" + +-- | Pure mapping from one received trace message to an alarm submission, or +-- 'Nothing' when the message is below the rule's threshold. The +-- @sourceEventId@ encodes a time window derived from the message's own +-- timestamp, so all matches from the same node and namespace within one +-- window share the store's idempotency key and collapse into one alarm -- +-- flood prevention without any extra state. +traceRuleRequest :: TraceAlarmRule -> Text -> TraceObject -> Maybe IngressRequest +traceRuleRequest TraceAlarmRule{..} nodeName trObj + | toSeverity trObj < tarThreshold = Nothing + | otherwise = Just IngressRequest + { irSourceEventId = Text.intercalate ":" + [unRuleId tarRuleId, nodeName, namespace, Text.pack (show windowIndex)] + , irRaisedAt = toTimestamp trObj + , irRuleId = tarRuleId + , irSeverity = toSeverity trObj + , irSummary = tarSummary + , irScope = Map.singleton "nodeId" nodeName + , irLabels = Map.insert "namespace" namespace tarLabels + , irDetails = Just $ object + [ "message" .= toMachine trObj + , "hostname" .= toHostname trObj + , "threadId" .= toThreadId trObj + ] + } + where + namespace = Text.intercalate "." (toNamespace trObj) + + windowIndex :: Word64 + windowIndex = floor (utcTimeToPOSIXSeconds (toTimestamp trObj)) `div` tarSuppressSecs diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Types.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Types.hs new file mode 100644 index 00000000000..7c069b44203 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Types.hs @@ -0,0 +1,204 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} + +-- | Pure domain types for the alarm subsystem (see +-- @cardano-tracer/docs/alarm-system-concept.md@). Deliberately depends only +-- on "Cardano.Tracer.Configuration"/"Cardano.Logging" -- never on +-- "Cardano.Tracer.Environment" -- mirroring how @Cardano.Timeseries.Component@ +-- is referenced by @Environment.hs@ but never references it back. +module Cardano.Tracer.Handlers.Alarms.Types + ( AlarmSource (..) + , RuleId (..) + , AlarmCursor (..) + , AlarmId + , IngressRequest (..) + , AlarmEvent (..) + , AlarmFilter (..) + , emptyAlarmFilter + , filterFromConfig + , matchesFilter + , filterNarrows + ) where + +import Cardano.Logging.Types (SeverityS) +import Cardano.Tracer.Configuration (AlarmFilterConfig (..), alarmSeverityToText, + parseAlarmSeverityText) + +import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, (.:), (.:?), (.=)) +import qualified Data.Aeson as Aeson +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Maybe (fromMaybe) +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Time.Clock (UTCTime) +import Data.Word (Word64) + +-- | The trusted identity of an alarm producer, derived from the producer +-- credential that authenticated the ingress request. Never taken from the +-- request body itself. +newtype AlarmSource = AlarmSource { unAlarmSource :: Text } + deriving stock (Eq, Ord, Show) + deriving newtype (FromJSON, ToJSON) + +newtype RuleId = RuleId { unRuleId :: Text } + deriving stock (Eq, Ord, Show) + deriving newtype (FromJSON, ToJSON) + +-- | A monotonic sequence number assigned by the store on insertion. Serves +-- as the pagination cursor for history reads. +newtype AlarmCursor = AlarmCursor Word64 + deriving stock (Eq, Ord, Show) + deriving newtype (FromJSON, ToJSON, Enum, Num) + +-- | The server-assigned event identifier included in the wire envelope. +type AlarmId = Text + +-- | The producer-submitted payload, once JSON-parsed. Only ever reads +-- producer-owned fields: @source@, @eventId@, and @receivedAt@ in the +-- design doc's envelope are assigned by the server, so a caller-supplied +-- value for any of them in the request body is simply not read here. +data IngressRequest = IngressRequest + { irSourceEventId :: !Text + , irRaisedAt :: !UTCTime + , irRuleId :: !RuleId + , irSeverity :: !SeverityS + , irSummary :: !Text + , irScope :: !(Map Text Text) + , irLabels :: !(Map Text Text) + , irDetails :: !(Maybe Aeson.Value) + } + deriving stock (Eq, Show) + +instance FromJSON IngressRequest where + parseJSON = withObject "IngressRequest" \o -> do + irSourceEventId <- o .: "sourceEventId" + irRaisedAt <- o .: "raisedAt" + irRuleId <- RuleId <$> o .: "ruleId" + severityText <- o .: "severity" + irSeverity <- parseSeverityOrFail severityText + irSummary <- o .: "summary" + irScope <- fromMaybe Map.empty <$> o .:? "scope" + irLabels <- fromMaybe Map.empty <$> o .:? "labels" + irDetails <- o .:? "details" + pure IngressRequest{..} + where + parseSeverityOrFail t = + maybe (fail ("unknown severity: " <> Text.unpack t)) pure (parseAlarmSeverityText t) + +-- | The immutable, versioned alarm envelope. @eventId@\/@receivedAt@\/@source@ +-- are assigned by the server; everything else is copied from the +-- producer's 'IngressRequest'. +data AlarmEvent = AlarmEvent + { schemaVersion :: !Int + , eventId :: !AlarmId + , sourceEventId :: !Text + , raisedAt :: !UTCTime + , receivedAt :: !UTCTime + , source :: !AlarmSource + , ruleId :: !RuleId + , severity :: !SeverityS + , summary :: !Text + , scope :: !(Map Text Text) + , labels :: !(Map Text Text) + , details :: !(Maybe Aeson.Value) + } + deriving stock (Eq, Show) + +instance ToJSON AlarmEvent where + toJSON AlarmEvent{..} = object + [ "schemaVersion" .= schemaVersion + , "eventId" .= eventId + , "sourceEventId" .= sourceEventId + , "raisedAt" .= raisedAt + , "receivedAt" .= receivedAt + , "source" .= unAlarmSource source + , "ruleId" .= unRuleId ruleId + , "severity" .= alarmSeverityToText severity + , "summary" .= summary + , "scope" .= scope + , "labels" .= labels + , "details" .= details + ] + +instance FromJSON AlarmEvent where + parseJSON = withObject "AlarmEvent" \o -> do + schemaVersion <- o .: "schemaVersion" + eventId <- o .: "eventId" + sourceEventId <- o .: "sourceEventId" + raisedAt <- o .: "raisedAt" + receivedAt <- o .: "receivedAt" + source <- AlarmSource <$> o .: "source" + ruleId <- RuleId <$> o .: "ruleId" + severityText <- o .: "severity" + severity <- parseSeverityOrFail severityText + summary <- o .: "summary" + scope <- fromMaybe Map.empty <$> o .:? "scope" + labels <- fromMaybe Map.empty <$> o .:? "labels" + details <- o .:? "details" + pure AlarmEvent{..} + where + parseSeverityOrFail t = + maybe (fail ("unknown severity: " <> Text.unpack t)) pure (parseAlarmSeverityText t) + +-- | A conjunctive selection over 'AlarmEvent's, used both for a static +-- consumer's configuration and for a reader credential's allowed +-- (ceiling) or requested filter. Scope\/label matching requires +-- the event to carry every key\/value pair listed here (a submap check), +-- not just an intersection. +data AlarmFilter = AlarmFilter + { afSource :: !(Maybe AlarmSource) + , afRuleId :: !(Maybe RuleId) + , afMinSeverity :: !(Maybe SeverityS) + , afScope :: !(Map Text Text) + , afLabels :: !(Map Text Text) + } + deriving stock (Eq, Show) + +emptyAlarmFilter :: AlarmFilter +emptyAlarmFilter = AlarmFilter Nothing Nothing Nothing Map.empty Map.empty + +filterFromConfig :: AlarmFilterConfig -> AlarmFilter +filterFromConfig AlarmFilterConfig{..} = AlarmFilter + { afSource = AlarmSource <$> afcSource + , afRuleId = RuleId <$> afcRuleId + , afMinSeverity = afcMinSeverity + , afScope = fromMaybe Map.empty afcScope + , afLabels = fromMaybe Map.empty afcLabels + } + +isSubmapOf :: (Ord k, Eq v) => Map k v -> Map k v -> Bool +isSubmapOf = Map.isSubmapOfBy (==) + +matchesFilter :: AlarmFilter -> AlarmEvent -> Bool +matchesFilter AlarmFilter{..} ev = + maybe True (== source ev) afSource + && maybe True (== ruleId ev) afRuleId + && maybe True (<= severity ev) afMinSeverity + && afScope `isSubmapOf` scope ev + && afLabels `isSubmapOf` labels ev + +-- | Is @requested@ at least as strict as @ceiling@ -- i.e. can it only narrow, +-- never broaden, what the credential allows? Used to reject a reader's +-- requested history filter that would see more than their credential +-- permits. +filterNarrows :: AlarmFilter -> AlarmFilter -> Bool +filterNarrows ceilingFilter requestedFilter = + narrowsEq (afSource ceilingFilter) (afSource requestedFilter) + && narrowsEq (afRuleId ceilingFilter) (afRuleId requestedFilter) + && narrowsSeverity (afMinSeverity ceilingFilter) (afMinSeverity requestedFilter) + && afScope ceilingFilter `isSubmapOf` afScope requestedFilter + && afLabels ceilingFilter `isSubmapOf` afLabels requestedFilter + where + -- A single-value equality constraint can only be kept as-is, never + -- dropped or changed, once the ceiling fixes it. + narrowsEq :: Eq a => Maybe a -> Maybe a -> Bool + narrowsEq Nothing _ = True + narrowsEq (Just c) (Just r) = c == r + narrowsEq (Just _) Nothing = False + + narrowsSeverity :: Maybe SeverityS -> Maybe SeverityS -> Bool + narrowsSeverity Nothing _ = True + narrowsSeverity (Just c) (Just r) = r >= c + narrowsSeverity (Just _) Nothing = False diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/TraceObjects.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/TraceObjects.hs index 003c938e6b8..646e000044d 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/TraceObjects.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/TraceObjects.hs @@ -10,12 +10,14 @@ module Cardano.Tracer.Handlers.Logs.TraceObjects import Cardano.Logging (TraceObject) import Cardano.Tracer.Configuration import Cardano.Tracer.Environment +import Cardano.Tracer.Handlers.Alarms.Registry (checkTraceObjectsForAlarms) import Cardano.Tracer.Handlers.Logs.File import Cardano.Tracer.Handlers.Logs.Journal import Cardano.Tracer.Types import Cardano.Tracer.Utils import Control.Concurrent.Async (forConcurrently_) +import Data.Foldable (for_) import qualified Data.Map as Map import System.IO (Handle, hClose) @@ -37,6 +39,8 @@ traceObjectsHandler tracerEnv nodeId traceObjects = do loggingParams nodeName teCurrentLogLock traceObjects JournalMode -> writeTraceObjectsToJournal logFormat nodeName traceObjects + for_ teAlarmRegistry \alarmRegistry -> + checkTraceObjectsForAlarms alarmRegistry nodeName traceObjects teReforwardTraceObjects traceObjects where TracerEnv { teConfig = TracerConfig{ logging, verbosity } @@ -44,6 +48,7 @@ traceObjectsHandler tracerEnv nodeId traceObjects = do , teReforwardTraceObjects , teRegistry , teTracer + , teAlarmRegistry } = tracerEnv deregisterNodeId :: TracerEnv -> NodeId -> IO () diff --git a/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs b/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs index 151e5e365a9..fe014d9198f 100644 --- a/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs +++ b/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs @@ -84,6 +84,24 @@ data TracerTrace { ttConnection :: HowToConnect , ttMessage :: String } + | TracerStartedAlarms + { ttAlarmsEndpoint :: Endpoint + } + | TracerAlarmAccepted + { ttAlarmAcceptedSource :: Text + , ttAlarmAcceptedRuleId :: Text + , ttAlarmAcceptedSeverity :: SeverityS + } + | TracerAlarmDuplicate + { ttAlarmDuplicateSource :: Text + , ttAlarmDuplicateSourceEventId :: Text + } + | TracerAlarmRejected + { ttAlarmRejectedReason :: Text + } + | TracerAlarmDispatched + { ttAlarmDispatchedConsumer :: Text + } deriving Show -- | A bundle of domain-split tracers used in the application. @@ -184,6 +202,29 @@ instance LogFormatting TracerTrace where , "conn" .= ttConnection , "message" .= ttMessage ] + TracerStartedAlarms{..} -> mconcat + [ "kind" .= AE.String "TracerStartedAlarms" + , "endpoint" .= ttAlarmsEndpoint + ] + TracerAlarmAccepted{..} -> mconcat + [ "kind" .= AE.String "TracerAlarmAccepted" + , "source" .= ttAlarmAcceptedSource + , "ruleId" .= ttAlarmAcceptedRuleId + , "severity" .= ttAlarmAcceptedSeverity + ] + TracerAlarmDuplicate{..} -> mconcat + [ "kind" .= AE.String "TracerAlarmDuplicate" + , "source" .= ttAlarmDuplicateSource + , "sourceEventId" .= ttAlarmDuplicateSourceEventId + ] + TracerAlarmRejected{..} -> mconcat + [ "kind" .= AE.String "TracerAlarmRejected" + , "reason" .= ttAlarmRejectedReason + ] + TracerAlarmDispatched{..} -> mconcat + [ "kind" .= AE.String "TracerAlarmDispatched" + , "consumer" .= ttAlarmDispatchedConsumer + ] instance MetaTrace TracerTrace where namespaceFor TracerBuildInfo {} = Namespace [] ["BuildInfo"] @@ -208,6 +249,11 @@ instance MetaTrace TracerTrace where namespaceFor TracerError {} = Namespace [] ["Error"] namespaceFor TracerResource {} = Namespace [] ["Resources"] namespaceFor TracerForwardingInterrupted {} = Namespace [] ["ForwardingInterrupted"] + namespaceFor TracerStartedAlarms {} = Namespace [] ["StartedAlarms"] + namespaceFor TracerAlarmAccepted {} = Namespace [] ["AlarmAccepted"] + namespaceFor TracerAlarmDuplicate {} = Namespace [] ["AlarmDuplicate"] + namespaceFor TracerAlarmRejected {} = Namespace [] ["AlarmRejected"] + namespaceFor TracerAlarmDispatched {} = Namespace [] ["AlarmDispatched"] severityFor (Namespace _ ["BuildInfo"]) _ = Just Info severityFor (Namespace _ ["ParamsAre"]) _ = Just Warning @@ -231,6 +277,11 @@ instance MetaTrace TracerTrace where severityFor (Namespace _ ["Error"]) _ = Just Error severityFor (Namespace _ ["Resources"]) _ = Just Info severityFor (Namespace _ ["ForwardingInterrupted"]) _ = Just Warning + severityFor (Namespace _ ["StartedAlarms"]) _ = Just Info + severityFor (Namespace _ ["AlarmAccepted"]) _ = Just Info + severityFor (Namespace _ ["AlarmDuplicate"]) _ = Just Debug + severityFor (Namespace _ ["AlarmRejected"]) _ = Just Warning + severityFor (Namespace _ ["AlarmDispatched"]) _ = Just Debug severityFor _ _ = Nothing documentFor _ = Just "" @@ -258,6 +309,11 @@ instance MetaTrace TracerTrace where , Namespace [] ["Error"] , Namespace [] ["Resources"] , Namespace [] ["ForwardingInterrupted"] + , Namespace [] ["StartedAlarms"] + , Namespace [] ["AlarmAccepted"] + , Namespace [] ["AlarmDuplicate"] + , Namespace [] ["AlarmRejected"] + , Namespace [] ["AlarmDispatched"] ] stderrShowTracer :: Show a => Trace IO a diff --git a/cardano-tracer/src/Cardano/Tracer/Run.hs b/cardano-tracer/src/Cardano/Tracer/Run.hs index bd605acec7e..db76810ebbe 100644 --- a/cardano-tracer/src/Cardano/Tracer/Run.hs +++ b/cardano-tracer/src/Cardano/Tracer/Run.hs @@ -16,6 +16,8 @@ import Cardano.Tracer.Acceptors.Run import Cardano.Tracer.CLI import Cardano.Tracer.Configuration import Cardano.Tracer.Environment +import qualified Cardano.Tracer.Handlers.Alarms.Registry as Alarms +import Cardano.Tracer.Handlers.Alarms.Server (runAlarms) import Cardano.Tracer.Handlers.Logs.Rotator import Cardano.Tracer.Handlers.Metrics.Servers import Cardano.Tracer.Handlers.ReForwarder @@ -92,6 +94,7 @@ doRunCardanoTracer config stateDir tr protocolsBrake dpRequestors = do registry <- newRegistry !timeseriesHandle <- for (hasTimeseries config) (const $ Timeseries.create @(Tree _) tr.timeseries Nothing) + !alarmRegistry <- for (alarms config) (Alarms.newAlarmRegistry tr.assorted) -- Environment for all following functions. let tracerEnv :: TracerEnv @@ -110,6 +113,7 @@ doRunCardanoTracer config stateDir tr protocolsBrake dpRequestors = do , teStateDir = stateDir , teMetricsHelp = mHelp , teTimeseriesHandle = timeseriesHandle + , teAlarmRegistry = alarmRegistry } -- Specify what should be done before 'cardano-tracer' stops. @@ -123,6 +127,7 @@ doRunCardanoTracer config stateDir tr protocolsBrake dpRequestors = do [ runLogsRotator tracerEnv , runMetricsServers tracerEnv , runAcceptors tracerEnv + , runAlarms tracerEnv ] -- NB. this fails silently if there's any read or decode error when an external JSON file is provided diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Acceptor.hs b/cardano-tracer/test/Cardano/Tracer/Test/Acceptor.hs index 6b4907730c3..5d74a6c2e41 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Acceptor.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Acceptor.hs @@ -63,6 +63,7 @@ launchAcceptorsSimple mode localSock dpName = do , teStateDir = Nothing , teMetricsHelp = [] , teTimeseriesHandle = Nothing + , teAlarmRegistry = Nothing } -- NOTE: no reforwarding in this acceptor. @@ -81,6 +82,7 @@ launchAcceptorsSimple mode localSock dpName = do , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , tlsCertificate = Nothing , logging = NE.fromList [LoggingParams "/tmp/demo-acceptor" FileMode ForHuman] , rotation = Nothing diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs new file mode 100644 index 00000000000..553d8d428b6 --- /dev/null +++ b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs @@ -0,0 +1,156 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +module Cardano.Tracer.Test.Alarms.Tests + ( tests + ) where + +import Cardano.Logging (DetailLevel (..), SeverityF (..), SeverityS (..), + TraceObject (..)) +import Cardano.Tracer.Configuration +import Cardano.Tracer.Handlers.Alarms.Registry +import Cardano.Tracer.Handlers.Alarms.Store +import Cardano.Tracer.Handlers.Alarms.Types +import Cardano.Tracer.MetaTrace (TraceBundle (..), mkTraceBundle) + +import Control.Concurrent.Async (forConcurrently) +import qualified Data.Map.Strict as Map +import qualified Data.Text as Text +import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime) + +import Test.Tasty +import Test.Tasty.QuickCheck + +tests :: TestTree +tests = localOption (QuickCheckTests 1) $ testGroup "Test.Alarms" + [ testProperty "AlarmEvent JSON round-trip uses lowercase severity" propSeverityLowercase + , testProperty "idempotent double-submit returns the same eventId" propIdempotentSubmit + , testProperty "concurrent double-submit creates exactly one event" propConcurrentSubmit + , testProperty "prune then resubmit issues a fresh eventId" propPruneThenResubmit + , testProperty "trace rule fires only at or above its threshold" propTraceRuleThreshold + , testProperty "trace rule suppresses repeats within one window" propTraceRuleSuppression + , testProperty "trace rule fires again in a new window" propTraceRuleNewWindow + ] + +mkRequest :: Text.Text -> SeverityS -> IO IngressRequest +mkRequest sourceEventId sev = do + now <- getCurrentTime + pure IngressRequest + { irSourceEventId = sourceEventId + , irRaisedAt = now + , irRuleId = RuleId "test-rule" + , irSeverity = sev + , irSummary = "test summary" + , irScope = Map.empty + , irLabels = Map.empty + , irDetails = Nothing + } + +testSource :: AlarmSource +testSource = AlarmSource "hermod-recon" + +-- | One trace rule: alarm on severity >= Error, at most one alarm per +-- (node, namespace) per 60-second window. +testAlarmsConfig :: AlarmsConfig +testAlarmsConfig = AlarmsConfig + { alEndpoint = Endpoint "127.0.0.1" 0 Nothing + , alAllowInsecure = Just True + , alRetention = Nothing + , alLimits = Nothing + , alAuthentication = AlarmsAuthConfig [] [] + , alConsumers = [] + , alTraceRules = Just + [ AlarmsTraceRuleConfig + { atrRuleId = "error-traces" + , atrSummary = Nothing + , atrThreshold = Error + , atrSuppressForSecs = Just 60 + , atrLabels = Nothing + } + ] + } + +newTestRegistry :: IO AlarmRegistry +newTestRegistry = do + bundle <- mkTraceBundle (SeverityF (Just Warning)) + newAlarmRegistry (assorted bundle) testAlarmsConfig + +mkTraceObject :: SeverityS -> UTCTime -> TraceObject +mkTraceObject sev at = TraceObject + { toHuman = Nothing + , toMachine = "{\"msg\":\"test\"}" + , toNamespace = ["Test", "Alarm"] + , toSeverity = sev + , toDetails = DNormal + , toTimestamp = at + , toHostname = "localhost" + , toThreadId = "1" + } + +propSeverityLowercase :: Property +propSeverityLowercase = once $ ioProperty do + pure (alarmSeverityToText Critical === "critical") + +propIdempotentSubmit :: Property +propIdempotentSubmit = once $ ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig Nothing Nothing) + req <- mkRequest "evt-1" Warning + now <- getCurrentTime + (_, ev1, created1) <- insertOrGetExisting store testSource now req + (_, ev2, created2) <- insertOrGetExisting store testSource now req + pure (created1 .&&. not created2 .&&. eventId ev1 === eventId ev2) + +propConcurrentSubmit :: Property +propConcurrentSubmit = once $ ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig Nothing Nothing) + req <- mkRequest "evt-concurrent" Warning + now <- getCurrentTime + results <- forConcurrently [1 .. (20 :: Int)] \_ -> + insertOrGetExisting store testSource now req + let createdCount = length (filter (\(_, _, c) -> c) results) + pure (createdCount === 1) + +propPruneThenResubmit :: Property +propPruneThenResubmit = once $ ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig (Just 0) Nothing) -- prune anything immediately + req <- mkRequest "evt-pruned" Warning + now <- getCurrentTime + (_, ev1, _) <- insertOrGetExisting store testSource now req + later <- getCurrentTime + pruneOnce store later + (_, ev2, created2) <- insertOrGetExisting store testSource later req + pure (created2 .&&. eventId ev1 =/= eventId ev2) + +propTraceRuleThreshold :: Property +propTraceRuleThreshold = once $ ioProperty do + registry <- newTestRegistry + now <- getCurrentTime + checkTraceObjectsForAlarms registry "node-1" + [mkTraceObject Info now, mkTraceObject Error now] + events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter + pure $ case events of + [(_, ev)] -> severity ev === Error .&&. source ev === AlarmSource "trace" + _ -> counterexample ("expected exactly one event, got " <> show (length events)) False + +propTraceRuleSuppression :: Property +propTraceRuleSuppression = once $ ioProperty do + registry <- newTestRegistry + now <- getCurrentTime + -- Same node, namespace, and timestamp: both matches land in the same + -- suppression window, whatever their severities are. + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error now] + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Critical now] + events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter + pure (length events === 1) + +propTraceRuleNewWindow :: Property +propTraceRuleNewWindow = once $ ioProperty do + registry <- newTestRegistry + now <- getCurrentTime + -- 120 s > the rule's 60 s window, so the window index always differs, + -- regardless of where 'now' falls within its window. + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error now] + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error (addUTCTime 120 now)] + events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter + pure (length events === 2) diff --git a/cardano-tracer/test/Cardano/Tracer/Test/DataPoint/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/DataPoint/Tests.hs index 017d1ff43ff..874de8e2f7a 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/DataPoint/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/DataPoint/Tests.hs @@ -89,6 +89,7 @@ propDataPoint ts@TestSetup{..} rootDir localSock = do , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , tlsCertificate = Nothing , logging = NE.fromList [LoggingParams rootDir FileMode ForHuman] , rotation = Nothing diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Logs/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Logs/Tests.hs index 02ea981e441..ff3948a2994 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Logs/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Logs/Tests.hs @@ -71,6 +71,7 @@ propLogs ts@TestSetup{..} format logRotLimitBytes logRotMaxAgeMinutes rootDir lo , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , logging = LoggingParams root FileMode format :| [] , rotation = Just $ RotationParams { rpFrequencySecs = 3 @@ -119,6 +120,7 @@ propMultiInit ts@TestSetup{..} format rootDir howToConnect1 howToConnect2 = do , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , tlsCertificate = Nothing , logging = LoggingParams rootDir FileMode format :| [] , rotation = Nothing @@ -165,6 +167,7 @@ propMultiResp ts@TestSetup{..} format rootDir howToConnect = do , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , tlsCertificate = Nothing , logging = LoggingParams rootDir FileMode format :| [] , rotation = Nothing diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Restart/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Restart/Tests.hs index f9969008ca4..0d8d54e9e68 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Restart/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Restart/Tests.hs @@ -94,6 +94,7 @@ mkConfig TestSetup{..} rootDir p = TracerConfig , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , tlsCertificate = Nothing , logging = NE.fromList [LoggingParams rootDir FileMode ForMachine] , rotation = Nothing diff --git a/cardano-tracer/test/cardano-tracer-test.hs b/cardano-tracer/test/cardano-tracer-test.hs index c862a6770d5..ef1cd63e354 100644 --- a/cardano-tracer/test/cardano-tracer-test.hs +++ b/cardano-tracer/test/cardano-tracer-test.hs @@ -1,6 +1,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# OPTIONS_GHC -Wno-unused-matches #-} +import qualified Cardano.Tracer.Test.Alarms.Tests as Alarms import qualified Cardano.Tracer.Test.DataPoint.Tests as DataPoint import qualified Cardano.Tracer.Test.Logs.Tests as Logs import Cardano.Tracer.Test.TestSetup @@ -47,6 +48,7 @@ main = do (testGroup "Tests" [ Logs.tests ts , DataPoint.tests ts + , Alarms.tests -- , Restart.tests ts -- , Queue.tests ts ]) From 1605e8a18534e90b3ccf3c3c41afcad41435475f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Nicklisch-Franken?= Date: Tue, 28 Jul 2026 14:51:38 +0200 Subject: [PATCH 2/8] Additional plans for testing of time series and porting queries --- .../grafana-alerts-as-timeseries-rules.md | 270 ++++++++++++++++++ .../docs/timeseries-alarm-testing.md | 159 +++++++++++ 2 files changed, 429 insertions(+) create mode 100644 cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md create mode 100644 cardano-tracer/docs/timeseries-alarm-testing.md diff --git a/cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md b/cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md new file mode 100644 index 00000000000..7c83e47737a --- /dev/null +++ b/cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md @@ -0,0 +1,270 @@ +# Porting the cardano-parts Grafana alerts to timeseries alarm rules + +Status: guide (companion to `alarm-system-concept.md`) + +The SRE alert catalogue in +[cardano-parts](https://github.com/input-output-hk/cardano-parts/tree/main/templates/cardano-parts-project/flake/opentofu/grafana/alerts) +defines Grafana-managed Prometheus alert rules for Cardano deployments. This +document shows how the `cardano-node` alerts from that catalogue map onto +timeseries rules in the new alarm system (Phase 2 of the concept doc) — what +translates directly, what changes, and what does not carry over. + +The comparison is against the four node-related files: +`cardano-node.nix-import`, `cardano-node-forge.nix-import`, +`cardano-node-network.nix-import`, `cardano-node-quality.nix-import`, and +`cardano-node-divergence.nix-import`. + +## Rule anatomy: Grafana vs. alarm system + +| Grafana / Prometheus rule | Alarm system `timeseriesRules` entry | +| --- | --- | +| `alert` (name) | `ruleId` | +| `expr` (PromQL) | `query` (`cardano-timeseries-io` language) | +| `for` | `for` | +| `labels.severity = "page"` | `severity:` — richer vocabulary; use `critical` for pages, lower levels become possible | +| `annotations.summary/description` | `summary` (static text; no `{{$labels.instance}}` templating) | +| node identity via `instance` label | series key labels (`node_name`) carried into the alarm's labels | +| evaluation interval (Grafana global) | `evaluateEvery` per rule | + +The alarm state machine matches Grafana's semantics: edge-triggered +publication after the expression has been true for the `for` duration, no +resolution events, optional `repeatEvery` reminders. + +## Query language correspondence + +The `cardano-timeseries-io` language covers almost every PromQL construct the +catalogue uses: + +| PromQL | timeseries query language | +| --- | --- | +| `m[5m]` (range) | `m[now - 5m; now]` | +| `m[360m:1m]` (subquery w/ step) | `m[now - 360m; now : 1m]` | +| `rate(...)`, `increase(...)` | `rate (...)`, `increase (...)` | +| `avg_over_time`, `sum_over_time` | same names | +| `quantile_over_time(0.95, v)` | `quantile_over_time 0.95 v` | +| `quantile by(environment) (0.2, v)` | `quantile_by ("environment") 0.2 v` | +| `min`/`max`/`avg`/`abs`/`round` | same names | +| `A unless B` | `unless A B` | +| `m{environment="mainnet"}` | `m{environment = "mainnet"}` (`=` and `!=` only) | +| `> bool`, `== bool` | plain `>`/`==` — comparisons are typed booleans already | +| `and` / `or` (scalar logic) | `&&` / `||` | +| — | extras: `let`/lambdas, `map`, `filter`, `join`, `to_scalar`, `earliest`, `latest`, `metrics` | + +Missing relative to PromQL: regex label matchers (`=~`, `!~`), vector-`or` as +union/fallback, and cross-series binary matching (`on() group_right()`). +Workarounds below. + +## The catalogue, translated + +Severity suggestions replace the one-size-fits-all `page`. Metric names are +the new-tracing names as forwarded by the node — verify against a live store +with the `metrics` query before deploying (see caveats). + +```yaml +alarms: + timeseriesRules: + # --- cardano-node.nix-import ------------------------------------------- + - ruleId: blockheight-unchanged # cardano_node_blockheight_unchanged + summary: "Blockheight unchanged for more than 7 minutes" + severity: critical + query: "rate (cardano_node_metrics_blockNum_int[now - 5m; now]) == 0" + evaluateEvery: 30s + for: 2m + + - ruleId: mempool-high # cardano_node_mempool_high + summary: "More than 200 transactions in mempool for over 10 minutes" + severity: warning + query: "cardano_node_metrics_txsInMempool_int > 200" + evaluateEvery: 30s + for: 10m + + - ruleId: mempool-soft-timeouts # cardano_node_mempool_soft_timeouts_detected + summary: "Mempool soft timeouts detected in the past hour" + severity: warning + query: "increase (cardano_node_metrics_txsMempoolTimeoutSoft_counter[now - 1h; now]) > 2" + evaluateEvery: 1m + for: 1m + + - ruleId: mempool-hard-timeouts # cardano_node_mempool_hard_timeouts_detected + summary: "Mempool hard timeouts detected in the past hour" + severity: error + query: "increase (cardano_node_metrics_txsMempoolTimeoutHard_counter[now - 1h; now]) > 0" + evaluateEvery: 1m + for: 1m + + - ruleId: blockheight-metric-missing # cardano_node_metric_missing + summary: "Blockheight metric missing for more than 10 minutes" + severity: error + query: >- + unless (sum_over_time ((cardano_node_metrics_blockNum_int != 0)[now - 360m; now : 1m]) < 350) + cardano_node_metrics_blockNum_int + evaluateEvery: 1m + for: 1m + + # --- cardano-node-forge.nix-import ------------------------------------- + - ruleId: no-blocks-forged-24h # cardano_node_forge_blocks_missing + summary: "No blocks forged in the past 24 hours" + severity: critical + query: "increase (cardano_node_metrics_blocksForged_int[now - 24h; now]) == 0" + evaluateEvery: 5m + for: 1m + + - ruleId: forged-not-adopted # cardano_node_forge_not_adopted_error + summary: "Failed to adopt one or more forged blocks in the past hour" + severity: error + query: "increase (cardano_node_metrics_Forge_didnt_adopt_counter[now - 1h; now]) > 0" + evaluateEvery: 1m + for: 1m + + - ruleId: cannot-forge # cardano_node_cannot_forge_new_tracing + summary: "Failed to forge one or more blocks in the past hour" + severity: error + query: "increase (cardano_node_metrics_nodeCannotForge_int[now - 1h; now]) > 0" + evaluateEvery: 1m + for: 1m + + - ruleId: slot-leadership-checks-missed # too_many_slot_leadership_checks_missed + summary: "Slot leadership checks missed for more than half of slots" + severity: critical + query: "rate (cardano_node_metrics_slotsMissed_int[now - 5m; now]) > 0.5" + evaluateEvery: 30s + for: 2m + + # KES: the richer severity vocabulary replaces three identical "page" + # alerts with an escalation ladder. + - ruleId: kes-expiry-10-periods # cardano_node_KES_expiration_metric_10period_notice + summary: "Less than 10 KES periods remaining" + severity: warning + query: "cardano_node_metrics_remainingKESPeriods_int <= 10" + evaluateEvery: 5m + for: 5m + - ruleId: kes-expiry-5-periods + summary: "Less than 5 KES periods remaining" + severity: error + query: "cardano_node_metrics_remainingKESPeriods_int <= 5" + evaluateEvery: 5m + for: 5m + - ruleId: kes-expiry-1-period + summary: "KES expires within 1 period" + severity: critical + query: "cardano_node_metrics_remainingKESPeriods_int <= 1" + evaluateEvery: 5m + for: 5m + + # --- cardano-node-network.nix-import ------------------------------------ + # This one is the concept doc's own example rule. + - ruleId: high-ping-latency # high_cardano_ping_latency + summary: "Average node ping latency above 500 ms" + severity: warning + query: "avg_over_time (netdata_statsd_cardano_node_ping_latency_ms_gauge_value_average[now - 5m; now]) > 500" + evaluateEvery: 30s + for: 60m + + - ruleId: block-adoption-delay # blocks_adoption_delay_too_high + summary: "95th-percentile block adoption delay above 4.5 s" + severity: warning + query: "avg (quantile_over_time 0.95 (cardano_node_metrics_blockfetchclient_blockdelay_real[now - 6h; now])) >= 4.5" + evaluateEvery: 5m + for: 1m + + - ruleId: block-utilization-high # blocks_utilization_too_high + summary: "Average block utilization above 95%" + severity: warning + query: "100 * avg (avg_over_time (cardano_node_metrics_blockfetchclient_blocksize_int[now - 6h; now]) / 90112) > 95" + evaluateEvery: 5m + for: 5m + + - ruleId: blockfetch-delay-high # cardano_blockfetchclient_blockdelay_high + summary: "Less than 90% of blocks arriving within 5 seconds" + severity: warning + query: "cardano_node_metrics_blockfetchclient_blockdelay_cdfFive_real < 0.90" + evaluateEvery: 1m + for: 10m + + - ruleId: blockfetch-delay-critical # cardano_blockfetchclient_blockdelay_critical + summary: "Less than 50% of blocks arriving within 5 seconds" + severity: critical + query: "cardano_node_metrics_blockfetchclient_blockdelay_cdfFive_real < 0.50" + evaluateEvery: 1m + for: 10m + + - ruleId: connection-count-high # cardano_connection_count_high + summary: "Incoming connection count above 450 (hard limit 512)" + severity: warning + query: "cardano_node_metrics_connectionManager_inboundConns_int > 450" + evaluateEvery: 1m + for: 10m + + # --- cardano-node-quality.nix-import ------------------------------------ + # Fleet-wide rule: only meaningful when the whole fleet forwards to this + # tracer (see caveats). The regex exclusion {environment!~"preview"} has + # no equivalent; use != or per-environment rules. + - ruleId: chain-density-degraded # chain_quality_degraded + summary: "More than 20% of nodes below 70% chain density" + severity: warning + query: '100 * quantile_by ("environment") 0.2 (cardano_node_metrics_density_real{environment != "preview"} * 20) < 70' + evaluateEvery: 1m + for: 5m +``` + +## Alerts that do not translate one-to-one + +**Block divergence (`cardano-node-divergence.nix-import`).** The PromQL uses +cross-series matching (`max(m) - on() group_right() m`) to compare each node +against the fleet maximum. The language has no vector matching, but its +functional extras can express the same idea: + +```text +let peak = to_scalar (max (cardano_node_metrics_blockNum_int)) in +map (\x -> abs (peak - x) > 6) cardano_node_metrics_blockNum_int +``` + +combined with the analogous slot-lag condition via `&&`. This is the one +family where the translation needs prototyping against the real interpreter +before committing to a rule. + +**Elevated restarts (`cardano_node_elevated_restarts`).** Uses +`time() - nodeStartTime` inside a subquery. `now` and timestamp arithmetic +exist, and subqueries with a step are supported, so this is expressible in +principle — but timestamp-vs-number typing needs verification. A simpler +first version: alarm when `cardano_node_metrics_nodeStartTime_int` changed +within the window, or leave restart detection to the trace severity rules +(the node logs its startup). + +**Old/new tracing metric-name pairs.** Half the catalogue exists twice +(`...Forge_forged_int` vs `...blocksForged_int`) joined by PromQL vector +`or`, because Prometheus may scrape either naming scheme. `cardano-tracer` +receives metrics from the node's own forwarder, i.e. the new-tracing names +only — the `_new_tracing` variants are the ones to port, and the `or` +fallback disappears. + +## General caveats + +- **Scope.** Grafana alerts run against a central Prometheus that scrapes the + whole fleet. A `cardano-tracer` store contains only the nodes forwarding to + that tracer instance. Per-node rules (mempool, KES, forging) translate + directly; fleet-wide statistics (chain density, divergence) are only + meaningful when the relevant fleet shares one tracer. +- **Labels.** The store's series label is `node_name` (set by the acceptor), + not Prometheus's `instance`/`environment`. Rules that filter on + `environment` need that label to exist in the store, or need the filter + dropped. Alarm routing dimensions come from the rule's `labels`/scope + config plus the series key, not from annotation templates. +- **Metric names.** Names pass through `sanitiseMetricName` on ingestion. + Before writing a rule, list what the store actually holds: + `GET /timeseries/query?query=metrics`. +- **Delivery.** Grafana pages via its contact points; here, delivery is the + consumer configuration (today `log`; `webhook`/`email` planned). Porting + the rules is independent of porting the paging integration. +- **Testing.** Each ported rule should get a Level-1 test as described in + `timeseries-alarm-testing.md`: insert samples that straddle the threshold + and the `for` window, evaluate at fixed timestamps, assert exactly one + alarm. + +## Prerequisite + +All of this targets the Phase-2 timeseries rule evaluator from the concept +doc, which is not yet implemented. The value of this exercise now: the +catalogue above is a realistic acceptance suite for that evaluator — if these +rules can be configured and fire correctly, Phase 2 is functionally complete +for SRE use. diff --git a/cardano-tracer/docs/timeseries-alarm-testing.md b/cardano-tracer/docs/timeseries-alarm-testing.md new file mode 100644 index 00000000000..0443d37a40f --- /dev/null +++ b/cardano-tracer/docs/timeseries-alarm-testing.md @@ -0,0 +1,159 @@ +# Writing test cases for timeseries alarm rules + +Status: guide (companion to `alarm-system-concept.md`) + +This document describes how to write test cases for alarms driven by +timeseries rules (Phase 2 of the alarm concept), using the pieces that exist +today: the timeseries store with its `insert`/`execute` API, the alarm +registry, and the test patterns already used in `cardano-tracer-test`. + +## The data entry point: `insert` + +Test data is added through the in-process API of `cardano-timeseries-io` +(`Cardano.Timeseries.Component`): + +```haskell +insert :: TimeseriesHandle -> SeriesIdentifier -> Timestamp -> [(MetricIdentifier, Double)] -> IO () +execute :: TimeseriesHandle -> Timestamp -> Text -> IO (Either ExecutionError Value) +``` + +- A series is identified by labels, e.g. `Set.fromList [("node_name", "node-1")]` -- + the same shape production uses when node metrics arrive + (`Cardano.Tracer.Acceptors.Utils.store`). +- Timestamps are milliseconds and are **supplied by the caller**. This is what + makes tests deterministic: insert samples at chosen times and evaluate the + query at a chosen `at`, instead of racing the real clock. + +There is deliberately **no HTTP insert endpoint** -- the timeseries server only +serves `query`, `prune`, `config`, and `nodes`. In production, data enters +only as metrics forwarded from nodes. Test strategies have to respect that. + +## Level 1 (recommended): in-process tests + +Follow the style of `Cardano.Tracer.Test.Alarms.Tests`: no server, no +processes, everything driven directly. This is the right level for testing +rule logic (thresholds, `for` durations, edge-triggering, repeat intervals). + +```haskell +propHighLatencyRaisesAlarm :: Property +propHighLatencyRaisesAlarm = once $ ioProperty do + bundle <- mkTraceBundle (SeverityF (Just Warning)) + handle <- Timeseries.create @(Tree _) (timeseries bundle) Nothing + registry <- newAlarmRegistry (assorted bundle) testAlarmsConfig + + -- 1. Arrange: five samples, 30 s apart, all above the threshold + let t0 = 1_700_000_000_000 -- fixed ms timestamp + series = Set.fromList [("node_name", "node-1")] + for_ [0 .. 4] \i -> + Timeseries.insert handle series (t0 + i * 30_000) [("ping_latency_ms", 640)] + + -- 2. Act: evaluate the rule once, at a fixed time + -- (evaluateTimeseriesRules is the Phase-2 evaluator entry point; + -- design it to take the evaluation timestamp as an argument!) + evaluateTimeseriesRules registry handle (t0 + 5 * 30_000) + + -- 3. Assert: exactly one alarm in the store + events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter + pure (length events === 1) +``` + +Design rule for the evaluator that follows from this: **pass the evaluation +time in explicitly** (like `pruneOnce` and `traceRuleRequest` already do). +Anything that calls `getCurrentTime` internally cannot be tested +deterministically. + +Until the evaluator exists, the same pattern already works for testing query +semantics alone: insert known samples, call `execute handle at query`, and +assert on the returned `Value` (thresholds crossing true/false, missing data, +window boundaries). + +## Level 2: end-to-end with a launched `cardano-tracer` + +One smoke test should cover the full wiring: config parsing, store, evaluator +thread, alarm history endpoint. The pattern is the one `Test.Logs` uses: +start the tracer and a forwarder in-process inside a tasty test, then assert +over HTTP. + +Configuration for the test (YAML): + +```yaml +networkMagic: 42 +network: + acceptAt: "tracer.sock" +logging: + - logRoot: "logs" + logMode: FileMode + logFormat: ForMachine +hasTimeseries: + epHost: "127.0.0.1" + epPort: 3300 +alarms: + endpoint: + epHost: "127.0.0.1" + epPort: 3210 + allowInsecure: true # test only -- no TLS certificate + authentication: + producers: [] + readers: + - name: test-reader + tokenFile: "reader.token" # written by the test setup + allowHistory: true + timeseriesRules: # Phase 2, not yet implemented + - ruleId: high-latency + summary: "latency above threshold" + severity: warning + query: "avg_over_time (ping_latency_ms[now - 5m; now]) > 500" + evaluateEvery: 1s # keep the test fast + consumers: [] +``` + +Feeding data -- two workable options: + +1. **Reuse the test forwarder** (`Cardano.Tracer.Test.Forwarder`, also + available as the `demo-forwarder` executable). It forwards EKG metrics over + `trace-forward` exactly like a node, and the acceptor inserts them into the + timeseries store with the `node_name` label. Extend it (or configure it) to + register a gauge with the metric name the rule queries. Caveats: metric + names pass through `sanitiseMetricName`, only numeric values are inserted, + and timestamps are "now" -- so the test asserts *eventually* (poll with a + deadline), not at exact instants. + +2. **A script cannot insert directly** -- there is no HTTP insert route, and + adding one for tests is not recommended (it would create a second, + test-only ingestion path that production never exercises). If a scriptable + entry point is ever needed, prefer a small Haskell driver that links the + forwarder library rather than a raw HTTP endpoint. + +Asserting: + +```bash +# data arrived? +curl 'http://127.0.0.1:3300/timeseries/query?query=ping_latency_ms' +# alarm raised? +curl -H "Authorization: Bearer $(cat reader.token)" \ + 'http://127.0.0.1:3210/alarms/v1/events?minSeverity=warning' +``` + +In a tasty test, do the same with a few lines of `http-client` and poll until +the history response contains the expected `ruleId` or a timeout expires. + +## What about preloading test data via the configuration? + +A config option like `hasTimeseries: { preloadFile: testdata.json }` -- a file +of `(labels, timestamp, metric, value)` rows loaded into the store right after +`Timeseries.create` in `Run.hs` -- would make end-to-end tests fully +deterministic (fixed timestamps, no forwarder needed) and is cheap to build. +It does not exist yet. If Level-2 tests become flaky because of the +"eventually" polling, this is the first improvement to make; until then, +Level 1 covers determinism and Level 2 covers wiring. + +## Summary + +| Level | What it tests | Data entry | Determinism | +| --- | --- | --- | --- | +| 1: in-process | query + rule semantics | `Timeseries.insert`, fixed timestamps | full | +| 2: launched tracer | config, threads, HTTP, history | test forwarder (EKG metrics) | poll with deadline | + +Put Level-1 properties in `Cardano.Tracer.Test.Alarms.Tests` (they need no +work directory) and the Level-2 smoke test in its own module following +`Test.Logs`' `propRunInLogsStructure` pattern. From 75a6179d0eefb3c2b8b6fc7625d1a713aa747598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Baldur=20Bl=C3=B6ndal?= Date: Tue, 11 Aug 2026 15:27:54 +0100 Subject: [PATCH 3/8] cardano-tracer: log consumer emits alarm payload, not just consumer name --- .../Node/Tracing/DefaultTraceConfig.hs | 2 + cardano-node/src/Cardano/Node/Tracing/Span.hs | 140 +++++++ .../src/Cardano/Node/Tracing/Tracers.hs | 12 + .../Cardano/Tracer/Handlers/Alarms/Auth.hs | 8 +- .../Tracer/Handlers/Alarms/Consumers.hs | 8 +- .../Tracer/Handlers/Alarms/Registry.hs | 12 + .../Cardano/Tracer/Handlers/Alarms/Server.hs | 1 + .../src/Cardano/Tracer/MetaTrace.hs | 22 +- .../test/Cardano/Tracer/Test/Alarms/Tests.hs | 386 +++++++++++++++--- 9 files changed, 519 insertions(+), 72 deletions(-) create mode 100644 cardano-node/src/Cardano/Node/Tracing/Span.hs diff --git a/cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs b/cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs index 43453aaebc8..181ffdcf283 100644 --- a/cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs +++ b/cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs @@ -26,6 +26,8 @@ defaultCardanoConfig = emptyTraceConfig { [ ConfSeverity (SeverityF Nothing)]) ,(["ChainDB"], [ ConfSeverity (SeverityF (Just Info))]) + ,(["Span"], + [ ConfSeverity (SeverityF (Just Info))]) ,(["ChainDB", "AddBlockEvent", "AddBlockValidation"], [ ConfSeverity (SeverityF Nothing)]) ,(["ChainSync", "Client"], diff --git a/cardano-node/src/Cardano/Node/Tracing/Span.hs b/cardano-node/src/Cardano/Node/Tracing/Span.hs new file mode 100644 index 00000000000..e43a18914d6 --- /dev/null +++ b/cardano-node/src/Cardano/Node/Tracing/Span.hs @@ -0,0 +1,140 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | OpenTelemetry-style spans on top of the @trace-dispatcher@ framework. +-- +-- A span is a pair of correlated trace messages — 'SpanBegin' and 'SpanEnd' — +-- sharing a single 'SpanId'. 'withSpan' brackets an action so that the end +-- message is always emitted, even on exception, which is what lets a +-- Loki-style alerting system fire on \"span that never ended\". +-- +-- The duration is measured /client-side/ (in this process, monotonic clock) +-- and shipped inside 'SpanEnd', so the consumer (cardano-tracer / timeseries / +-- Prometheus) stays stateless — it never has to pair begin\/end itself. +-- +-- * For __timeseries\/Prometheus__: 'asMetrics' emits @spanDurationMs@. +-- * For __Loki (LogQL)__: 'forMachine' emits a flat JSON object with a +-- stable @span_id@ (log field, high cardinality) and @name@ +-- (label-friendly, low cardinality) plus @event=begin|end@. +module Cardano.Node.Tracing.Span + ( SpanId (..) + , SpanTrace (..) + , newSpanId + , withSpan + ) where + +import Cardano.Logging + +import Control.Exception.Safe (MonadMask, finally) +import Control.Monad.IO.Class (MonadIO, liftIO) +import Data.Aeson (Value (String), (.=)) +import Data.Text (Text) +import Data.Unique (hashUnique, newUnique) +import Data.Word (Word64) +import GHC.Clock (getMonotonicTimeNSec) + +-- | Correlation id shared by a 'SpanBegin' / 'SpanEnd' pair. +-- Process-unique for the lifetime of the run (see 'newSpanId'). +newtype SpanId = SpanId { unSpanId :: Word64 } + deriving (Eq, Ord, Show) + +-- | The two ends of a span. +data SpanTrace + = SpanBegin !SpanId !Text + -- ^ Start of a span: id + human name of the operation. + | SpanEnd !SpanId !Text !Double + -- ^ End of a span: id, name, and measured duration in milliseconds. + deriving (Show) + +-- | Allocate a fresh, process-unique span id. +-- +-- Uses 'Data.Unique' so it needs no extra dependency and never blocks. Ids +-- are unique within a single node run; they are /not/ stable across restarts +-- (fine for correlating one begin with one end — which is all we need). +newSpanId :: MonadIO m => m SpanId +newSpanId = liftIO (SpanId . fromIntegral . hashUnique <$> newUnique) + +-- | Run @action@ inside a span, emitting 'SpanBegin' before and 'SpanEnd' +-- after — even if @action@ throws. +-- +-- @ +-- withSpan tr \"replayLedger\" $ do +-- ...work... +-- @ +withSpan + :: (MonadIO m, MonadMask m) + => Trace m SpanTrace -- ^ where to emit the span messages + -> Text -- ^ human name of the operation + -> m a -- ^ the work to measure + -> m a +withSpan tr name action = do + sid <- newSpanId + !t0 <- liftIO getMonotonicTimeNSec + traceWith tr (SpanBegin sid name) + action `finally` do + !t1 <- liftIO getMonotonicTimeNSec + let !ms = fromIntegral (t1 - t0) / 1e6 :: Double + traceWith tr (SpanEnd sid name ms) + +-------------------------------------------------------------------------------- +-- Formatting +-------------------------------------------------------------------------------- + +instance LogFormatting SpanTrace where + forMachine _ (SpanBegin sid name) = + mconcat + [ "kind" .= String "SpanBegin" + , "event" .= String "begin" + , "span_id" .= unSpanId sid + , "name" .= name + ] + forMachine _ (SpanEnd sid name ms) = + mconcat + [ "kind" .= String "SpanEnd" + , "event" .= String "end" + , "span_id" .= unSpanId sid + , "name" .= name + , "duration_ms" .= ms + ] + + forHuman (SpanBegin sid name) = + "Span begin [" <> showT (unSpanId sid) <> "] " <> name + forHuman (SpanEnd sid name ms) = + "Span end [" <> showT (unSpanId sid) <> "] " <> name + <> " (" <> showT ms <> " ms)" + + -- Only the end carries a measurement. The metric name embeds the span name so + -- distinct operations are distinguishable; keep the set of names SMALL to + -- avoid Prometheus/timeseries cardinality blow-up. + asMetrics (SpanBegin _ _) = [] + asMetrics (SpanEnd _ name ms) = + [ DoubleM ("spanDurationMs." <> name) ms ] + +-------------------------------------------------------------------------------- +-- Documentation / metadata +-------------------------------------------------------------------------------- + +instance MetaTrace SpanTrace where + namespaceFor SpanBegin{} = Namespace [] ["Span", "Begin"] + namespaceFor SpanEnd{} = Namespace [] ["Span", "End"] + + severityFor (Namespace _ ["Span", "Begin"]) _ = Just Info + severityFor (Namespace _ ["Span", "End"]) _ = Just Info + severityFor _ _ = Nothing + + documentFor (Namespace _ ["Span", "Begin"]) = Just + "Start of a correlated span. Carries a span_id shared with the matching \ + \Span.End, and the human name of the operation." + documentFor (Namespace _ ["Span", "End"]) = Just + "End of a correlated span. Carries the same span_id as Span.Begin plus \ + \the client-side measured duration in milliseconds." + documentFor _ = Nothing + + metricsDocFor (Namespace _ ["Span", "End"]) = + [("spanDurationMs", "Client-side measured span duration, in milliseconds")] + metricsDocFor _ = [] + + allNamespaces = + [ Namespace [] ["Span", "Begin"] + , Namespace [] ["Span", "End"] + ] diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers.hs index 9d131d4ef89..a0c5df9a199 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers.hs @@ -36,6 +36,7 @@ import Cardano.Node.Tracing.Tracers.NodeVersion (getNodeVersion) import Cardano.Node.Tracing.Tracers.Rpc () import Cardano.Node.Tracing.Tracers.Shutdown () import Cardano.Node.Tracing.Tracers.Startup () +import Cardano.Node.Tracing.Span (withSpan) import Ouroboros.Consensus.Ledger.Inspect (LedgerEvent) import Ouroboros.Consensus.MiniProtocol.ChainSync.Client (TraceChainSyncClientEvent) import qualified Ouroboros.Consensus.Network.NodeToClient as NodeToClient @@ -156,6 +157,12 @@ mkDispatchTracers nodeKernel trBase trForward mbTrEKG trDataPoint trConfig forgi !rpcTr <- mkCardanoTracer trBase trForward mbTrEKG ["RPC"] configureTracers configReflection trConfig [rpcTr] + -- Span tracer: emits correlated Span.Begin / Span.End pairs. Registered + -- through the standard pipeline so spans are forwarded to cardano-tracer + -- (and thus visible to Loki) and exposed on Prometheus/EKG as metrics. + !spanTr <- mkCardanoTracer trBase trForward mbTrEKG [] + configureTracers configReflection trConfig [spanTr] + traceTracerInfo trBase trForward configReflection let warnings = checkNodeTraceConfiguration' trConfig @@ -166,6 +173,11 @@ mkDispatchTracers nodeKernel trBase trForward mbTrEKG trDataPoint trConfig forgi traceWith nodeVersionTr getNodeVersion + -- Demonstration span emitted once at tracer initialisation. This proves the + -- begin/end pair flows through the pipeline; replace with real withSpan + -- calls around the operations you want to measure. + withSpan spanTr "nodeTracersInit" (pure ()) + pure Tracers { chainDBTracer = mkTracer (traceWith chainDBTr') diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs index d178afbd7c5..da4f38fdd56 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs @@ -38,7 +38,8 @@ data ProducerCredential = ProducerCredential deriving stock (Eq, Show) data ReaderCredential = ReaderCredential - { rcrAllowHistory :: !Bool + { rcrName :: !Text + , rcrAllowHistory :: !Bool , rcrFilter :: !AlarmFilter } deriving stock (Eq, Show) @@ -65,11 +66,12 @@ loadCredentials AlarmsAuthConfig{aacProducers, aacReaders} = do token <- readTokenFile pcTokenFile pure (token, ProducerCredential (AlarmSource pcSource)) - loadReader ReaderCredentialConfig{rcTokenFile, rcAllowHistory, rcFilter} = do + loadReader ReaderCredentialConfig{rcName, rcTokenFile, rcAllowHistory, rcFilter} = do token <- readTokenFile rcTokenFile pure ( token , ReaderCredential - { rcrAllowHistory = fromMaybe False rcAllowHistory + { rcrName = rcName + , rcrAllowHistory = fromMaybe False rcAllowHistory , rcrFilter = maybe emptyAlarmFilter filterFromConfig rcFilter } ) diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs index 9d1a59d2a60..8405afc5c24 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs @@ -36,5 +36,11 @@ consumerFromConfig AlarmsConsumerLog{aclName, aclEnabled, aclFilter} = ConsumerL dispatch :: Trace IO TracerTrace -> AlarmConsumer -> AlarmEvent -> IO () dispatch tracer consumer@ConsumerLog{clEnabled, clFilter} ev | clEnabled && matchesFilter clFilter ev = - traceWith tracer TracerAlarmDispatched { ttAlarmDispatchedConsumer = clName consumer } + traceWith tracer TracerAlarmDispatched + { ttAlarmDispatchedConsumer = clName consumer + , ttAlarmDispatchedSource = unAlarmSource (source ev) + , ttAlarmDispatchedRuleId = unRuleId (ruleId ev) + , ttAlarmDispatchedSeverity = severity ev + , ttAlarmDispatchedSummary = summary ev + } | otherwise = pure () diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs index a7f72e60497..f57579d6228 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs @@ -13,6 +13,7 @@ module Cardano.Tracer.Handlers.Alarms.Registry , rejectEvent , checkTraceObjectsForAlarms , readHistoryFiltered + , traceHistoryRead , lookupProducerCredential , lookupReaderCredential ) where @@ -98,6 +99,17 @@ checkTraceObjectsForAlarms registry nodeName traceObjects = readHistoryFiltered :: AlarmRegistry -> Maybe AlarmCursor -> Int -> AlarmFilter -> IO [(AlarmCursor, AlarmEvent)] readHistoryFiltered registry = readHistory (arStore registry) +-- | Emit a meta-trace recording a successful history read: which reader +-- credential fired it and how many events the response carried. Consumers +-- of the ingress path have 'TracerAlarmAccepted'/'TracerAlarmDuplicate'; +-- this is the corresponding observability hook for the read path. +traceHistoryRead :: AlarmRegistry -> Text -> Int -> IO () +traceHistoryRead registry readerName resultCount = + traceWith (arTracer registry) TracerAlarmHistoryRead + { ttAlarmHistoryReader = readerName + , ttAlarmHistoryResultCount = resultCount + } + lookupProducerCredential :: AlarmRegistry -> Text -> Maybe ProducerCredential lookupProducerCredential registry = lookupProducer (arAuth registry) diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs index 0b982112c28..13593e4dfaf 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs @@ -164,6 +164,7 @@ handleHistory registry request send = then send (forbidden "requested filter is not permitted by this credential") else do events <- readHistoryFiltered registry after limit requested + traceHistoryRead registry (rcrName credential) (length events) send $ okResponse $ object [ "events" .= map (Aeson.toJSON . snd) events , "nextCursor" .= case events of diff --git a/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs b/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs index fe014d9198f..efa5fdec7d0 100644 --- a/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs +++ b/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs @@ -101,6 +101,14 @@ data TracerTrace } | TracerAlarmDispatched { ttAlarmDispatchedConsumer :: Text + , ttAlarmDispatchedSource :: Text + , ttAlarmDispatchedRuleId :: Text + , ttAlarmDispatchedSeverity :: SeverityS + , ttAlarmDispatchedSummary :: Text + } + | TracerAlarmHistoryRead + { ttAlarmHistoryReader :: Text + , ttAlarmHistoryResultCount :: Int } deriving Show @@ -224,6 +232,15 @@ instance LogFormatting TracerTrace where TracerAlarmDispatched{..} -> mconcat [ "kind" .= AE.String "TracerAlarmDispatched" , "consumer" .= ttAlarmDispatchedConsumer + , "source" .= ttAlarmDispatchedSource + , "ruleId" .= ttAlarmDispatchedRuleId + , "severity" .= ttAlarmDispatchedSeverity + , "summary" .= ttAlarmDispatchedSummary + ] + TracerAlarmHistoryRead{..} -> mconcat + [ "kind" .= AE.String "TracerAlarmHistoryRead" + , "reader" .= ttAlarmHistoryReader + , "resultCount" .= ttAlarmHistoryResultCount ] instance MetaTrace TracerTrace where @@ -254,6 +271,7 @@ instance MetaTrace TracerTrace where namespaceFor TracerAlarmDuplicate {} = Namespace [] ["AlarmDuplicate"] namespaceFor TracerAlarmRejected {} = Namespace [] ["AlarmRejected"] namespaceFor TracerAlarmDispatched {} = Namespace [] ["AlarmDispatched"] + namespaceFor TracerAlarmHistoryRead {} = Namespace [] ["AlarmHistoryRead"] severityFor (Namespace _ ["BuildInfo"]) _ = Just Info severityFor (Namespace _ ["ParamsAre"]) _ = Just Warning @@ -281,7 +299,8 @@ instance MetaTrace TracerTrace where severityFor (Namespace _ ["AlarmAccepted"]) _ = Just Info severityFor (Namespace _ ["AlarmDuplicate"]) _ = Just Debug severityFor (Namespace _ ["AlarmRejected"]) _ = Just Warning - severityFor (Namespace _ ["AlarmDispatched"]) _ = Just Debug + severityFor (Namespace _ ["AlarmDispatched"]) _ = Just Info + severityFor (Namespace _ ["AlarmHistoryRead"]) _ = Just Info severityFor _ _ = Nothing documentFor _ = Just "" @@ -314,6 +333,7 @@ instance MetaTrace TracerTrace where , Namespace [] ["AlarmDuplicate"] , Namespace [] ["AlarmRejected"] , Namespace [] ["AlarmDispatched"] + , Namespace [] ["AlarmHistoryRead"] ] stderrShowTracer :: Show a => Trace IO a diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs index 553d8d428b6..c3cf4dd1756 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs @@ -1,7 +1,15 @@ +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} +-- | Real property tests for the alarm subsystem. Each property targets a +-- concrete promise from @cardano-tracer/docs/alarm-system-concept.md@: +-- idempotency, cursor ordering, retention, filter algebra, reader +-- ceiling, and trace-rule flood suppression. Fixture-style tests (a +-- single hand-picked input) are kept only where the behaviour is truly +-- singleton — everything else is generator-driven. module Cardano.Tracer.Test.Alarms.Tests ( tests ) where @@ -15,29 +23,145 @@ import Cardano.Tracer.Handlers.Alarms.Types import Cardano.Tracer.MetaTrace (TraceBundle (..), mkTraceBundle) import Control.Concurrent.Async (forConcurrently) +import Data.Aeson (decode, encode) +import qualified Data.List as List +import Data.Map.Strict (Map) import qualified Data.Map.Strict as Map +import Data.Text (Text) import qualified Data.Text as Text import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime) +import Data.Time.Clock.POSIX (posixSecondsToUTCTime, utcTimeToPOSIXSeconds) import Test.Tasty import Test.Tasty.QuickCheck +-------------------------------------------------------------------------------- +-- Test suite +-------------------------------------------------------------------------------- + tests :: TestTree -tests = localOption (QuickCheckTests 1) $ testGroup "Test.Alarms" - [ testProperty "AlarmEvent JSON round-trip uses lowercase severity" propSeverityLowercase - , testProperty "idempotent double-submit returns the same eventId" propIdempotentSubmit - , testProperty "concurrent double-submit creates exactly one event" propConcurrentSubmit - , testProperty "prune then resubmit issues a fresh eventId" propPruneThenResubmit - , testProperty "trace rule fires only at or above its threshold" propTraceRuleThreshold - , testProperty "trace rule suppresses repeats within one window" propTraceRuleSuppression - , testProperty "trace rule fires again in a new window" propTraceRuleNewWindow +tests = testGroup "Test.Alarms" + [ testGroup "envelope" + [ testProperty "severity vocabulary lowercase round-trip" propSeverityRoundTrip + , testProperty "AlarmEvent JSON round-trip" propAlarmEventRoundTrip + ] + , testGroup "store" + [ testProperty "idempotent double-submit returns same eventId" propIdempotentSubmit + , testProperty "concurrent submit creates exactly one" propConcurrentSubmit + , testProperty "prune then resubmit issues a fresh eventId" propPruneThenResubmit + , testProperty "cursor is strictly monotonic across inserts" propCursorMonotonic + , testProperty "retention maxEvents keeps exactly maxN" propRetentionMaxEvents + , testProperty "retention maxAgeSeconds drops older events" propRetentionMaxAge + , testProperty "readHistory after=cursor is exclusive" propHistoryAfterExclusive + ] + , testGroup "filter algebra" + [ testProperty "empty filter accepts every event" propEmptyFilterAccepts + , testProperty "scope is submap, not intersection" propScopeIsSubmap + , testProperty "minSeverity is <= (inclusive)" propMinSeverityInclusive + , testProperty "filterNarrows is reflexive" propFilterNarrowsReflexive + , testProperty "filterNarrows is transitive" propFilterNarrowsTransitive + , testProperty "broader-than-ceiling filter is rejected" propBroaderRejected + ] + , testGroup "trace rules" + [ testProperty "rule fires only at or above threshold" propTraceRuleThreshold + , testProperty "matches in one window collapse to one alarm" propTraceRuleSuppression + , testProperty "matches in a new window raise a fresh alarm" propTraceRuleNewWindow + ] ] -mkRequest :: Text.Text -> SeverityS -> IO IngressRequest -mkRequest sourceEventId sev = do +-------------------------------------------------------------------------------- +-- Generators +-------------------------------------------------------------------------------- + +genSeverity :: Gen SeverityS +genSeverity = elements [Debug, Info, Notice, Warning, Error, Critical, Alert, Emergency] + +genShortText :: Gen Text +genShortText = do + n <- chooseInt (1, 16) + Text.pack <$> vectorOf n (elements (['a'..'z'] <> ['0'..'9'] <> "_-")) + +genSource :: Gen AlarmSource +genSource = AlarmSource <$> elements ["trace", "hermod-recon", "timeseries", "test-src"] + +genRule :: Gen RuleId +genRule = RuleId <$> elements ["rule-a", "rule-b", "rule-c", "rule-d"] + +genLabels :: Gen (Map Text Text) +genLabels = do + n <- chooseInt (0, 3) + Map.fromList <$> vectorOf n ((,) <$> elements ["team", "site", "region", "shard"] + <*> elements ["ops", "dev", "eu", "us", "0", "1"]) + +genUTC :: Gen UTCTime +genUTC = do + s <- chooseInteger (1_700_000_000, 1_900_000_000) + pure (posixSecondsToUTCTime (fromIntegral s)) + +genIngress :: Gen IngressRequest +genIngress = IngressRequest + <$> genShortText + <*> genUTC + <*> genRule + <*> genSeverity + <*> genShortText + <*> genLabels + <*> genLabels + <*> pure Nothing + +genEvent :: Gen AlarmEvent +genEvent = do + ir <- genIngress + src <- genSource + now <- genUTC + eid <- genShortText + pure AlarmEvent + { schemaVersion = 1 + , eventId = eid + , sourceEventId = irSourceEventId ir + , raisedAt = irRaisedAt ir + , receivedAt = now + , source = src + , ruleId = irRuleId ir + , severity = irSeverity ir + , summary = irSummary ir + , scope = irScope ir + , labels = irLabels ir + , details = irDetails ir + } + +genFilter :: Gen AlarmFilter +genFilter = AlarmFilter + <$> oneof [pure Nothing, Just <$> genSource] + <*> oneof [pure Nothing, Just <$> genRule] + <*> oneof [pure Nothing, Just <$> genSeverity] + <*> genLabels + <*> genLabels + +-------------------------------------------------------------------------------- +-- Envelope properties +-------------------------------------------------------------------------------- + +propSeverityRoundTrip :: Property +propSeverityRoundTrip = forAll genSeverity \sev -> + parseAlarmSeverityText (alarmSeverityToText sev) === Just sev + +propAlarmEventRoundTrip :: Property +propAlarmEventRoundTrip = forAll genEvent \ev -> + decode (encode ev) === Just ev + +-------------------------------------------------------------------------------- +-- Store properties +-------------------------------------------------------------------------------- + +testSource :: AlarmSource +testSource = AlarmSource "hermod-recon" + +mkRequestFromParts :: Text -> SeverityS -> IO IngressRequest +mkRequestFromParts sid sev = do now <- getCurrentTime pure IngressRequest - { irSourceEventId = sourceEventId + { irSourceEventId = sid , irRaisedAt = now , irRuleId = RuleId "test-rule" , irSeverity = sev @@ -47,11 +171,162 @@ mkRequest sourceEventId sev = do , irDetails = Nothing } -testSource :: AlarmSource -testSource = AlarmSource "hermod-recon" +propIdempotentSubmit :: Property +propIdempotentSubmit = forAll genShortText \sid -> forAll genSeverity \sev -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig Nothing Nothing) + req <- mkRequestFromParts sid sev + now <- getCurrentTime + (_, ev1, created1) <- insertOrGetExisting store testSource now req + (_, ev2, created2) <- insertOrGetExisting store testSource now req + pure $ conjoin + [ counterexample "first submit must create" (property created1) + , counterexample "second submit must not create" (property (not created2)) + , counterexample "eventIds must match" (eventId ev1 === eventId ev2) + ] --- | One trace rule: alarm on severity >= Error, at most one alarm per --- (node, namespace) per 60-second window. +propConcurrentSubmit :: Property +propConcurrentSubmit = forAll genShortText \sid -> forAll genSeverity \sev -> + forAll (chooseInt (2, 32)) \fanout -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig Nothing Nothing) + req <- mkRequestFromParts sid sev + now <- getCurrentTime + results <- forConcurrently [1 .. fanout] \_ -> + insertOrGetExisting store testSource now req + let createdCount = length (filter (\(_, _, c) -> c) results) + pure (createdCount === 1) + +propPruneThenResubmit :: Property +propPruneThenResubmit = forAll genShortText \sid -> forAll genSeverity \sev -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig (Just 0) Nothing) + req <- mkRequestFromParts sid sev + now <- getCurrentTime + (_, ev1, _) <- insertOrGetExisting store testSource now req + later <- getCurrentTime + pruneOnce store later + (_, ev2, created2) <- insertOrGetExisting store testSource later req + pure $ conjoin + [ counterexample "resubmission after prune must create anew" (property created2) + , counterexample "eventIds must differ" (eventId ev1 =/= eventId ev2) + ] + +propCursorMonotonic :: Property +propCursorMonotonic = forAll (chooseInt (2, 16)) \n -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig Nothing Nothing) + now <- getCurrentTime + cursors <- traverse (\i -> do + req <- mkRequestFromParts (Text.pack ("evt-" <> show i)) Warning + (cursor, _, _) <- insertOrGetExisting store testSource now req + pure cursor) [1 .. n] + pure (cursors === List.sort cursors .&&. length cursors === length (List.nub cursors)) + +-- | Insert N events; configure retention to keep exactly K < N; after +-- 'pruneOnce', exactly K events remain. +propRetentionMaxEvents :: Property +propRetentionMaxEvents = + forAll (chooseInt (4, 12)) \n -> + forAll (chooseInt (1, n - 1)) \k -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig Nothing (Just (fromIntegral k))) + now <- getCurrentTime + _ <- traverse (\i -> do + req <- mkRequestFromParts (Text.pack ("evt-" <> show i)) Warning + insertOrGetExisting store testSource now req) [1 .. n] + pruneOnce store now + kept <- readHistory store Nothing 100 emptyAlarmFilter + pure (length kept === k) + +-- | An event received long ago is dropped by a maxAgeSeconds=1 prune. +propRetentionMaxAge :: Property +propRetentionMaxAge = forAll genShortText \sid -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig (Just 1) Nothing) + now <- getCurrentTime + let long_ago = addUTCTime (negate 3600) now + req <- mkRequestFromParts sid Warning + _ <- insertOrGetExisting store testSource long_ago req + pruneOnce store now + kept <- readHistory store Nothing 100 emptyAlarmFilter + pure (length kept === 0) + +-- | 'readHistory's 'after' parameter is exclusive: every returned +-- cursor is strictly greater than 'after'. +propHistoryAfterExclusive :: Property +propHistoryAfterExclusive = forAll (chooseInt (2, 8)) \n -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig Nothing Nothing) + now <- getCurrentTime + _ <- traverse (\i -> do + req <- mkRequestFromParts (Text.pack ("evt-" <> show i)) Warning + insertOrGetExisting store testSource now req) [1 .. n] + all_ <- readHistory store Nothing 100 emptyAlarmFilter + case all_ of + [] -> pure (property False) + ((c, _) : _) -> do + rest <- readHistory store (Just c) 100 emptyAlarmFilter + pure $ counterexample ("first cursor " <> show c <> " leaked into after-read") + (all ((> c) . fst) rest) + +-------------------------------------------------------------------------------- +-- Filter algebra properties +-------------------------------------------------------------------------------- + +-- | The empty filter accepts every event unconditionally. +propEmptyFilterAccepts :: Property +propEmptyFilterAccepts = forAll genEvent \ev -> + property (matchesFilter emptyAlarmFilter ev) + +-- | Concept doc: "requires the event to carry every key/value pair +-- listed here (a submap check), not just an intersection." Test: if +-- the filter demands a key the event does not have, the match must +-- fail even when the event has other scope entries. +propScopeIsSubmap :: Property +propScopeIsSubmap = + forAll genEvent \ev -> + forAll genLabels \extraScope -> + let evWithScope = ev { scope = Map.union (scope ev) extraScope } + filt = emptyAlarmFilter { afScope = Map.insert "not-in-event" "x" (scope evWithScope) } + in property (not (matchesFilter filt evWithScope)) + +-- | 'minSeverity' is inclusive: an event whose severity equals the +-- filter's minimum matches. +propMinSeverityInclusive :: Property +propMinSeverityInclusive = forAll genEvent \ev -> + let filt = emptyAlarmFilter { afMinSeverity = Just (severity ev) } + in property (matchesFilter filt ev) + +-- | Any filter narrows itself. Reflexivity is the base case of the +-- ceiling algebra: without it a reader cannot even keep its own +-- credential's filter. +propFilterNarrowsReflexive :: Property +propFilterNarrowsReflexive = forAll genFilter \f -> + property (filterNarrows f f) + +-- | If 'b' narrows 'a' and 'c' narrows 'b', then 'c' narrows 'a'. +-- Guarantees the ceiling relation composes across chained checks. +propFilterNarrowsTransitive :: Property +propFilterNarrowsTransitive = + forAll genFilter \a -> + forAll genFilter \b -> + forAll genFilter \c -> + filterNarrows a b && filterNarrows b c ==> + counterexample ("a=" <> show a <> " b=" <> show b <> " c=" <> show c) + (filterNarrows a c) + +-- | Broadening is rejected: given a ceiling that fixes 'source', a +-- requested filter that drops the source field or picks a different +-- value is not permitted. +propBroaderRejected :: Property +propBroaderRejected = + let ceiling_ = emptyAlarmFilter { afSource = Just (AlarmSource "trace") } + dropped = emptyAlarmFilter -- afSource = Nothing broadens the ceiling + differing = emptyAlarmFilter { afSource = Just (AlarmSource "somewhere-else") } + in conjoin + [ counterexample "dropping 'source' must be rejected" (not (filterNarrows ceiling_ dropped)) + , counterexample "differing 'source' value must be rejected" (not (filterNarrows ceiling_ differing)) + ] + +-------------------------------------------------------------------------------- +-- Trace-rule properties +-------------------------------------------------------------------------------- + +-- | One trace rule at Error threshold, 60s suppression window. testAlarmsConfig :: AlarmsConfig testAlarmsConfig = AlarmsConfig { alEndpoint = Endpoint "127.0.0.1" 0 Nothing @@ -88,69 +363,46 @@ mkTraceObject sev at = TraceObject , toThreadId = "1" } -propSeverityLowercase :: Property -propSeverityLowercase = once $ ioProperty do - pure (alarmSeverityToText Critical === "critical") - -propIdempotentSubmit :: Property -propIdempotentSubmit = once $ ioProperty do - store <- newAlarmStore (AlarmsRetentionConfig Nothing Nothing) - req <- mkRequest "evt-1" Warning - now <- getCurrentTime - (_, ev1, created1) <- insertOrGetExisting store testSource now req - (_, ev2, created2) <- insertOrGetExisting store testSource now req - pure (created1 .&&. not created2 .&&. eventId ev1 === eventId ev2) - -propConcurrentSubmit :: Property -propConcurrentSubmit = once $ ioProperty do - store <- newAlarmStore (AlarmsRetentionConfig Nothing Nothing) - req <- mkRequest "evt-concurrent" Warning - now <- getCurrentTime - results <- forConcurrently [1 .. (20 :: Int)] \_ -> - insertOrGetExisting store testSource now req - let createdCount = length (filter (\(_, _, c) -> c) results) - pure (createdCount === 1) - -propPruneThenResubmit :: Property -propPruneThenResubmit = once $ ioProperty do - store <- newAlarmStore (AlarmsRetentionConfig (Just 0) Nothing) -- prune anything immediately - req <- mkRequest "evt-pruned" Warning - now <- getCurrentTime - (_, ev1, _) <- insertOrGetExisting store testSource now req - later <- getCurrentTime - pruneOnce store later - (_, ev2, created2) <- insertOrGetExisting store testSource later req - pure (created2 .&&. eventId ev1 =/= eventId ev2) +-- | Round a UTCTime down to the nearest 'window'-second boundary. +snapToWindow :: Integer -> UTCTime -> UTCTime +snapToWindow window t = + let epoch = floor (utcTimeToPOSIXSeconds t) :: Integer + snap = (epoch `div` window) * window + in posixSecondsToUTCTime (fromIntegral snap) +-- | For any severity, the rule fires iff sev >= threshold. propTraceRuleThreshold :: Property -propTraceRuleThreshold = once $ ioProperty do +propTraceRuleThreshold = forAll genSeverity \sev -> ioProperty do registry <- newTestRegistry - now <- getCurrentTime - checkTraceObjectsForAlarms registry "node-1" - [mkTraceObject Info now, mkTraceObject Error now] + now <- getCurrentTime + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject sev now] events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter - pure $ case events of - [(_, ev)] -> severity ev === Error .&&. source ev === AlarmSource "trace" - _ -> counterexample ("expected exactly one event, got " <> show (length events)) False + let expected = if sev >= Error then 1 else 0 + pure (length events === expected) +-- | Two matches whose timestamps land in the same 60s window collapse +-- into one alarm regardless of their severities. Both timestamps are +-- taken relative to a window-aligned boundary so we can guarantee they +-- land in the same window index. propTraceRuleSuppression :: Property -propTraceRuleSuppression = once $ ioProperty do +propTraceRuleSuppression = ioProperty $ do registry <- newTestRegistry - now <- getCurrentTime - -- Same node, namespace, and timestamp: both matches land in the same - -- suppression window, whatever their severities are. - checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error now] - checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Critical now] + now <- getCurrentTime + let boundary = snapToWindow 60 now + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error boundary] + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Critical (addUTCTime 30 boundary)] events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter pure (length events === 1) +-- | A match one full window after another raises a fresh alarm. The +-- second timestamp is +60s from a window boundary, guaranteed to land +-- in the next window index. propTraceRuleNewWindow :: Property -propTraceRuleNewWindow = once $ ioProperty do +propTraceRuleNewWindow = ioProperty $ do registry <- newTestRegistry - now <- getCurrentTime - -- 120 s > the rule's 60 s window, so the window index always differs, - -- regardless of where 'now' falls within its window. - checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error now] - checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error (addUTCTime 120 now)] + now <- getCurrentTime + let boundary = snapToWindow 60 now + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error boundary] + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error (addUTCTime 60 boundary)] events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter pure (length events === 2) From 708fc4364a8321961d2c25390cc5a09eec5e3d78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Baldur=20Bl=C3=B6ndal?= Date: Wed, 26 Aug 2026 12:55:33 +0100 Subject: [PATCH 4/8] cardano-tracer: timeseries alarm rules --- cardano-tracer/cardano-tracer.cabal | 1 + .../src/Cardano/Tracer/Configuration.hs | 64 ++++- .../Tracer/Handlers/Alarms/TimeseriesRules.hs | 227 ++++++++++++++++++ .../test/Cardano/Tracer/Test/Alarms/Tests.hs | 17 +- 4 files changed, 292 insertions(+), 17 deletions(-) create mode 100644 cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs diff --git a/cardano-tracer/cardano-tracer.cabal b/cardano-tracer/cardano-tracer.cabal index 7a2239b06ff..02235bb6ca9 100644 --- a/cardano-tracer/cardano-tracer.cabal +++ b/cardano-tracer/cardano-tracer.cabal @@ -66,6 +66,7 @@ library Cardano.Tracer.Handlers.Alarms.Registry Cardano.Tracer.Handlers.Alarms.Server Cardano.Tracer.Handlers.Alarms.Store + Cardano.Tracer.Handlers.Alarms.TimeseriesRules Cardano.Tracer.Handlers.Alarms.TraceRules Cardano.Tracer.Handlers.Alarms.Types diff --git a/cardano-tracer/src/Cardano/Tracer/Configuration.hs b/cardano-tracer/src/Cardano/Tracer/Configuration.hs index 69eab453304..5e22edd5aae 100644 --- a/cardano-tracer/src/Cardano/Tracer/Configuration.hs +++ b/cardano-tracer/src/Cardano/Tracer/Configuration.hs @@ -17,6 +17,7 @@ module Cardano.Tracer.Configuration , AlarmsConsumerConfig (..) , AlarmsLimitsConfig (..) , AlarmsRetentionConfig (..) + , AlarmsTimeseriesRuleConfig (..) , AlarmsTraceRuleConfig (..) , Certificate (..) , Net.HowToConnect (..) @@ -216,8 +217,8 @@ data AlarmsRetentionConfig = AlarmsRetentionConfig deriving stock (Eq, Show, Generic) deriving anyclass (FromJSON, ToJSON) -data AlarmsLimitsConfig = AlarmsLimitsConfig - { alcMaxEventBytes :: !(Maybe Word64) +newtype AlarmsLimitsConfig = AlarmsLimitsConfig + { alcMaxEventBytes :: Maybe Word64 } deriving stock (Eq, Show, Generic) deriving anyclass (FromJSON, ToJSON) @@ -259,18 +260,63 @@ instance ToJSON AlarmsTraceRuleConfig where , ("labels" .=) <$> atrLabels ] +-- | A rule that periodically evaluates a @cardano-timeseries-io@ query and +-- raises an alarm when the query returns 'Truth' (or a truthy +-- 'InstantVector' entry) for at least @for@ seconds. The @sourceEventId@ +-- embeds the rule id and a canonical series key from the sample's labels +-- so per-series edges are deduplicated by the store. +data AlarmsTimeseriesRuleConfig = AlarmsTimeseriesRuleConfig + { atsRuleId :: !Text + , atsSummary :: !(Maybe Text) + , atsSeverity :: !SeverityS + , atsQuery :: !Text + , atsEvaluateEvery :: !Word64 -- ^ seconds between evaluations + , atsFor :: !(Maybe Word64) -- ^ seconds the sample must stay truthy before publishing + , atsRepeatEvery :: !(Maybe Word64) -- ^ seconds between reminder alarms while still true + , atsLabels :: !(Maybe (Map Text Text)) + } + deriving stock (Eq, Show, Generic) + +instance FromJSON AlarmsTimeseriesRuleConfig where + parseJSON = withObject "AlarmsTimeseriesRuleConfig" \o -> do + atsRuleId <- o .: "ruleId" + atsSummary <- o .:? "summary" + severityText <- o .: "severity" + atsSeverity <- maybe (fail ("unknown severity: " <> Text.unpack severityText)) pure + (parseAlarmSeverityText severityText) + atsQuery <- o .: "query" + atsEvaluateEvery <- o .: "evaluateEvery" + atsFor <- o .:? "for" + atsRepeatEvery <- o .:? "repeatEvery" + atsLabels <- o .:? "labels" + pure AlarmsTimeseriesRuleConfig{..} + +instance ToJSON AlarmsTimeseriesRuleConfig where + toJSON AlarmsTimeseriesRuleConfig{..} = object $ + [ "ruleId" .= atsRuleId + , "severity" .= alarmSeverityToText atsSeverity + , "query" .= atsQuery + , "evaluateEvery" .= atsEvaluateEvery + ] <> catMaybes + [ ("summary" .=) <$> atsSummary + , ("for" .=) <$> atsFor + , ("repeatEvery" .=) <$> atsRepeatEvery + , ("labels" .=) <$> atsLabels + ] + -- | Configuration for the alarm subsystem (see -- @cardano-tracer/docs/alarm-system-concept.md@). @Nothing@ for the -- enclosing 'Maybe' in 'TracerConfig' is the on/off switch for the whole -- subsystem; every field below is only meaningful once it's turned on. data AlarmsConfig = AlarmsConfig - { alEndpoint :: !Endpoint - , alAllowInsecure :: !(Maybe Bool) - , alRetention :: !(Maybe AlarmsRetentionConfig) - , alLimits :: !(Maybe AlarmsLimitsConfig) - , alAuthentication :: !AlarmsAuthConfig - , alConsumers :: ![AlarmsConsumerConfig] - , alTraceRules :: !(Maybe [AlarmsTraceRuleConfig]) + { alEndpoint :: !Endpoint + , alAllowInsecure :: !(Maybe Bool) + , alRetention :: !(Maybe AlarmsRetentionConfig) + , alLimits :: !(Maybe AlarmsLimitsConfig) + , alAuthentication :: !AlarmsAuthConfig + , alConsumers :: ![AlarmsConsumerConfig] + , alTraceRules :: !(Maybe [AlarmsTraceRuleConfig]) + , alTimeseriesRules :: !(Maybe [AlarmsTimeseriesRuleConfig]) } deriving stock (Eq, Show, Generic) deriving anyclass (FromJSON, ToJSON) diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs new file mode 100644 index 00000000000..6096ec4aa74 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs @@ -0,0 +1,227 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ScopedTypeVariables #-} + +-- | The timeseries-query alarm producer (concept-doc § Producers — +-- Timeseries rules). Periodically evaluates a @cardano-timeseries-io@ +-- boolean query against the in-process 'TimeseriesHandle' and, per +-- output series, runs a small edge-triggered state machine: +-- +-- false/missing → pending → publish +-- ^ | | +-- `--------- '--- false ---' +-- +-- Each output series has a deterministic key derived from its labels; +-- the @sourceEventId@ embeds that key so the alarm store dedupes +-- per-series edges. When @repeatEvery@ is configured, a still-true +-- series republishes at that interval with a fresh window-index in the +-- key. +-- +-- Errors and missing data are traced as health information; they never +-- publish false alarms. Query execution is bounded by a timeout so a +-- pathological rule cannot starve normal timeseries ingestion. +module Cardano.Tracer.Handlers.Alarms.TimeseriesRules + ( TimeseriesAlarmRule + , timeseriesAlarmSource + , timeseriesRuleFromConfig + , SamplePoint (..) + , evaluateOnce + , ruleRequests + , SeriesState (..) + ) where + +import Cardano.Logging.Types (SeverityS) +import Cardano.Tracer.Configuration (AlarmsTimeseriesRuleConfig (..)) +import Cardano.Tracer.Handlers.Alarms.Types + +import Data.Aeson (object, (.=)) +import Data.IORef (IORef, atomicModifyIORef', newIORef) +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Maybe (fromMaybe) +import qualified Data.Set as Set +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Time.Clock (UTCTime, diffUTCTime) +import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) +import Data.Word (Word64) + +-- | Runtime form of a timeseries rule. The mutable 'IORef' holds the +-- per-series state machines — inactive, pending-with-since, or +-- active-since-published. +data TimeseriesAlarmRule = TimeseriesAlarmRule + { tarRuleId :: !RuleId + , tarSummary :: !Text + , tarSeverity :: !SeverityS + , tarQuery :: !Text + , tarEvaluateEvery :: !Word64 -- ^ seconds + , tarFor :: !Word64 -- ^ seconds; 0 means "publish on first true" + , tarRepeatEvery :: !(Maybe Word64) -- ^ seconds between reminders while still true + , tarLabels :: !(Map Text Text) + , tarSeriesState :: !(IORef (Map SeriesKey SeriesState)) + } + +-- | Canonical, deterministic key derived from a sample's labels. Encoded +-- as @k1=v1,k2=v2@ with keys sorted so semantically equal maps produce +-- the same key. +newtype SeriesKey = SeriesKey { unSeriesKey :: Text } + deriving stock (Eq, Ord, Show) + +-- | Per-series state. +data SeriesState + = Inactive + -- ^ Last observed sample was false or missing. + | Pending !UTCTime + -- ^ Sample is truthy; waiting for the @for@ duration to elapse + -- before publishing. + | Active !UTCTime + -- ^ Sample stayed truthy for @for@ seconds and has been published + -- at least once; the field is @publishedAt@ (last publish time). + deriving stock (Eq, Show) + +-- | One boolean sample decoded from a query response, tagged with its +-- labels (the source of 'SeriesKey'). +data SamplePoint = SamplePoint + { spLabels :: !(Map Text Text) + , spTruth :: !Bool + } + deriving stock (Eq, Show) + +-- | The fixed trusted 'source' for alarms raised by timeseries rules. +-- Internal producer; never passes HTTP authentication. +timeseriesAlarmSource :: AlarmSource +timeseriesAlarmSource = AlarmSource "timeseries" + +-------------------------------------------------------------------------------- +-- Config → runtime +-------------------------------------------------------------------------------- + +timeseriesRuleFromConfig :: AlarmsTimeseriesRuleConfig -> IO TimeseriesAlarmRule +timeseriesRuleFromConfig AlarmsTimeseriesRuleConfig{..} = do + stateRef <- newIORef Map.empty + pure TimeseriesAlarmRule + { tarRuleId = RuleId atsRuleId + , tarSummary = fromMaybe defaultSummary atsSummary + , tarSeverity = atsSeverity + , tarQuery = atsQuery + , tarEvaluateEvery = max 1 atsEvaluateEvery + , tarFor = fromMaybe 0 atsFor + , tarRepeatEvery = atsRepeatEvery + , tarLabels = fromMaybe Map.empty atsLabels + , tarSeriesState = stateRef + } + where + defaultSummary = "Timeseries rule " <> atsRuleId <> " triggered" + +-------------------------------------------------------------------------------- +-- Series key +-------------------------------------------------------------------------------- + +-- | Deterministic canonical encoding of a label map. Sorted by key so +-- two 'Map's with the same content always produce the same key. +seriesKeyOf :: Map Text Text -> SeriesKey +seriesKeyOf labels = + SeriesKey (Text.intercalate "," [ k <> "=" <> v | (k, v) <- Map.toAscList labels ]) + +-------------------------------------------------------------------------------- +-- Evaluation → ingress requests +-------------------------------------------------------------------------------- + +-- | Apply one round of samples to the rule. Advances every touched +-- series through its state machine; returns one 'IngressRequest' per +-- series that just published (either a fresh edge or a scheduled +-- reminder). +-- +-- Series that appear as 'False' or as missing (not in the samples map) +-- transition to 'Inactive'. Series that appear as 'True' advance +-- 'Inactive → Pending' or, having sat in 'Pending' for at least +-- @tarFor@ seconds, transition to 'Active' and publish. +-- +-- Pure enough to be tested: takes @now@ as an argument, returns the new +-- 'SeriesState' map alongside the requests, and writes back to the +-- 'IORef' only in 'evaluateOnce'. +ruleRequests + :: TimeseriesAlarmRule + -> UTCTime -- ^ evaluation timestamp + -> [SamplePoint] -- ^ decoded query response + -> Map SeriesKey SeriesState -- ^ previous state + -> (Map SeriesKey SeriesState, [IngressRequest]) +ruleRequests rule@TimeseriesAlarmRule{tarFor, tarRepeatEvery} now samples prev = + let touched = Map.fromList [ (seriesKeyOf (spLabels sp), sp) | sp <- samples ] + allKeys = Set.toAscList (Map.keysSet touched <> Map.keysSet prev) + results = map + (\k -> advance k (Map.lookup k touched) (Map.findWithDefault Inactive k prev)) + allKeys + newState = Map.fromList [ (k, s) | (k, s, _) <- results ] + reqs = [ req | (_, _, Just req) <- results ] + in (newState, reqs) + where + advance :: SeriesKey + -> Maybe SamplePoint + -> SeriesState + -> (SeriesKey, SeriesState, Maybe IngressRequest) + advance key mSample state = case (state, isTrue mSample) of + (_, False) -> + (key, Inactive, Nothing) + (Inactive, True) -> + if tarFor == 0 + then let req = buildRequest rule key mSample now + in (key, Active now, Just req) + else (key, Pending now, Nothing) + (Pending since, True) -> + let elapsed = diffUTCTime now since + in if realToFrac elapsed >= (fromIntegral tarFor :: Double) + then let req = buildRequest rule key mSample now + in (key, Active now, Just req) + else (key, Pending since, Nothing) + (Active publishedAt, True) -> + case tarRepeatEvery of + Nothing -> (key, Active publishedAt, Nothing) + Just repeatSecs -> + let elapsed = diffUTCTime now publishedAt + in if realToFrac elapsed >= (fromIntegral repeatSecs :: Double) + then let req = buildRequest rule key mSample now + in (key, Active now, Just req) + else (key, Active publishedAt, Nothing) + + isTrue :: Maybe SamplePoint -> Bool + isTrue = maybe False spTruth + +buildRequest :: TimeseriesAlarmRule -> SeriesKey -> Maybe SamplePoint -> UTCTime -> IngressRequest +buildRequest TimeseriesAlarmRule{tarRuleId, tarSummary, tarSeverity, tarLabels} + seriesKey mSample now = + IngressRequest + { irSourceEventId = "ts:" <> unRuleId tarRuleId <> ":" <> unSeriesKey seriesKey + <> ":" <> Text.pack (show (windowIndex now)) + , irRaisedAt = now + , irRuleId = tarRuleId + , irSeverity = tarSeverity + , irSummary = tarSummary + , irScope = Map.empty + , irLabels = Map.union sampleLabels tarLabels + , irDetails = Just $ object + [ "seriesKey" .= unSeriesKey seriesKey + , "labels" .= sampleLabels + ] + } + where + sampleLabels = maybe Map.empty spLabels mSample + -- Millisecond-resolution window index so successive publishes (from + -- 'repeatEvery') always get distinct source-event-ids. + windowIndex :: UTCTime -> Integer + windowIndex t = floor (realToFrac (utcTimeToPOSIXSeconds t) * (1000 :: Double)) + +-- | 'IO' wrapper around 'ruleRequests': reads the previous per-series +-- state atomically, computes the new state, writes it back, and +-- returns the requests to submit. +evaluateOnce + :: TimeseriesAlarmRule + -> UTCTime + -> [SamplePoint] + -> IO [IngressRequest] +evaluateOnce rule now samples = + atomicModifyIORef' (tarSeriesState rule) \prev -> + let (newState, reqs) = ruleRequests rule now samples prev + in (newState, reqs) + diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs index c3cf4dd1756..e79c332939b 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs @@ -33,7 +33,7 @@ import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime) import Data.Time.Clock.POSIX (posixSecondsToUTCTime, utcTimeToPOSIXSeconds) import Test.Tasty -import Test.Tasty.QuickCheck +import Test.Tasty.QuickCheck hiding (labels) -------------------------------------------------------------------------------- -- Test suite @@ -329,13 +329,13 @@ propBroaderRejected = -- | One trace rule at Error threshold, 60s suppression window. testAlarmsConfig :: AlarmsConfig testAlarmsConfig = AlarmsConfig - { alEndpoint = Endpoint "127.0.0.1" 0 Nothing - , alAllowInsecure = Just True - , alRetention = Nothing - , alLimits = Nothing - , alAuthentication = AlarmsAuthConfig [] [] - , alConsumers = [] - , alTraceRules = Just + { alEndpoint = Endpoint "127.0.0.1" 0 Nothing + , alAllowInsecure = Just True + , alRetention = Nothing + , alLimits = Nothing + , alAuthentication = AlarmsAuthConfig [] [] + , alConsumers = [] + , alTraceRules = Just [ AlarmsTraceRuleConfig { atrRuleId = "error-traces" , atrSummary = Nothing @@ -344,6 +344,7 @@ testAlarmsConfig = AlarmsConfig , atrLabels = Nothing } ] + , alTimeseriesRules = Nothing } newTestRegistry :: IO AlarmRegistry From 154d3e47c8702b2fd1a90385ad0a2c843984874b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Baldur=20Bl=C3=B6ndal?= Date: Wed, 26 Aug 2026 13:54:36 +0100 Subject: [PATCH 5/8] cardano-tracer: timeseries alarm evaluator --- cardano-tracer/bench/cardano-tracer-bench.hs | 1 + .../Tracer/Handlers/Alarms/Registry.hs | 67 +++++++++++++++---- .../Tracer/Handlers/Alarms/TimeseriesRules.hs | 23 +++++++ .../src/Cardano/Tracer/MetaTrace.hs | 12 ++++ cardano-tracer/src/Cardano/Tracer/Run.hs | 3 + 5 files changed, 93 insertions(+), 13 deletions(-) diff --git a/cardano-tracer/bench/cardano-tracer-bench.hs b/cardano-tracer/bench/cardano-tracer-bench.hs index 29290cba4e4..a6d8e291983 100644 --- a/cardano-tracer/bench/cardano-tracer-bench.hs +++ b/cardano-tracer/bench/cardano-tracer-bench.hs @@ -61,6 +61,7 @@ main = do , teStateDir = Nothing , teMetricsHelp = [] , teTimeseriesHandle = Nothing + , teAlarmRegistry = Nothing } removePathForcibly root diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs index f57579d6228..1f4ac2c93af 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs @@ -13,43 +13,54 @@ module Cardano.Tracer.Handlers.Alarms.Registry , rejectEvent , checkTraceObjectsForAlarms , readHistoryFiltered + , runTimeseriesEvaluator , traceHistoryRead , lookupProducerCredential , lookupReaderCredential ) where import Cardano.Logging.Types (TraceObject) +import Cardano.Timeseries.AsText (asText) +import Cardano.Timeseries.Component (TimeseriesHandle, execute) import Cardano.Tracer.Configuration (AlarmsConfig (..), AlarmsRetentionConfig (..)) import Cardano.Tracer.Handlers.Alarms.Auth import Cardano.Tracer.Handlers.Alarms.Consumers import Cardano.Tracer.Handlers.Alarms.Store +import Cardano.Tracer.Handlers.Alarms.TimeseriesRules import Cardano.Tracer.Handlers.Alarms.TraceRules import Cardano.Tracer.Handlers.Alarms.Types import Cardano.Tracer.MetaTrace (TracerTrace (..), Trace, traceWith) +import Control.Concurrent (threadDelay) +import Control.Concurrent.Async (async, link) +import Control.Monad (forever) import Data.Foldable (for_) import Data.Maybe (fromMaybe, mapMaybe) import Data.Text (Text) import Data.Time.Clock (getCurrentTime) +import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) data AlarmRegistry = AlarmRegistry - { arStore :: !AlarmStoreHandle - , arConsumers :: ![AlarmConsumer] -- ^ static, fixed at startup - , arTraceRules :: ![TraceAlarmRule] - , arAuth :: !AuthTables - , arTracer :: !(Trace IO TracerTrace) + { arStore :: !AlarmStoreHandle + , arConsumers :: ![AlarmConsumer] -- ^ static, fixed at startup + , arTraceRules :: ![TraceAlarmRule] + , arTimeseriesRules :: ![TimeseriesAlarmRule] + , arAuth :: !AuthTables + , arTracer :: !(Trace IO TracerTrace) } newAlarmRegistry :: Trace IO TracerTrace -> AlarmsConfig -> IO AlarmRegistry -newAlarmRegistry tracer AlarmsConfig{alRetention, alConsumers, alAuthentication, alTraceRules} = do - store <- newAlarmStore (fromMaybe emptyRetention alRetention) - authTable <- loadCredentials alAuthentication +newAlarmRegistry tracer AlarmsConfig{alRetention, alConsumers, alAuthentication, alTraceRules, alTimeseriesRules} = do + store <- newAlarmStore (fromMaybe emptyRetention alRetention) + authTable <- loadCredentials alAuthentication + tsRules <- traverse timeseriesRuleFromConfig (fromMaybe [] alTimeseriesRules) pure AlarmRegistry - { arStore = store - , arConsumers = map consumerFromConfig alConsumers - , arTraceRules = map traceRuleFromConfig (fromMaybe [] alTraceRules) - , arAuth = authTable - , arTracer = tracer + { arStore = store + , arConsumers = map consumerFromConfig alConsumers + , arTraceRules = map traceRuleFromConfig (fromMaybe [] alTraceRules) + , arTimeseriesRules = tsRules + , arAuth = authTable + , arTracer = tracer } where emptyRetention = AlarmsRetentionConfig Nothing Nothing @@ -110,6 +121,36 @@ traceHistoryRead registry readerName resultCount = , ttAlarmHistoryResultCount = resultCount } +-- | Spawn one supervised thread per timeseries rule. Each ticks on the +-- rule's evaluateEvery interval and pushes any resulting alarms +-- through 'acceptEvent'. +runTimeseriesEvaluator :: AlarmRegistry -> TimeseriesHandle -> IO () +runTimeseriesEvaluator registry tsHandle = + for_ (arTimeseriesRules registry) \rule -> do + a <- async (evaluatorLoop registry tsHandle rule) + link a + +evaluatorLoop :: AlarmRegistry -> TimeseriesHandle -> TimeseriesAlarmRule -> IO () +evaluatorLoop registry tsHandle rule = forever $ do + threadDelay (fromIntegral (tarEvaluateEvery rule) * 1_000_000) + now <- getCurrentTime + result <- execute tsHandle (round (utcTimeToPOSIXSeconds now * 1000)) (tarQuery rule) + case result of + Left err -> + failed (asText err) + Right val -> case decodeSamples val of + Nothing -> failed "unexpected value shape from query" + Just samples -> do + reqs <- evaluateOnce rule now samples + for_ reqs \req -> acceptEvent registry timeseriesAlarmSource req + where + failed :: Text -> IO () + failed reason = + traceWith (arTracer registry) TracerAlarmTimeseriesEvalFailed + { ttAlarmTimeseriesEvalFailedRule = unRuleId (tarRuleId rule) + , ttAlarmTimeseriesEvalFailedReason = reason + } + lookupProducerCredential :: AlarmRegistry -> Text -> Maybe ProducerCredential lookupProducerCredential registry = lookupProducer (arAuth registry) diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs index 6096ec4aa74..3e7cd8d78f8 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs @@ -23,15 +23,21 @@ -- pathological rule cannot starve normal timeseries ingestion. module Cardano.Tracer.Handlers.Alarms.TimeseriesRules ( TimeseriesAlarmRule + , tarRuleId + , tarQuery + , tarEvaluateEvery , timeseriesAlarmSource , timeseriesRuleFromConfig , SamplePoint (..) + , decodeSamples , evaluateOnce , ruleRequests , SeriesState (..) ) where import Cardano.Logging.Types (SeverityS) +import Cardano.Timeseries.API (Value (..)) +import Cardano.Timeseries.Domain.Instant (Instant (..)) import Cardano.Tracer.Configuration (AlarmsTimeseriesRuleConfig (..)) import Cardano.Tracer.Handlers.Alarms.Types @@ -212,6 +218,23 @@ buildRequest TimeseriesAlarmRule{tarRuleId, tarSummary, tarSeverity, tarLabels} windowIndex :: UTCTime -> Integer windowIndex t = floor (realToFrac (utcTimeToPOSIXSeconds t) * (1000 :: Double)) +-- | Decode a query result into per-series boolean samples. Only three +-- shapes are meaningful; anything else is a shape error. +decodeSamples :: Value -> Maybe [SamplePoint] +decodeSamples val = case val of + Truth -> Just [SamplePoint Map.empty True] + Falsity -> Just [SamplePoint Map.empty False] + InstantVector v -> traverse decodeInstant v + _ -> Nothing + where + decodeInstant :: Instant Value -> Maybe SamplePoint + decodeInstant (Instant labels _ inner) = do + truth <- case inner of + Truth -> Just True + Falsity -> Just False + _ -> Nothing + pure (SamplePoint (Map.fromList (Set.toList labels)) truth) + -- | 'IO' wrapper around 'ruleRequests': reads the previous per-series -- state atomically, computes the new state, writes it back, and -- returns the requests to submit. diff --git a/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs b/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs index efa5fdec7d0..d0dab1076f7 100644 --- a/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs +++ b/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs @@ -110,6 +110,10 @@ data TracerTrace { ttAlarmHistoryReader :: Text , ttAlarmHistoryResultCount :: Int } + | TracerAlarmTimeseriesEvalFailed + { ttAlarmTimeseriesEvalFailedRule :: Text + , ttAlarmTimeseriesEvalFailedReason :: Text + } deriving Show -- | A bundle of domain-split tracers used in the application. @@ -242,6 +246,11 @@ instance LogFormatting TracerTrace where , "reader" .= ttAlarmHistoryReader , "resultCount" .= ttAlarmHistoryResultCount ] + TracerAlarmTimeseriesEvalFailed{..} -> mconcat + [ "kind" .= AE.String "TracerAlarmTimeseriesEvalFailed" + , "rule" .= ttAlarmTimeseriesEvalFailedRule + , "reason" .= ttAlarmTimeseriesEvalFailedReason + ] instance MetaTrace TracerTrace where namespaceFor TracerBuildInfo {} = Namespace [] ["BuildInfo"] @@ -272,6 +281,7 @@ instance MetaTrace TracerTrace where namespaceFor TracerAlarmRejected {} = Namespace [] ["AlarmRejected"] namespaceFor TracerAlarmDispatched {} = Namespace [] ["AlarmDispatched"] namespaceFor TracerAlarmHistoryRead {} = Namespace [] ["AlarmHistoryRead"] + namespaceFor TracerAlarmTimeseriesEvalFailed {} = Namespace [] ["AlarmTimeseriesEvalFailed"] severityFor (Namespace _ ["BuildInfo"]) _ = Just Info severityFor (Namespace _ ["ParamsAre"]) _ = Just Warning @@ -301,6 +311,7 @@ instance MetaTrace TracerTrace where severityFor (Namespace _ ["AlarmRejected"]) _ = Just Warning severityFor (Namespace _ ["AlarmDispatched"]) _ = Just Info severityFor (Namespace _ ["AlarmHistoryRead"]) _ = Just Info + severityFor (Namespace _ ["AlarmTimeseriesEvalFailed"]) _ = Just Warning severityFor _ _ = Nothing documentFor _ = Just "" @@ -334,6 +345,7 @@ instance MetaTrace TracerTrace where , Namespace [] ["AlarmRejected"] , Namespace [] ["AlarmDispatched"] , Namespace [] ["AlarmHistoryRead"] + , Namespace [] ["AlarmTimeseriesEvalFailed"] ] stderrShowTracer :: Show a => Trace IO a diff --git a/cardano-tracer/src/Cardano/Tracer/Run.hs b/cardano-tracer/src/Cardano/Tracer/Run.hs index db76810ebbe..d894f2cc3e7 100644 --- a/cardano-tracer/src/Cardano/Tracer/Run.hs +++ b/cardano-tracer/src/Cardano/Tracer/Run.hs @@ -95,6 +95,9 @@ doRunCardanoTracer config stateDir tr protocolsBrake dpRequestors = do !timeseriesHandle <- for (hasTimeseries config) (const $ Timeseries.create @(Tree _) tr.timeseries Nothing) !alarmRegistry <- for (alarms config) (Alarms.newAlarmRegistry tr.assorted) + case (alarmRegistry, timeseriesHandle) of + (Just reg, Just ts) -> Alarms.runTimeseriesEvaluator reg ts + _ -> pure () -- Environment for all following functions. let tracerEnv :: TracerEnv From 7d5d4a87867424e8cd74d8dbbdffd24cbf0fb900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Nicklisch-Franken?= Date: Fri, 28 Aug 2026 14:12:13 +0200 Subject: [PATCH 6/8] First alarm test cases --- cardano-node.code-workspace | 10 + cardano-tracer/cardano-tracer.cabal | 2 + cardano-tracer/docs/alarm-system-concept.md | 2 +- .../grafana-alerts-as-timeseries-rules.md | 143 +++++---- .../docs/timeseries-alarm-testing.md | 85 ++++-- .../Tracer/Handlers/Alarms/Registry.hs | 27 +- .../Cardano/Tracer/Handlers/Alarms/Store.hs | 24 +- .../Tracer/Handlers/Alarms/TimeseriesRules.hs | 22 +- .../test/Cardano/Tracer/Test/Alarms/Tests.hs | 37 ++- .../Tracer/Test/Alarms/TimeseriesTests.hs | 286 ++++++++++++++++++ cardano-tracer/test/cardano-tracer-test.hs | 2 + 11 files changed, 536 insertions(+), 104 deletions(-) create mode 100644 cardano-node.code-workspace create mode 100644 cardano-tracer/test/Cardano/Tracer/Test/Alarms/TimeseriesTests.hs diff --git a/cardano-node.code-workspace b/cardano-node.code-workspace new file mode 100644 index 00000000000..2f9eb356abb --- /dev/null +++ b/cardano-node.code-workspace @@ -0,0 +1,10 @@ +{ + "folders": [ + { + "path": "." + }, + { + "path": "../hermod-tracing" + } + ] +} \ No newline at end of file diff --git a/cardano-tracer/cardano-tracer.cabal b/cardano-tracer/cardano-tracer.cabal index 02235bb6ca9..f4a1e153a5d 100644 --- a/cardano-tracer/cardano-tracer.cabal +++ b/cardano-tracer/cardano-tracer.cabal @@ -288,6 +288,7 @@ test-suite cardano-tracer-test main-is: cardano-tracer-test.hs other-modules: Cardano.Tracer.Test.Alarms.Tests + Cardano.Tracer.Test.Alarms.TimeseriesTests Cardano.Tracer.Test.Forwarder Cardano.Tracer.Test.DataPoint.Tests Cardano.Tracer.Test.Logs.Tests @@ -298,6 +299,7 @@ test-suite cardano-tracer-test build-depends: aeson , async , bytestring + , cardano-timeseries-io , cardano-tracer , cborg , containers diff --git a/cardano-tracer/docs/alarm-system-concept.md b/cardano-tracer/docs/alarm-system-concept.md index 0824d60c891..b8186a1df11 100644 --- a/cardano-tracer/docs/alarm-system-concept.md +++ b/cardano-tracer/docs/alarm-system-concept.md @@ -485,7 +485,7 @@ The following choices do not block the architecture: 1. Which push consumers are required first (`webhook`, `email`, a message broker, or another integration). 2. Which embedded storage backend should implement `AlarmStore`. -3. Whether Hermod gains a native HTTP alarm sink immediately or initially uses an +3. Whether Hermod Recon gains a native HTTP alarm sink immediately or initially uses an adapter. 4. Whether configuration hot reload is needed. 5. Whether consumer retry queues must survive process restart in the first diff --git a/cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md b/cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md index 7c83e47737a..51042aec841 100644 --- a/cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md +++ b/cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md @@ -37,16 +37,17 @@ catalogue uses: | PromQL | timeseries query language | | --- | --- | -| `m[5m]` (range) | `m[now - 5m; now]` | +| instant selector `m` | `m now` — a metric is a *function of time* and must be applied | +| `m[5m]` (range) | `m[now - 5m; now]` (the metric stays unapplied inside a range) | | `m[360m:1m]` (subquery w/ step) | `m[now - 360m; now : 1m]` | -| `rate(...)`, `increase(...)` | `rate (...)`, `increase (...)` | +| `rate(...)`, `increase(...)` | `rate (...)`, `increase (...)` — **per millisecond**, not per second, and without counter-reset handling or window extrapolation | | `avg_over_time`, `sum_over_time` | same names | | `quantile_over_time(0.95, v)` | `quantile_over_time 0.95 v` | | `quantile by(environment) (0.2, v)` | `quantile_by ("environment") 0.2 v` | | `min`/`max`/`avg`/`abs`/`round` | same names | | `A unless B` | `unless A B` | -| `m{environment="mainnet"}` | `m{environment = "mainnet"}` (`=` and `!=` only) | -| `> bool`, `== bool` | plain `>`/`==` — comparisons are typed booleans already | +| `m{environment="mainnet"}` | `(m now){"environment" = "mainnet"}` (`=` and `!=` only; label keys are quoted) | +| `expr > k` (alert firing) | plain `>`/`==`/… on an instant vector is a **filter**, exactly like PromQL alert expressions: series failing the comparison drop out of the result, and the rule evaluator treats a series' *presence* as "condition true". Scalar comparisons yield real booleans. | | `and` / `or` (scalar logic) | `&&` / `||` | | — | extras: `let`/lambdas, `map`, `filter`, `join`, `to_scalar`, `earliest`, `latest`, `metrics` | @@ -54,12 +55,23 @@ Missing relative to PromQL: regex label matchers (`=~`, `!~`), vector-`or` as union/fallback, and cross-series binary matching (`on() group_right()`). Workarounds below. +Two further differences that bit during prototyping: `rate` values are per +millisecond, so PromQL thresholds on rates must be divided by 1000 (`> 0.5` +per second becomes `> 0.0005`; comparisons against `0` are unaffected), and +`rate` over a window that holds only a single sample is an evaluation error +(traced, never published as an alarm). Durations in the rule configuration +(`evaluateEvery`, `for`, `repeatEvery`) are plain integer **seconds**, not +Prometheus duration strings. + ## The catalogue, translated Severity suggestions replace the one-size-fits-all `page`. Metric names are the new-tracing names as forwarded by the node — verify against a live store with the `metrics` query before deploying (see caveats). +Durations (`evaluateEvery`, `for`) are plain integer seconds. Instant +selectors are applied to `now`; range windows take the metric unapplied. + ```yaml alarms: timeseriesRules: @@ -68,88 +80,93 @@ alarms: summary: "Blockheight unchanged for more than 7 minutes" severity: critical query: "rate (cardano_node_metrics_blockNum_int[now - 5m; now]) == 0" - evaluateEvery: 30s - for: 2m + evaluateEvery: 30 + for: 120 - ruleId: mempool-high # cardano_node_mempool_high summary: "More than 200 transactions in mempool for over 10 minutes" severity: warning - query: "cardano_node_metrics_txsInMempool_int > 200" - evaluateEvery: 30s - for: 10m + query: "cardano_node_metrics_txsInMempool_int now > 200" + evaluateEvery: 30 + for: 600 - ruleId: mempool-soft-timeouts # cardano_node_mempool_soft_timeouts_detected summary: "Mempool soft timeouts detected in the past hour" severity: warning query: "increase (cardano_node_metrics_txsMempoolTimeoutSoft_counter[now - 1h; now]) > 2" - evaluateEvery: 1m - for: 1m + evaluateEvery: 60 + for: 60 - ruleId: mempool-hard-timeouts # cardano_node_mempool_hard_timeouts_detected summary: "Mempool hard timeouts detected in the past hour" severity: error query: "increase (cardano_node_metrics_txsMempoolTimeoutHard_counter[now - 1h; now]) > 0" - evaluateEvery: 1m - for: 1m + evaluateEvery: 60 + for: 60 + # Needs prototyping before use: the bare comparison inside the subquery + # ((m != 0)[...]) does not elaborate -- a metric must be applied to a + # time, which inside a subquery requires a lambda. Like the divergence + # family below, verify against the real interpreter first. - ruleId: blockheight-metric-missing # cardano_node_metric_missing summary: "Blockheight metric missing for more than 10 minutes" severity: error query: >- unless (sum_over_time ((cardano_node_metrics_blockNum_int != 0)[now - 360m; now : 1m]) < 350) cardano_node_metrics_blockNum_int - evaluateEvery: 1m - for: 1m + evaluateEvery: 60 + for: 60 # --- cardano-node-forge.nix-import ------------------------------------- - ruleId: no-blocks-forged-24h # cardano_node_forge_blocks_missing summary: "No blocks forged in the past 24 hours" severity: critical query: "increase (cardano_node_metrics_blocksForged_int[now - 24h; now]) == 0" - evaluateEvery: 5m - for: 1m + evaluateEvery: 300 + for: 60 - ruleId: forged-not-adopted # cardano_node_forge_not_adopted_error summary: "Failed to adopt one or more forged blocks in the past hour" severity: error query: "increase (cardano_node_metrics_Forge_didnt_adopt_counter[now - 1h; now]) > 0" - evaluateEvery: 1m - for: 1m + evaluateEvery: 60 + for: 60 - ruleId: cannot-forge # cardano_node_cannot_forge_new_tracing summary: "Failed to forge one or more blocks in the past hour" severity: error query: "increase (cardano_node_metrics_nodeCannotForge_int[now - 1h; now]) > 0" - evaluateEvery: 1m - for: 1m + evaluateEvery: 60 + for: 60 - ruleId: slot-leadership-checks-missed # too_many_slot_leadership_checks_missed summary: "Slot leadership checks missed for more than half of slots" severity: critical - query: "rate (cardano_node_metrics_slotsMissed_int[now - 5m; now]) > 0.5" - evaluateEvery: 30s - for: 2m + # rate is per millisecond: PromQL's 0.5/s threshold becomes 0.0005/ms. + query: "rate (cardano_node_metrics_slotsMissed_int[now - 5m; now]) > 0.0005" + evaluateEvery: 30 + for: 120 # KES: the richer severity vocabulary replaces three identical "page" # alerts with an escalation ladder. - ruleId: kes-expiry-10-periods # cardano_node_KES_expiration_metric_10period_notice summary: "Less than 10 KES periods remaining" severity: warning - query: "cardano_node_metrics_remainingKESPeriods_int <= 10" - evaluateEvery: 5m - for: 5m + query: "cardano_node_metrics_remainingKESPeriods_int now <= 10" + evaluateEvery: 300 + for: 300 - ruleId: kes-expiry-5-periods summary: "Less than 5 KES periods remaining" severity: error - query: "cardano_node_metrics_remainingKESPeriods_int <= 5" - evaluateEvery: 5m - for: 5m + query: "cardano_node_metrics_remainingKESPeriods_int now <= 5" + evaluateEvery: 300 + for: 300 - ruleId: kes-expiry-1-period summary: "KES expires within 1 period" severity: critical - query: "cardano_node_metrics_remainingKESPeriods_int <= 1" - evaluateEvery: 5m - for: 5m + query: "cardano_node_metrics_remainingKESPeriods_int now <= 1" + evaluateEvery: 300 + for: 300 # --- cardano-node-network.nix-import ------------------------------------ # This one is the concept doc's own example rule. @@ -157,43 +174,43 @@ alarms: summary: "Average node ping latency above 500 ms" severity: warning query: "avg_over_time (netdata_statsd_cardano_node_ping_latency_ms_gauge_value_average[now - 5m; now]) > 500" - evaluateEvery: 30s - for: 60m + evaluateEvery: 30 + for: 3600 - ruleId: block-adoption-delay # blocks_adoption_delay_too_high summary: "95th-percentile block adoption delay above 4.5 s" severity: warning query: "avg (quantile_over_time 0.95 (cardano_node_metrics_blockfetchclient_blockdelay_real[now - 6h; now])) >= 4.5" - evaluateEvery: 5m - for: 1m + evaluateEvery: 300 + for: 60 - ruleId: block-utilization-high # blocks_utilization_too_high summary: "Average block utilization above 95%" severity: warning query: "100 * avg (avg_over_time (cardano_node_metrics_blockfetchclient_blocksize_int[now - 6h; now]) / 90112) > 95" - evaluateEvery: 5m - for: 5m + evaluateEvery: 300 + for: 300 - ruleId: blockfetch-delay-high # cardano_blockfetchclient_blockdelay_high summary: "Less than 90% of blocks arriving within 5 seconds" severity: warning - query: "cardano_node_metrics_blockfetchclient_blockdelay_cdfFive_real < 0.90" - evaluateEvery: 1m - for: 10m + query: "cardano_node_metrics_blockfetchclient_blockdelay_cdfFive_real now < 0.90" + evaluateEvery: 60 + for: 600 - ruleId: blockfetch-delay-critical # cardano_blockfetchclient_blockdelay_critical summary: "Less than 50% of blocks arriving within 5 seconds" severity: critical - query: "cardano_node_metrics_blockfetchclient_blockdelay_cdfFive_real < 0.50" - evaluateEvery: 1m - for: 10m + query: "cardano_node_metrics_blockfetchclient_blockdelay_cdfFive_real now < 0.50" + evaluateEvery: 60 + for: 600 - ruleId: connection-count-high # cardano_connection_count_high summary: "Incoming connection count above 450 (hard limit 512)" severity: warning - query: "cardano_node_metrics_connectionManager_inboundConns_int > 450" - evaluateEvery: 1m - for: 10m + query: "cardano_node_metrics_connectionManager_inboundConns_int now > 450" + evaluateEvery: 60 + for: 600 # --- cardano-node-quality.nix-import ------------------------------------ # Fleet-wide rule: only meaningful when the whole fleet forwards to this @@ -202,9 +219,9 @@ alarms: - ruleId: chain-density-degraded # chain_quality_degraded summary: "More than 20% of nodes below 70% chain density" severity: warning - query: '100 * quantile_by ("environment") 0.2 (cardano_node_metrics_density_real{environment != "preview"} * 20) < 70' - evaluateEvery: 1m - for: 5m + query: '100 * quantile_by ("environment") 0.2 ((cardano_node_metrics_density_real now){"environment" != "preview"} * 20) < 70' + evaluateEvery: 60 + for: 300 ``` ## Alerts that do not translate one-to-one @@ -259,12 +276,18 @@ fallback disappears. - **Testing.** Each ported rule should get a Level-1 test as described in `timeseries-alarm-testing.md`: insert samples that straddle the threshold and the `for` window, evaluate at fixed timestamps, assert exactly one - alarm. - -## Prerequisite - -All of this targets the Phase-2 timeseries rule evaluator from the concept -doc, which is not yet implemented. The value of this exercise now: the -catalogue above is a realistic acceptance suite for that evaluator — if these -rules can be configured and fire correctly, Phase 2 is functionally complete -for SRE use. + alarm. Three rules of this catalogue — `mempool-high`, + `blockheight-unchanged`, and `high-ping-latency` — already have such + tests (`Cardano.Tracer.Test.Alarms.TimeseriesTests`), one alarming and + one quiet dataset each; they are the template for porting the rest. + +## Status + +The Phase-2 timeseries rule evaluator from the concept doc is implemented +(`Cardano.Tracer.Handlers.Alarms.TimeseriesRules`, driven by +`runTimeseriesEvaluator`). The catalogue above is its acceptance suite — +three rules are verified by the example tests; the queries of the remaining +rules follow the same corrected conventions but should each get their +Level-1 test before being relied on. The two families flagged above +(divergence, `blockheight-metric-missing`) still need prototyping against +the interpreter. diff --git a/cardano-tracer/docs/timeseries-alarm-testing.md b/cardano-tracer/docs/timeseries-alarm-testing.md index 0443d37a40f..ca27966f11d 100644 --- a/cardano-tracer/docs/timeseries-alarm-testing.md +++ b/cardano-tracer/docs/timeseries-alarm-testing.md @@ -38,34 +38,83 @@ rule logic (thresholds, `for` durations, edge-triggering, repeat intervals). propHighLatencyRaisesAlarm :: Property propHighLatencyRaisesAlarm = once $ ioProperty do bundle <- mkTraceBundle (SeverityF (Just Warning)) - handle <- Timeseries.create @(Tree _) (timeseries bundle) Nothing + handle <- Timeseries.create @(Tree Double) (timeseries bundle) (Just noPruneConfig) registry <- newAlarmRegistry (assorted bundle) testAlarmsConfig - -- 1. Arrange: five samples, 30 s apart, all above the threshold + -- 1. Arrange: insert all samples up front (queries only look backward) let t0 = 1_700_000_000_000 -- fixed ms timestamp series = Set.fromList [("node_name", "node-1")] - for_ [0 .. 4] \i -> + for_ [0 .. 10] \i -> Timeseries.insert handle series (t0 + i * 30_000) [("ping_latency_ms", 640)] - -- 2. Act: evaluate the rule once, at a fixed time - -- (evaluateTimeseriesRules is the Phase-2 evaluator entry point; - -- design it to take the evaluation timestamp as an argument!) - evaluateTimeseriesRules registry handle (t0 + 5 * 30_000) + -- 2. Act: evaluate all configured rules at fixed instants, one round + -- per simulated 30 s (the state machine needs several rounds to + -- cross the rule's `for` duration) + for_ [1 .. 10] \k -> + evaluateTimeseriesRules registry handle + (posixSecondsToUTCTime (fromIntegral (t0 `div` 1000 + 30 * k))) -- 3. Assert: exactly one alarm in the store events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter pure (length events === 1) ``` -Design rule for the evaluator that follows from this: **pass the evaluation -time in explicitly** (like `pruneOnce` and `traceRuleRequest` already do). -Anything that calls `getCurrentTime` internally cannot be tested -deterministically. - -Until the evaluator exists, the same pattern already works for testing query -semantics alone: insert known samples, call `execute handle at query`, and -assert on the returned `Value` (thresholds crossing true/false, missing data, -window boundaries). +`evaluateTimeseriesRules :: AlarmRegistry -> TimeseriesHandle -> UTCTime -> IO ()` +(and its single-rule sibling `evaluateTimeseriesRule`) in +`Cardano.Tracer.Handlers.Alarms.Registry` **take the evaluation time +explicitly** (like `pruneOnce` and `traceRuleRequest` already do); only the +production `evaluatorLoop` supplies `getCurrentTime`. This is what makes the +rounds above deterministic. + +The same pattern also works for testing query semantics alone: insert known +samples, call `execute handle at query`, and assert on the returned `Value` +(thresholds crossing true/false, missing data, window boundaries). + +Determinism pitfalls, learned the hard way (all verified against +`cardano-timeseries-io`): + +- **Create the store with pruning disabled** (`pruningPeriodMillis = + Nothing`). The pruner compares against the real wall clock and silently + deletes fixed historical test timestamps. +- **Instant lookups have a 300 s staleness bound** — a rule evaluated at `t` + sees a series only if it has a sample in `(t - 300 s, t]`, so test data + must keep sampling at least that often across the whole horizon. +- **Keep timestamps large.** The Tree store's staleness-window arithmetic is + on `Word64` and underflows for query times below 300 000 ms. +- **`rate` has no value on a single-point series** (hard error, traced as an + evaluation failure). Start evaluation rounds only after the range window + contains at least two populated grid points. +- **A bare metric is a function of time**: instant threshold rules must + apply it, `m now > 200`; range forms `m[now - 5m; now]` take the metric + unapplied. +- **Comparisons on instant vectors are filters**, not boolean vectors: the + result keeps the surviving series with their original values, and the + evaluator treats series *presence* as "condition holds" (exactly PromQL's + alert semantics). + +## Example test cases + +`Cardano.Tracer.Test.Alarms.TimeseriesTests` instantiates this pattern with +three rules ported from the Grafana catalogue +(`grafana-alerts-as-timeseries-rules.md`), each driven once with data that +must not raise an alarm and once with data that must raise exactly one — six +cases total. Samples arrive every 30 s from `t0`, one evaluation round per +30 simulated seconds starting at round 1, and the history is read back +through `readHistoryFiltered` for inspection. + +| Rule (query construct) | Quiet case — why no alarm | Alarm case — publish round | +| --- | --- | --- | +| `mempool-high` — `cardano_node_metrics_txsInMempool_int now > 200`, for 600 s | mempool hovers at 120–180: expression never true | constant 250: true from round 1, `for` satisfied at round 21; extra rounds prove the edge publishes once | +| `blockheight-unchanged` — `rate (cardano_node_metrics_blockNum_int[now - 5m; now]) == 0`, for 120 s | chain grows by 1 block per sample: rate > 0, series filtered out every round | constant block height: rate exactly 0, publishes at round 5 with severity `critical` | +| `high-ping-latency` — `avg_over_time (…ping_latency…[now - 5m; now]) > 500`, for 3600 s | 10 min spike at 600 ms, then recovery to 100 ms: expression true far shorter than `for`, pending state resets | constant 800 ms for 62 min: publishes at round 121 | + +The three quiet cases deliberately cover the three distinct ways a rule stays +silent: the expression is never true (mempool), the series is filtered out of +the result (blockheight), and the expression is true for less than the `for` +duration before recovering (ping latency). The alarm cases assert exactly one +event and inspect its `ruleId`, `severity`, `source` (`timeseries`), the +`node_name` label carried over from the series key, and the +per-series `sourceEventId` prefix (`ts::node_name=node-1:`). ## Level 2: end-to-end with a launched `cardano-tracer` @@ -98,12 +147,12 @@ alarms: - name: test-reader tokenFile: "reader.token" # written by the test setup allowHistory: true - timeseriesRules: # Phase 2, not yet implemented + timeseriesRules: - ruleId: high-latency summary: "latency above threshold" severity: warning query: "avg_over_time (ping_latency_ms[now - 5m; now]) > 500" - evaluateEvery: 1s # keep the test fast + evaluateEvery: 1 # seconds; keep the test fast consumers: [] ``` diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs index 1f4ac2c93af..d5bd75e45be 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs @@ -1,4 +1,5 @@ {-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-} -- | The central alarm registry: static consumers (fixed at startup) sitting @@ -14,6 +15,8 @@ module Cardano.Tracer.Handlers.Alarms.Registry , checkTraceObjectsForAlarms , readHistoryFiltered , runTimeseriesEvaluator + , evaluateTimeseriesRule + , evaluateTimeseriesRules , traceHistoryRead , lookupProducerCredential , lookupReaderCredential @@ -27,7 +30,8 @@ import Cardano.Tracer.Handlers.Alarms.Auth import Cardano.Tracer.Handlers.Alarms.Consumers import Cardano.Tracer.Handlers.Alarms.Store import Cardano.Tracer.Handlers.Alarms.TimeseriesRules -import Cardano.Tracer.Handlers.Alarms.TraceRules +import Cardano.Tracer.Handlers.Alarms.TraceRules (TraceAlarmRule, traceAlarmSource, + traceRuleFromConfig, traceRuleRequest) import Cardano.Tracer.Handlers.Alarms.Types import Cardano.Tracer.MetaTrace (TracerTrace (..), Trace, traceWith) @@ -37,7 +41,7 @@ import Control.Monad (forever) import Data.Foldable (for_) import Data.Maybe (fromMaybe, mapMaybe) import Data.Text (Text) -import Data.Time.Clock (getCurrentTime) +import Data.Time.Clock (UTCTime, getCurrentTime) import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) data AlarmRegistry = AlarmRegistry @@ -133,7 +137,18 @@ runTimeseriesEvaluator registry tsHandle = evaluatorLoop :: AlarmRegistry -> TimeseriesHandle -> TimeseriesAlarmRule -> IO () evaluatorLoop registry tsHandle rule = forever $ do threadDelay (fromIntegral (tarEvaluateEvery rule) * 1_000_000) - now <- getCurrentTime + now <- getCurrentTime + evaluateTimeseriesRule registry tsHandle now rule + +-- | Evaluate one rule at an explicit timestamp: run its query against +-- the store, decode the samples, advance the per-series state +-- machines, and push any resulting alarms through 'acceptEvent'. +-- Query and shape failures are traced as evaluation failures, never +-- published as alarms. The evaluation time is an argument (rather +-- than 'getCurrentTime') so tests can drive rules deterministically; +-- the production 'evaluatorLoop' passes the current time. +evaluateTimeseriesRule :: AlarmRegistry -> TimeseriesHandle -> UTCTime -> TimeseriesAlarmRule -> IO () +evaluateTimeseriesRule registry tsHandle now rule = do result <- execute tsHandle (round (utcTimeToPOSIXSeconds now * 1000)) (tarQuery rule) case result of Left err -> @@ -151,6 +166,12 @@ evaluatorLoop registry tsHandle rule = forever $ do , ttAlarmTimeseriesEvalFailedReason = reason } +-- | Evaluate every configured timeseries rule once at the given +-- timestamp. +evaluateTimeseriesRules :: AlarmRegistry -> TimeseriesHandle -> UTCTime -> IO () +evaluateTimeseriesRules registry tsHandle now = + for_ (arTimeseriesRules registry) (evaluateTimeseriesRule registry tsHandle now) + lookupProducerCredential :: AlarmRegistry -> Text -> Maybe ProducerCredential lookupProducerCredential registry = lookupProducer (arAuth registry) diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Store.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Store.hs index 67dc7ce2187..ff6c4e2f260 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Store.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Store.hs @@ -43,9 +43,12 @@ data AlarmStoreHandle = AlarmStoreHandle , ashWake :: !(MVar ()) -- ^ unused for now (no dynamic reconfiguration in this sketch); kept so the pruner loop matches Cardano.Timeseries.Component.create's shape } --- | Prune every 60 seconds when any retention limit is configured; otherwise --- the pruner thread just parks, mirroring --- @Cardano.Timeseries.Component.create@'s pruner. +-- | Prune every 60 seconds when any retention limit is configured. With no +-- retention there is nothing to prune, ever, and no pruner is started: a +-- thread parked forever on 'ashWake' would be killed by the RTS's deadlock +-- detector once the handle becomes garbage, and 'link' would then rethrow +-- into whatever the creating thread is doing by that time (observed as +-- spurious failures in unrelated tests). pruneIntervalMicros :: Int pruneIntervalMicros = 60 * 1000 * 1000 @@ -57,17 +60,16 @@ newAlarmStore retention = do wake <- newEmptyMVar startTag <- Text.pack . show <$> getTimeMs let handle = AlarmStoreHandle events index nextSeq startTag retention wake - async (runPruner handle) >>= link + case (arcMaxAgeSeconds retention, arcMaxEvents retention) of + (Nothing, Nothing) -> pure () + _ -> async (runPruner handle) >>= link pure handle where runPruner :: AlarmStoreHandle -> IO () - runPruner handle = forever $ - case (arcMaxAgeSeconds (ashRetention handle), arcMaxEvents (ashRetention handle)) of - (Nothing, Nothing) -> takeMVar (ashWake handle) -- no retention configured; nothing to do, ever - _ -> do - now <- getCurrentTime - pruneOnce handle now - race_ (threadDelay pruneIntervalMicros) (takeMVar (ashWake handle)) + runPruner handle = forever $ do + now <- getCurrentTime + pruneOnce handle now + race_ (threadDelay pruneIntervalMicros) (takeMVar (ashWake handle)) -- | Insert a new event, or return the already-accepted event for a replayed -- @(source, sourceEventId)@ pair without dispatching it again. One STM diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs index 3e7cd8d78f8..391d576df07 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs @@ -218,8 +218,19 @@ buildRequest TimeseriesAlarmRule{tarRuleId, tarSummary, tarSeverity, tarLabels} windowIndex :: UTCTime -> Integer windowIndex t = floor (realToFrac (utcTimeToPOSIXSeconds t) * (1000 :: Double)) --- | Decode a query result into per-series boolean samples. Only three --- shapes are meaningful; anything else is a shape error. +-- | Decode a query result into per-series boolean samples. +-- +-- Three top-level shapes are meaningful; anything else is a shape +-- error. Inside an instant vector two element shapes are accepted: +-- +-- * 'Truth'\/'Falsity' — produced by explicitly boolean queries such +-- as @map (\\x -> x > 200) (m now)@; +-- * 'Scalar' — produced by comparison filters such as @m now > 200@, +-- which keep only the series satisfying the relation. Presence of +-- a series in the result means the condition holds for it, exactly +-- like a PromQL alert expression, so a surviving 'Scalar' decodes +-- as true. Series filtered out are absent from the vector and are +-- treated as missing (false) by the state machine. decodeSamples :: Value -> Maybe [SamplePoint] decodeSamples val = case val of Truth -> Just [SamplePoint Map.empty True] @@ -230,9 +241,10 @@ decodeSamples val = case val of decodeInstant :: Instant Value -> Maybe SamplePoint decodeInstant (Instant labels _ inner) = do truth <- case inner of - Truth -> Just True - Falsity -> Just False - _ -> Nothing + Truth -> Just True + Falsity -> Just False + Scalar _ -> Just True + _ -> Nothing pure (SamplePoint (Map.fromList (Set.toList labels)) truth) -- | 'IO' wrapper around 'ruleRequests': reads the previous per-series diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs index e79c332939b..d62bcfc38a9 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs @@ -298,16 +298,41 @@ propFilterNarrowsReflexive :: Property propFilterNarrowsReflexive = forAll genFilter \f -> property (filterNarrows f f) +-- | Generate a filter that narrows the given ceiling: fixed single-value +-- constraints are kept verbatim, open ones may become fixed, the minimum +-- severity may only rise, and the scope/label maps may only gain entries. +genNarrowing :: AlarmFilter -> Gen AlarmFilter +genNarrowing ceiling_ = do + src <- keepOrFix (afSource ceiling_) genSource + rid <- keepOrFix (afRuleId ceiling_) genRule + sev <- case afMinSeverity ceiling_ of + Nothing -> oneof [pure Nothing, Just <$> genSeverity] + Just s -> Just <$> genSeverity `suchThat` (>= s) + -- left-biased union: every ceiling entry survives with its value + scope <- Map.union (afScope ceiling_) <$> genLabels + lbls <- Map.union (afLabels ceiling_) <$> genLabels + pure (AlarmFilter src rid sev scope lbls) + where + keepOrFix :: Maybe a -> Gen a -> Gen (Maybe a) + keepOrFix (Just c) _ = pure (Just c) + keepOrFix Nothing gen = oneof [pure Nothing, Just <$> gen] + -- | If 'b' narrows 'a' and 'c' narrows 'b', then 'c' narrows 'a'. --- Guarantees the ceiling relation composes across chained checks. +-- Guarantees the ceiling relation composes across chained checks. The +-- chain is generated by construction ('genNarrowing') — random +-- independent filters almost never narrow each other, so an '==>' +-- precondition would discard virtually every case. propFilterNarrowsTransitive :: Property propFilterNarrowsTransitive = forAll genFilter \a -> - forAll genFilter \b -> - forAll genFilter \c -> - filterNarrows a b && filterNarrows b c ==> - counterexample ("a=" <> show a <> " b=" <> show b <> " c=" <> show c) - (filterNarrows a c) + forAll (genNarrowing a) \b -> + forAll (genNarrowing b) \c -> + counterexample ("a=" <> show a <> " b=" <> show b <> " c=" <> show c) $ + conjoin + [ counterexample "generator must narrow: b vs a" (filterNarrows a b) + , counterexample "generator must narrow: c vs b" (filterNarrows b c) + , counterexample "transitivity violated: c vs a" (filterNarrows a c) + ] -- | Broadening is rejected: given a ceiling that fixes 'source', a -- requested filter that drops the source field or picks a different diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/TimeseriesTests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/TimeseriesTests.hs new file mode 100644 index 00000000000..d283d263914 --- /dev/null +++ b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/TimeseriesTests.hs @@ -0,0 +1,286 @@ +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | Example test cases for timeseries alarm rules: three rules ported +-- from the Grafana catalogue in +-- @cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md@, each +-- driven once with data that must NOT raise an alarm and once with data +-- that MUST raise exactly one. +-- +-- These are Level-1 tests in the sense of +-- @cardano-tracer/docs/timeseries-alarm-testing.md@: no server and no +-- forwarder — samples are inserted at fixed millisecond timestamps and +-- the rules are evaluated at fixed instants through +-- 'evaluateTimeseriesRules', so every case is fully deterministic. The +-- whole production pipeline below the evaluator loop is exercised: +-- query execution against the store, sample decoding, the per-series +-- @for@ state machine, ingress via the alarm registry, and the history +-- read used for inspection. +module Cardano.Tracer.Test.Alarms.TimeseriesTests + ( tests + ) where + +import Cardano.Logging (SeverityF (..), SeverityS (..)) +import Cardano.Timeseries.API (Config (..), Tree) +import Cardano.Timeseries.Component (TimeseriesConfig (..)) +import qualified Cardano.Timeseries.Component as Timeseries +import Cardano.Tracer.Configuration +import Cardano.Tracer.Handlers.Alarms.Registry +import Cardano.Tracer.Handlers.Alarms.Types +import Cardano.Tracer.MetaTrace (TraceBundle (..), mkTraceBundle) + +import Data.Foldable (for_) +import qualified Data.Map.Strict as Map +import qualified Data.Set as Set +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Time.Clock (UTCTime) +import Data.Time.Clock.POSIX (posixSecondsToUTCTime) +import Data.Word (Word64) + +import Test.Tasty +import Test.Tasty.QuickCheck hiding (labels) + +tests :: TestTree +tests = testGroup "Test.Alarms.Timeseries" + [ testGroup "mempool-high" + [ testProperty "mempool below threshold raises no alarm" propMempoolQuiet + , testProperty "sustained high mempool raises exactly one alarm" propMempoolAlarm + ] + , testGroup "blockheight-unchanged" + [ testProperty "growing chain raises no alarm" propBlockheightQuiet + , testProperty "stalled blockheight raises exactly one alarm" propBlockheightAlarm + ] + , testGroup "high-ping-latency" + [ testProperty "latency spike shorter than for raises no alarm" propPingQuiet + , testProperty "sustained high latency raises exactly one alarm" propPingAlarm + ] + ] + +-------------------------------------------------------------------------------- +-- Fixture +-------------------------------------------------------------------------------- + +-- | Fixed base timestamp (ms since epoch). Must stay far above the +-- store's 300 s staleness window: the Tree store's window arithmetic +-- is on 'Word64' and underflows for query times below it. +t0 :: Word64 +t0 = 1_700_000_000_000 + +-- | Every sample belongs to one node, labelled the way the acceptor +-- labels forwarded metrics ("Cardano.Tracer.Acceptors.Utils"). +testSeries :: Set.Set (Text, Text) +testSeries = Set.fromList [("node_name", "node-1")] + +-- | Store configuration for tests. The pruner compares against the real +-- wall clock, so the retention window must be large enough that its +-- cutoff (now - retention) stays below the fixed 2023-era sample +-- timestamps — but small enough that the 'Word64' subtraction cannot +-- wrap. 10^12 ms (~32 years) satisfies both for decades. The pruning +-- period must be 'Just': with 'Nothing' the pruner parks forever on an +-- MVar, and once the abandoned test handle becomes garbage the RTS's +-- deadlock detector kills it, rethrowing through 'link' into whatever +-- test runs later. +testTimeseriesConfig :: TimeseriesConfig +testTimeseriesConfig = TimeseriesConfig + { retentionMillis = 1_000_000_000_000 + , pruningPeriodMillis = Just (24 * 60 * 60 * 1000) + , interpCfg = Config { defaultRangeSamplingRateMillis = 15_000 } + } + +mkAlarmsConfig :: AlarmsTimeseriesRuleConfig -> AlarmsConfig +mkAlarmsConfig rule = AlarmsConfig + { alEndpoint = Endpoint "127.0.0.1" 0 Nothing + , alAllowInsecure = Just True + , alRetention = Nothing + , alLimits = Nothing + , alAuthentication = AlarmsAuthConfig [] [] + , alConsumers = [] + , alTraceRules = Nothing + , alTimeseriesRules = Just [rule] + } + +-- | Evaluation timestamp of round @k@: 30 simulated seconds per round +-- after 't0'. +roundTime :: Word64 -> UTCTime +roundTime k = posixSecondsToUTCTime (fromIntegral (t0 `div` 1000 + 30 * k)) + +-- | One sample per 30 s starting at 't0': offsets and values for +-- indices @0 .. count-1@. +samplesEvery30s :: Word64 -> (Word64 -> Double) -> [(Word64, Double)] +samplesEvery30s count valueAt = [ (i * 30_000, valueAt i) | i <- [0 .. count - 1] ] + +-- | Build a store and a registry holding one rule, insert all samples +-- of one metric up front (every query looks strictly backward, so +-- later samples are invisible to earlier rounds), evaluate the rule +-- at rounds @k = 1 .. rounds@, and return the alarm history. +-- +-- Rounds start at @k = 1@: at @k = 0@ a range window holds a single +-- populated grid point and @rate@ has no defined value there. +runScenario + :: AlarmsTimeseriesRuleConfig + -> Text -- ^ metric name + -> [(Word64, Double)] -- ^ samples: (ms after 't0', value) + -> Word64 -- ^ number of 30 s evaluation rounds + -> IO [(AlarmCursor, AlarmEvent)] +runScenario rule metric samples rounds = do + bundle <- mkTraceBundle (SeverityF (Just Warning)) + handle <- Timeseries.create @(Tree Double) (timeseries bundle) (Just testTimeseriesConfig) + registry <- newAlarmRegistry (assorted bundle) (mkAlarmsConfig rule) + for_ samples \(offset, value) -> + Timeseries.insert handle testSeries (t0 + offset) [(metric, value)] + for_ [1 .. rounds] \k -> + evaluateTimeseriesRules registry handle (roundTime k) + readHistoryFiltered registry Nothing 100 emptyAlarmFilter + +-------------------------------------------------------------------------------- +-- Assertions +-------------------------------------------------------------------------------- + +expectNoAlarm :: [(AlarmCursor, AlarmEvent)] -> Property +expectNoAlarm events = + counterexample ("unexpected alarms: " <> show (map snd events)) + (length events === 0) + +-- | Exactly one alarm, carrying the rule identity, the severity, the +-- trusted source, the series label, and a per-series source event id. +expectOneAlarm :: AlarmsTimeseriesRuleConfig -> [(AlarmCursor, AlarmEvent)] -> Property +expectOneAlarm rule events = case events of + [(_, ev)] -> conjoin + [ counterexample "ruleId" (ruleId ev === RuleId (atsRuleId rule)) + , counterexample "severity" (severity ev === atsSeverity rule) + , counterexample "source" (source ev === AlarmSource "timeseries") + , counterexample "summary" (Just (summary ev) === atsSummary rule) + , counterexample "node_name label" + (Map.lookup "node_name" (labels ev) === Just "node-1") + , counterexample ("sourceEventId: " <> show (sourceEventId ev)) + (property (("ts:" <> atsRuleId rule <> ":node_name=node-1:") + `Text.isPrefixOf` sourceEventId ev)) + ] + _ -> counterexample ("expected exactly one alarm, got: " <> show (map snd events)) + (property False) + +-------------------------------------------------------------------------------- +-- mempool-high: instant threshold +-------------------------------------------------------------------------------- + +-- | Catalogue rule @cardano_node_mempool_high@. Note the applied @now@: +-- a bare metric is a function of time in the query language. +mempoolRule :: AlarmsTimeseriesRuleConfig +mempoolRule = AlarmsTimeseriesRuleConfig + { atsRuleId = "mempool-high" + , atsSummary = Just "More than 200 transactions in mempool for over 10 minutes" + , atsSeverity = Warning + , atsQuery = "cardano_node_metrics_txsInMempool_int now > 200" + , atsEvaluateEvery = 30 + , atsFor = Just 600 + , atsRepeatEvery = Nothing + , atsLabels = Nothing + } + +mempoolMetric :: Text +mempoolMetric = "cardano_node_metrics_txsInMempool_int" + +-- | The mempool hovers between 120 and 180 transactions — always below +-- the threshold, so the expression is never true. +propMempoolQuiet :: Property +propMempoolQuiet = once $ ioProperty do + events <- runScenario mempoolRule mempoolMetric + (samplesEvery30s 25 \i -> 120 + fromIntegral (i `mod` 3) * 30) + 24 + pure (expectNoAlarm events) + +-- | The mempool sits at 250 transactions for the whole horizon. The +-- expression turns true at round 1, satisfies @for@ = 600 s at round +-- 21, and the three extra rounds prove the edge stays published only +-- once. +propMempoolAlarm :: Property +propMempoolAlarm = once $ ioProperty do + events <- runScenario mempoolRule mempoolMetric + (samplesEvery30s 25 (const 250)) + 24 + pure (expectOneAlarm mempoolRule events) + +-------------------------------------------------------------------------------- +-- blockheight-unchanged: rate over a range window +-------------------------------------------------------------------------------- + +-- | Catalogue rule @cardano_node_blockheight_unchanged@. The range form +-- needs no applied @now@; @== 0@ keeps exactly the series whose block +-- height did not move inside the 5 min window. +blockheightRule :: AlarmsTimeseriesRuleConfig +blockheightRule = AlarmsTimeseriesRuleConfig + { atsRuleId = "blockheight-unchanged" + , atsSummary = Just "Blockheight unchanged for more than 7 minutes" + , atsSeverity = Critical + , atsQuery = "rate (cardano_node_metrics_blockNum_int[now - 5m; now]) == 0" + , atsEvaluateEvery = 30 + , atsFor = Just 120 + , atsRepeatEvery = Nothing + , atsLabels = Nothing + } + +blockheightMetric :: Text +blockheightMetric = "cardano_node_metrics_blockNum_int" + +-- | The chain grows by one block per sample, so the rate over the +-- window is strictly positive and the comparison filters the series +-- out of the result on every round. +propBlockheightQuiet :: Property +propBlockheightQuiet = once $ ioProperty do + events <- runScenario blockheightRule blockheightMetric + (samplesEvery30s 17 \i -> 1000 + fromIntegral i) + 16 + pure (expectNoAlarm events) + +-- | The block height never moves: the rate is exactly 0 on every round, +-- and after @for@ = 120 s (round 5) exactly one critical alarm is +-- published. +propBlockheightAlarm :: Property +propBlockheightAlarm = once $ ioProperty do + events <- runScenario blockheightRule blockheightMetric + (samplesEvery30s 17 (const 1000)) + 16 + pure (expectOneAlarm blockheightRule events) + +-------------------------------------------------------------------------------- +-- high-ping-latency: avg_over_time over a range window +-------------------------------------------------------------------------------- + +-- | Catalogue rule @high_cardano_ping_latency@ (also the concept doc's +-- own example rule). +pingRule :: AlarmsTimeseriesRuleConfig +pingRule = AlarmsTimeseriesRuleConfig + { atsRuleId = "high-ping-latency" + , atsSummary = Just "Average node ping latency above 500 ms" + , atsSeverity = Warning + , atsQuery = "avg_over_time (netdata_statsd_cardano_node_ping_latency_ms_gauge_value_average[now - 5m; now]) > 500" + , atsEvaluateEvery = 30 + , atsFor = Just 3600 + , atsRepeatEvery = Nothing + , atsLabels = Nothing + } + +pingMetric :: Text +pingMetric = "netdata_statsd_cardano_node_ping_latency_ms_gauge_value_average" + +-- | A 10-minute latency spike (600 ms), then recovery to 100 ms. The +-- expression is true while the spike dominates the trailing 5 min +-- average — far shorter than @for@ = 60 min — then turns false and +-- resets the pending state, so nothing may ever be published. +propPingQuiet :: Property +propPingQuiet = once $ ioProperty do + events <- runScenario pingRule pingMetric + (samplesEvery30s 61 \i -> if i < 20 then 600 else 100) + 60 + pure (expectNoAlarm events) + +-- | Latency stays at 800 ms for 62 minutes. The expression is true from +-- round 1 on, satisfies @for@ = 3600 s at round 121, and exactly one +-- warning alarm is published. +propPingAlarm :: Property +propPingAlarm = once $ ioProperty do + events <- runScenario pingRule pingMetric + (samplesEvery30s 125 (const 800)) + 124 + pure (expectOneAlarm pingRule events) diff --git a/cardano-tracer/test/cardano-tracer-test.hs b/cardano-tracer/test/cardano-tracer-test.hs index ef1cd63e354..c3db8dcdcba 100644 --- a/cardano-tracer/test/cardano-tracer-test.hs +++ b/cardano-tracer/test/cardano-tracer-test.hs @@ -2,6 +2,7 @@ {-# OPTIONS_GHC -Wno-unused-matches #-} import qualified Cardano.Tracer.Test.Alarms.Tests as Alarms +import qualified Cardano.Tracer.Test.Alarms.TimeseriesTests as AlarmsTimeseries import qualified Cardano.Tracer.Test.DataPoint.Tests as DataPoint import qualified Cardano.Tracer.Test.Logs.Tests as Logs import Cardano.Tracer.Test.TestSetup @@ -49,6 +50,7 @@ main = do [ Logs.tests ts , DataPoint.tests ts , Alarms.tests + , AlarmsTimeseries.tests -- , Restart.tests ts -- , Queue.tests ts ]) From 8f28919bec89b5e60768b69b5be39bf9f90f37ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Nicklisch-Franken?= Date: Wed, 16 Sep 2026 10:29:33 +0200 Subject: [PATCH 7/8] cardano-tracer: opt-in unauthenticated ("open") alarm producer Let a trusted local producer submit alarm events to POST /alarms/v1/events without a bearer token. The alarm design doc already anticipates hermod-recon as an external producer feeding this endpoint via a small adapter process; for that trusted-local-network case the token mechanism is pure overhead. A producer credential's tokenFile is now optional. A producer with no token file is an "open producer" and is honoured only for requests that carry no Authorization header at all. Two guards keep this an explicit, visible choice rather than a silent weakening of the default posture: - an open producer is rejected by config validation unless alAllowInsecure is true (the existing "this endpoint is not locked down" flag); - at most one open producer may be configured, so an unauthenticated request never maps ambiguously. A request that does carry an Authorization header still goes through the normal token lookup and is rejected if the token is unknown; it never falls back to the open producer, so a mistyped or stale token cannot be mistaken for an anonymous submission. The producer's source identity continues to come from config (pcSource), never from the request. Configs with no tokenless producer behave exactly as before. cardano-tracer: wire alarms-smoke.sh into the cabal test suite New test-suite cardano-tracer-alarms-smoke runs the HTTP smoke test against a freshly-started cardano-tracer via build-tool-depends. Co-Authored-By: Claude Opus 4.7 --- .../Node/Tracing/DefaultTraceConfig.hs | 2 - cardano-node/src/Cardano/Node/Tracing/Span.hs | 140 ------------- .../src/Cardano/Node/Tracing/Tracers.hs | 12 -- cardano-tracer/cardano-tracer.cabal | 21 ++ .../configuration/alarms-example.json | 38 ++++ cardano-tracer/docs/alarm-system-concept.md | 12 ++ .../src/Cardano/Tracer/Configuration.hs | 89 +++++++- .../Cardano/Tracer/Handlers/Alarms/Auth.hs | 42 +++- .../Tracer/Handlers/Alarms/Consumers.hs | 193 +++++++++++++++--- .../Tracer/Handlers/Alarms/Registry.hs | 50 +++-- .../Cardano/Tracer/Handlers/Alarms/Server.hs | 19 +- .../Tracer/Handlers/Alarms/TimeseriesRules.hs | 63 +++++- .../test/Cardano/Tracer/Test/Alarms/Tests.hs | 162 +++++++++++++++ cardano-tracer/test/alarms-smoke.hs | 13 ++ cardano-tracer/test/alarms-smoke.sh | 57 ++++++ 15 files changed, 691 insertions(+), 222 deletions(-) delete mode 100644 cardano-node/src/Cardano/Node/Tracing/Span.hs create mode 100644 cardano-tracer/configuration/alarms-example.json create mode 100644 cardano-tracer/test/alarms-smoke.hs create mode 100755 cardano-tracer/test/alarms-smoke.sh diff --git a/cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs b/cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs index 181ffdcf283..43453aaebc8 100644 --- a/cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs +++ b/cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs @@ -26,8 +26,6 @@ defaultCardanoConfig = emptyTraceConfig { [ ConfSeverity (SeverityF Nothing)]) ,(["ChainDB"], [ ConfSeverity (SeverityF (Just Info))]) - ,(["Span"], - [ ConfSeverity (SeverityF (Just Info))]) ,(["ChainDB", "AddBlockEvent", "AddBlockValidation"], [ ConfSeverity (SeverityF Nothing)]) ,(["ChainSync", "Client"], diff --git a/cardano-node/src/Cardano/Node/Tracing/Span.hs b/cardano-node/src/Cardano/Node/Tracing/Span.hs deleted file mode 100644 index e43a18914d6..00000000000 --- a/cardano-node/src/Cardano/Node/Tracing/Span.hs +++ /dev/null @@ -1,140 +0,0 @@ -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE OverloadedStrings #-} - --- | OpenTelemetry-style spans on top of the @trace-dispatcher@ framework. --- --- A span is a pair of correlated trace messages — 'SpanBegin' and 'SpanEnd' — --- sharing a single 'SpanId'. 'withSpan' brackets an action so that the end --- message is always emitted, even on exception, which is what lets a --- Loki-style alerting system fire on \"span that never ended\". --- --- The duration is measured /client-side/ (in this process, monotonic clock) --- and shipped inside 'SpanEnd', so the consumer (cardano-tracer / timeseries / --- Prometheus) stays stateless — it never has to pair begin\/end itself. --- --- * For __timeseries\/Prometheus__: 'asMetrics' emits @spanDurationMs@. --- * For __Loki (LogQL)__: 'forMachine' emits a flat JSON object with a --- stable @span_id@ (log field, high cardinality) and @name@ --- (label-friendly, low cardinality) plus @event=begin|end@. -module Cardano.Node.Tracing.Span - ( SpanId (..) - , SpanTrace (..) - , newSpanId - , withSpan - ) where - -import Cardano.Logging - -import Control.Exception.Safe (MonadMask, finally) -import Control.Monad.IO.Class (MonadIO, liftIO) -import Data.Aeson (Value (String), (.=)) -import Data.Text (Text) -import Data.Unique (hashUnique, newUnique) -import Data.Word (Word64) -import GHC.Clock (getMonotonicTimeNSec) - --- | Correlation id shared by a 'SpanBegin' / 'SpanEnd' pair. --- Process-unique for the lifetime of the run (see 'newSpanId'). -newtype SpanId = SpanId { unSpanId :: Word64 } - deriving (Eq, Ord, Show) - --- | The two ends of a span. -data SpanTrace - = SpanBegin !SpanId !Text - -- ^ Start of a span: id + human name of the operation. - | SpanEnd !SpanId !Text !Double - -- ^ End of a span: id, name, and measured duration in milliseconds. - deriving (Show) - --- | Allocate a fresh, process-unique span id. --- --- Uses 'Data.Unique' so it needs no extra dependency and never blocks. Ids --- are unique within a single node run; they are /not/ stable across restarts --- (fine for correlating one begin with one end — which is all we need). -newSpanId :: MonadIO m => m SpanId -newSpanId = liftIO (SpanId . fromIntegral . hashUnique <$> newUnique) - --- | Run @action@ inside a span, emitting 'SpanBegin' before and 'SpanEnd' --- after — even if @action@ throws. --- --- @ --- withSpan tr \"replayLedger\" $ do --- ...work... --- @ -withSpan - :: (MonadIO m, MonadMask m) - => Trace m SpanTrace -- ^ where to emit the span messages - -> Text -- ^ human name of the operation - -> m a -- ^ the work to measure - -> m a -withSpan tr name action = do - sid <- newSpanId - !t0 <- liftIO getMonotonicTimeNSec - traceWith tr (SpanBegin sid name) - action `finally` do - !t1 <- liftIO getMonotonicTimeNSec - let !ms = fromIntegral (t1 - t0) / 1e6 :: Double - traceWith tr (SpanEnd sid name ms) - --------------------------------------------------------------------------------- --- Formatting --------------------------------------------------------------------------------- - -instance LogFormatting SpanTrace where - forMachine _ (SpanBegin sid name) = - mconcat - [ "kind" .= String "SpanBegin" - , "event" .= String "begin" - , "span_id" .= unSpanId sid - , "name" .= name - ] - forMachine _ (SpanEnd sid name ms) = - mconcat - [ "kind" .= String "SpanEnd" - , "event" .= String "end" - , "span_id" .= unSpanId sid - , "name" .= name - , "duration_ms" .= ms - ] - - forHuman (SpanBegin sid name) = - "Span begin [" <> showT (unSpanId sid) <> "] " <> name - forHuman (SpanEnd sid name ms) = - "Span end [" <> showT (unSpanId sid) <> "] " <> name - <> " (" <> showT ms <> " ms)" - - -- Only the end carries a measurement. The metric name embeds the span name so - -- distinct operations are distinguishable; keep the set of names SMALL to - -- avoid Prometheus/timeseries cardinality blow-up. - asMetrics (SpanBegin _ _) = [] - asMetrics (SpanEnd _ name ms) = - [ DoubleM ("spanDurationMs." <> name) ms ] - --------------------------------------------------------------------------------- --- Documentation / metadata --------------------------------------------------------------------------------- - -instance MetaTrace SpanTrace where - namespaceFor SpanBegin{} = Namespace [] ["Span", "Begin"] - namespaceFor SpanEnd{} = Namespace [] ["Span", "End"] - - severityFor (Namespace _ ["Span", "Begin"]) _ = Just Info - severityFor (Namespace _ ["Span", "End"]) _ = Just Info - severityFor _ _ = Nothing - - documentFor (Namespace _ ["Span", "Begin"]) = Just - "Start of a correlated span. Carries a span_id shared with the matching \ - \Span.End, and the human name of the operation." - documentFor (Namespace _ ["Span", "End"]) = Just - "End of a correlated span. Carries the same span_id as Span.Begin plus \ - \the client-side measured duration in milliseconds." - documentFor _ = Nothing - - metricsDocFor (Namespace _ ["Span", "End"]) = - [("spanDurationMs", "Client-side measured span duration, in milliseconds")] - metricsDocFor _ = [] - - allNamespaces = - [ Namespace [] ["Span", "Begin"] - , Namespace [] ["Span", "End"] - ] diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers.hs index a0c5df9a199..9d131d4ef89 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers.hs @@ -36,7 +36,6 @@ import Cardano.Node.Tracing.Tracers.NodeVersion (getNodeVersion) import Cardano.Node.Tracing.Tracers.Rpc () import Cardano.Node.Tracing.Tracers.Shutdown () import Cardano.Node.Tracing.Tracers.Startup () -import Cardano.Node.Tracing.Span (withSpan) import Ouroboros.Consensus.Ledger.Inspect (LedgerEvent) import Ouroboros.Consensus.MiniProtocol.ChainSync.Client (TraceChainSyncClientEvent) import qualified Ouroboros.Consensus.Network.NodeToClient as NodeToClient @@ -157,12 +156,6 @@ mkDispatchTracers nodeKernel trBase trForward mbTrEKG trDataPoint trConfig forgi !rpcTr <- mkCardanoTracer trBase trForward mbTrEKG ["RPC"] configureTracers configReflection trConfig [rpcTr] - -- Span tracer: emits correlated Span.Begin / Span.End pairs. Registered - -- through the standard pipeline so spans are forwarded to cardano-tracer - -- (and thus visible to Loki) and exposed on Prometheus/EKG as metrics. - !spanTr <- mkCardanoTracer trBase trForward mbTrEKG [] - configureTracers configReflection trConfig [spanTr] - traceTracerInfo trBase trForward configReflection let warnings = checkNodeTraceConfiguration' trConfig @@ -173,11 +166,6 @@ mkDispatchTracers nodeKernel trBase trForward mbTrEKG trDataPoint trConfig forgi traceWith nodeVersionTr getNodeVersion - -- Demonstration span emitted once at tracer initialisation. This proves the - -- begin/end pair flows through the pipeline; replace with real withSpan - -- calls around the operations you want to measure. - withSpan spanTr "nodeTracersInit" (pure ()) - pure Tracers { chainDBTracer = mkTracer (traceWith chainDBTr') diff --git a/cardano-tracer/cardano-tracer.cabal b/cardano-tracer/cardano-tracer.cabal index f4a1e153a5d..47f65a8ab8b 100644 --- a/cardano-tracer/cardano-tracer.cabal +++ b/cardano-tracer/cardano-tracer.cabal @@ -15,6 +15,7 @@ license-files: LICENSE build-type: Simple extra-doc-files: README.md CHANGELOG.md +data-files: test/alarms-smoke.sh flag systemd description: Enable systemd support. @@ -30,6 +31,7 @@ common project-config , GeneralizedNewtypeDeriving , ImportQualifiedPost , InstanceSigs + , NumericUnderscores , ScopedTypeVariables , StandaloneKindSignatures , TypeApplications @@ -131,6 +133,8 @@ library , ekg-wai , extra , filepath + , http-client + , http-client-tls , http-types , mime-mail , network @@ -391,6 +395,23 @@ test-suite cardano-tracer-test-ext -rtsopts -with-rtsopts=-N +test-suite cardano-tracer-alarms-smoke + import: project-config + type: exitcode-stdio-1.0 + hs-source-dirs: test + main-is: alarms-smoke.hs + + other-modules: Paths_cardano_tracer + autogen-modules: Paths_cardano_tracer + + build-tool-depends: cardano-tracer:cardano-tracer + + if os(windows) + buildable: False + + build-depends: base + , process + benchmark cardano-tracer-bench import: project-config type: exitcode-stdio-1.0 diff --git a/cardano-tracer/configuration/alarms-example.json b/cardano-tracer/configuration/alarms-example.json new file mode 100644 index 00000000000..00d5b0ce5fb --- /dev/null +++ b/cardano-tracer/configuration/alarms-example.json @@ -0,0 +1,38 @@ +{ + "networkMagic": 764824073, + "network": { + "tag": "AcceptAt", + "contents": "/tmp/forwarder.sock" + }, + "logging": [ + { + "logRoot": "/tmp/cardano-tracer-logs", + "logMode": "FileMode", + "logFormat": "ForMachine" + } + ], + "alarms": { + "alEndpoint": { + "epHost": "127.0.0.1", + "epPort": 1234 + }, + "alAllowInsecure": true, + "alAuthentication": { + "aacProducers": [ + { + "pcName": "hermod-bridge", + "pcTokenFile": null, + "pcSource": "hermod-recon" + } + ], + "aacReaders": [ + { + "rcName": "local-reader", + "rcTokenFile": "/tmp/cardano-tracer-alarms-demo/reader.token", + "rcAllowHistory": true + } + ] + }, + "alConsumers": [] + } +} diff --git a/cardano-tracer/docs/alarm-system-concept.md b/cardano-tracer/docs/alarm-system-concept.md index b8186a1df11..c4307986d1c 100644 --- a/cardano-tracer/docs/alarm-system-concept.md +++ b/cardano-tracer/docs/alarm-system-concept.md @@ -378,6 +378,18 @@ delivery when retries are enabled. - Refuse to start an externally reachable clear-text endpoint unless an explicit insecure setting is enabled. +A producer credential may omit `tokenFile` entirely, making it an "open +producer": ingress requests that carry no `Authorization` header at all are +attributed to it with no authentication whatsoever. This is only accepted +when `allowInsecure: true`, and at most one such producer may be configured +(otherwise an unauthenticated request would be ambiguous as to which producer +it belongs to); any request that does carry an `Authorization` header is +still resolved by the usual token lookup, and an unrecognised token is still +rejected. This relaxation is intended for a producer that runs on +infrastructure already trusted for the alarm endpoint's operator -- the same +trust model this document already accepts for node-to-tracer forwarding, +which has no encryption or authentication either. + The first version does not require a remote administration API. Rules, consumers, and credentials are managed through configuration and take effect on restart. Hot reload can be considered separately. diff --git a/cardano-tracer/src/Cardano/Tracer/Configuration.hs b/cardano-tracer/src/Cardano/Tracer/Configuration.hs index 5e22edd5aae..99f03dd1aab 100644 --- a/cardano-tracer/src/Cardano/Tracer/Configuration.hs +++ b/cardano-tracer/src/Cardano/Tracer/Configuration.hs @@ -7,6 +7,13 @@ {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} +-- 'AlarmsConsumerConfig' is a sum type whose two constructors define disjoint +-- record fields (acl* for 'AlarmsConsumerLog', acw* for 'AlarmsConsumerWebhook'). +-- Every use in this module is via 'RecordWildCards'\/'NamedFieldPuns' pattern +-- binding, never as a selector function, so the generated partial selectors +-- are harmless here. +{-# OPTIONS_GHC -Wno-partial-fields #-} + {- HLINT ignore "Use any" -} module Cardano.Tracer.Configuration @@ -36,6 +43,7 @@ module Cardano.Tracer.Configuration , alarmSeverityToText , parseAlarmSeverityText , readTracerConfig + , wellFormed ) where import Cardano.Logging.Types (HowToConnect, SeverityS (..)) @@ -53,7 +61,7 @@ import Data.List.Extra (notNull) import Data.List.NonEmpty (NonEmpty) import qualified Data.List.NonEmpty as NE import Data.Map.Strict (Map) -import Data.Maybe (catMaybes, fromMaybe) +import Data.Maybe (catMaybes, fromMaybe, isNothing) import Data.String (fromString) import Data.Text (Text) import qualified Data.Text as Text @@ -155,11 +163,21 @@ instance ToJSON AlarmFilterConfig where -- | A statically configured alarm dispatch target. The only variant -- implemented so far is @log@; @webhook@/@email@ are the obvious follow-up -- extension (see @Cardano.Tracer.Handlers.Alarms.Consumers@). -data AlarmsConsumerConfig = AlarmsConsumerLog - { aclName :: !Text - , aclEnabled :: !Bool - , aclFilter :: !(Maybe AlarmFilterConfig) - } +data AlarmsConsumerConfig + = AlarmsConsumerLog + { aclName :: !Text + , aclEnabled :: !Bool + , aclFilter :: !(Maybe AlarmFilterConfig) + } + | AlarmsConsumerWebhook + { acwName :: !Text + , acwEnabled :: !Bool + , acwFilter :: !(Maybe AlarmFilterConfig) + , acwUrl :: !Text -- ^ POST target + , acwBearerToken :: !(Maybe Text) -- ^ optional; sent as "Authorization: Bearer …" + , acwQueueCap :: !(Maybe Int) -- ^ bounded buffer (default 1000) + , acwMaxBackoff :: !(Maybe Int) -- ^ retry backoff cap, seconds (default 60) + } deriving stock (Eq, Show, Generic) instance FromJSON AlarmsConsumerConfig where @@ -170,6 +188,14 @@ instance FromJSON AlarmsConsumerConfig where <$> o .: "name" <*> (fromMaybe True <$> o .:? "enabled") <*> o .:? "filter" + "webhook" -> AlarmsConsumerWebhook + <$> o .: "name" + <*> (fromMaybe True <$> o .:? "enabled") + <*> o .:? "filter" + <*> o .: "url" + <*> o .:? "bearerToken" + <*> o .:? "queueCapacity" + <*> o .:? "maxBackoffSecs" other -> fail ("unknown alarm consumer type: " <> Text.unpack other) instance ToJSON AlarmsConsumerConfig where @@ -179,13 +205,31 @@ instance ToJSON AlarmsConsumerConfig where , "enabled" .= aclEnabled , "filter" .= aclFilter ] + toJSON AlarmsConsumerWebhook{..} = object $ + [ "type" .= ("webhook" :: Text) + , "name" .= acwName + , "enabled" .= acwEnabled + , "url" .= acwUrl + ] <> catMaybes + [ ("filter" .=) <$> acwFilter + , ("bearerToken" .=) <$> acwBearerToken + , ("queueCapacity" .=) <$> acwQueueCap + , ("maxBackoffSecs" .=) <$> acwMaxBackoff + ] -- | A producer credential: a bearer token (read once from @pcTokenFile@) that -- authenticates alarm ingress requests as coming from @pcSource@. The -- source is never taken from the request body itself. +-- +-- @pcTokenFile = Nothing@ (the YAML field omitted, or explicit @null@) +-- designates the "open producer": ingress requests carrying no +-- @Authorization@ header at all are attributed to it, with no token check. +-- This is opt-in and deliberately narrow -- see 'wellFormed', which allows +-- at most one open producer, and only when @alAllowInsecure@ is @Just +-- True@. data ProducerCredentialConfig = ProducerCredentialConfig { pcName :: !Text - , pcTokenFile :: !FilePath + , pcTokenFile :: !(Maybe FilePath) , pcSource :: !Text } deriving stock (Eq, Show, Generic) @@ -315,8 +359,11 @@ data AlarmsConfig = AlarmsConfig , alLimits :: !(Maybe AlarmsLimitsConfig) , alAuthentication :: !AlarmsAuthConfig , alConsumers :: ![AlarmsConsumerConfig] - , alTraceRules :: !(Maybe [AlarmsTraceRuleConfig]) - , alTimeseriesRules :: !(Maybe [AlarmsTimeseriesRuleConfig]) + , alTraceRules :: !(Maybe [AlarmsTraceRuleConfig]) + , alTimeseriesRules :: !(Maybe [AlarmsTimeseriesRuleConfig]) + , alTimeseriesStatePath :: !(Maybe FilePath) + -- ^ JSON file where per-rule series state is persisted across + -- restarts. 'Nothing' disables persistence. } deriving stock (Eq, Show, Generic) deriving anyclass (FromJSON, ToJSON) @@ -457,6 +504,8 @@ wellFormed TracerConfig , check "no host in alarms endpoint" . nullEndpoint . alEndpoint =<< alarms , check "alarms: no producer or reader credentials configured" . noCredentials =<< alarms , check "alarms: duplicate consumer names" . hasDuplicateConsumerNames =<< alarms + , check "alarms: more than one open (tokenless) producer configured" . hasMultipleOpenProducers =<< alarms + , check "alarms: open (tokenless) producer requires allowInsecure: true" . openProducerWithoutAllowInsecure =<< alarms ] -- NB. every internal service's endpoint port is included here, including @@ -474,12 +523,32 @@ wellFormed TracerConfig noCredentials AlarmsConfig{alAuthentication = AlarmsAuthConfig{aacProducers, aacReaders}} = null aacProducers && null aacReaders + -- | At most one producer may omit @tokenFile@: with more than one, an + -- unauthenticated request would be ambiguous as to which producer's + -- source it should be attributed to. + hasMultipleOpenProducers :: AlarmsConfig -> Bool + hasMultipleOpenProducers AlarmsConfig{alAuthentication = AlarmsAuthConfig{aacProducers}} = + length (filter isOpenProducer aacProducers) > 1 + + -- | An open (tokenless) producer relaxes the alarm ingress endpoint's + -- default deny-by-default authentication, so it is only accepted once an + -- operator has explicitly acknowledged that with @allowInsecure: true@ -- + -- the same flag that already gates running the alarms HTTP server + -- without TLS. + openProducerWithoutAllowInsecure :: AlarmsConfig -> Bool + openProducerWithoutAllowInsecure AlarmsConfig{alAuthentication = AlarmsAuthConfig{aacProducers}, alAllowInsecure} = + any isOpenProducer aacProducers && alAllowInsecure /= Just True + + isOpenProducer :: ProducerCredentialConfig -> Bool + isOpenProducer ProducerCredentialConfig{pcTokenFile} = isNothing pcTokenFile + hasDuplicateConsumerNames :: AlarmsConfig -> Bool hasDuplicateConsumerNames AlarmsConfig{alConsumers} = hasDuplicates (map consumerName alConsumers) consumerName :: AlarmsConsumerConfig -> Text - consumerName AlarmsConsumerLog{aclName} = aclName + consumerName AlarmsConsumerLog{aclName} = aclName + consumerName AlarmsConsumerWebhook{acwName} = acwName nullAddress :: Address -> Bool nullAddress (Net.LocalPipe address) = null address diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs index da4f38fdd56..aee2c6d4f0e 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs @@ -15,6 +15,7 @@ module Cardano.Tracer.Handlers.Alarms.Auth , bearerToken , lookupProducer , lookupReader + , openProducer ) where import Cardano.Tracer.Configuration (AlarmsAuthConfig (..), ProducerCredentialConfig (..), @@ -22,9 +23,10 @@ import Cardano.Tracer.Configuration (AlarmsAuthConfig (..), ProducerCr import Cardano.Tracer.Handlers.Alarms.Types import qualified Data.ByteString.Char8 as BSC +import Data.Either (partitionEithers) import Data.Map.Strict (Map) import qualified Data.Map.Strict as Map -import Data.Maybe (fromMaybe) +import Data.Maybe (fromMaybe, listToMaybe) import Data.Text (Text) import qualified Data.Text as Text import qualified Data.Text.Encoding as TE @@ -45,8 +47,14 @@ data ReaderCredential = ReaderCredential deriving stock (Eq, Show) data AuthTables = AuthTables - { atProducers :: !(Map Text ProducerCredential) - , atReaders :: !(Map Text ReaderCredential) + { atProducers :: !(Map Text ProducerCredential) + , atOpenProducer :: !(Maybe ProducerCredential) + -- ^ At most one, per 'Cardano.Tracer.Configuration.wellFormed': the + -- producer configured with no @tokenFile@ at all, matched against + -- ingress requests that carry no @Authorization@ header (see + -- 'Cardano.Tracer.Handlers.Alarms.Server.handleIngress'). 'Nothing' + -- (the common case) preserves the original deny-by-default behaviour. + , atReaders :: !(Map Text ReaderCredential) } -- | Reads every configured token file once, at registry-construction time @@ -55,16 +63,23 @@ data AuthTables = AuthTables -- needs 'IO'. loadCredentials :: AlarmsAuthConfig -> IO AuthTables loadCredentials AlarmsAuthConfig{aacProducers, aacReaders} = do - producers <- traverse loadProducer aacProducers - readers <- traverse loadReader aacReaders + producerEntries <- traverse loadProducer aacProducers + readers <- traverse loadReader aacReaders + let (openProducers, tokenProducers) = partitionEithers producerEntries pure AuthTables - { atProducers = Map.fromList producers - , atReaders = Map.fromList readers + { atProducers = Map.fromList tokenProducers + , atOpenProducer = listToMaybe openProducers + -- ^ 'Cardano.Tracer.Configuration.wellFormed' guarantees at most one. + , atReaders = Map.fromList readers } where - loadProducer ProducerCredentialConfig{pcTokenFile, pcSource} = do - token <- readTokenFile pcTokenFile - pure (token, ProducerCredential (AlarmSource pcSource)) + -- A producer with no 'pcTokenFile' is the open producer ('Left'); one with + -- a token file is added to the token-keyed map as before ('Right'). + loadProducer ProducerCredentialConfig{pcTokenFile, pcSource} = case pcTokenFile of + Nothing -> pure (Left (ProducerCredential (AlarmSource pcSource))) + Just tokenFile -> do + token <- readTokenFile tokenFile + pure (Right (token, ProducerCredential (AlarmSource pcSource))) loadReader ReaderCredentialConfig{rcName, rcTokenFile, rcAllowHistory, rcFilter} = do token <- readTokenFile rcTokenFile @@ -94,3 +109,10 @@ lookupProducer tables token = Map.lookup token (atProducers tables) lookupReader :: AuthTables -> Text -> Maybe ReaderCredential lookupReader tables token = Map.lookup token (atReaders tables) + +-- | The open (tokenless) producer, if one is configured. Callers must only +-- use this to attribute a request that carries no @Authorization@ header +-- at all -- an unrecognised token must still be rejected, never fall back +-- here (see 'Cardano.Tracer.Handlers.Alarms.Server.handleIngress'). +openProducer :: AuthTables -> Maybe ProducerCredential +openProducer = atOpenProducer diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs index 8405afc5c24..534123e6408 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs @@ -1,13 +1,26 @@ +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} --- | Statically configured alarm dispatch targets. Phase 1 (this sketch) --- implements only 'ConsumerLog', matching the design doc's own staging --- ("Add a log consumer to exercise filtering and dispatch"). Adding --- @webhook@\/@email@ consumers later is one new constructor here and one new --- case in 'dispatch' -- @email@ in particular can reuse the SMTP-sending --- primitives in "Cardano.Tracer.Handlers.Notifications.Email" -- not --- speculative machinery added now. +-- 'Consumer' is a sum type whose two constructors define disjoint record fields +-- (cl* for 'ConsumerLog', cw* for 'ConsumerWebhook'). Every use is via +-- 'RecordWildCards'\/'NamedFieldPuns' pattern binding, never as a selector. +{-# OPTIONS_GHC -Wno-partial-fields #-} + +-- | Statically configured alarm dispatch targets. Two variants: +-- +-- * 'ConsumerLog' — synchronous meta-trace with the alarm payload. +-- * 'ConsumerWebhook' — enqueues onto a bounded 'TBQueue'; a worker +-- thread POSTs each event to the configured URL with exponential +-- backoff on transient failures (5xx and network errors) and +-- drop-on-permanent-failure (4xx other than 429). Never blocks the +-- caller: when the queue is full, the submission is dropped and +-- logged to stderr, matching the concept doc's requirement that a +-- slow destination must not block ingestion. module Cardano.Tracer.Handlers.Alarms.Consumers ( AlarmConsumer (..) , consumerFromConfig @@ -18,29 +31,153 @@ import Cardano.Tracer.Configuration (AlarmsConsumerConfig (..)) import Cardano.Tracer.Handlers.Alarms.Types import Cardano.Tracer.MetaTrace (TracerTrace (..), Trace, traceWith) +import Control.Concurrent (threadDelay) +import Control.Concurrent.Async (Async, async, link) +import Control.Concurrent.STM (atomically) +import Control.Concurrent.STM.TBQueue (TBQueue, isFullTBQueue, newTBQueueIO, readTBQueue, + writeTBQueue) +import Control.Exception (SomeException, try) +import Control.Monad (forever) +import Data.Aeson (encode) +import qualified Data.ByteString.Char8 as BSC +import Data.Maybe (fromMaybe) import Data.Text (Text) +import qualified Data.Text as Text +import qualified Data.Text.IO as TIO +import Network.HTTP.Client (HttpException, Manager, RequestBody (..), + defaultManagerSettings, httpLbs, method, newManager, parseRequest, requestBody, + requestHeaders, responseStatus) +import Network.HTTP.Client.TLS (tlsManagerSettings) +import Network.HTTP.Types (Status (..)) +import System.IO (stderr) -data AlarmConsumer = ConsumerLog - { clName :: !Text - , clEnabled :: !Bool - , clFilter :: !AlarmFilter - } +data AlarmConsumer + = ConsumerLog + { clName :: !Text + , clEnabled :: !Bool + , clFilter :: !AlarmFilter + } + | ConsumerWebhook + { cwName :: !Text + , cwEnabled :: !Bool + , cwFilter :: !AlarmFilter + , cwQueue :: !(TBQueue AlarmEvent) + , cwWorker :: !(Async ()) + } -consumerFromConfig :: AlarmsConsumerConfig -> AlarmConsumer -consumerFromConfig AlarmsConsumerLog{aclName, aclEnabled, aclFilter} = ConsumerLog - { clName = aclName - , clEnabled = aclEnabled - , clFilter = maybe emptyAlarmFilter filterFromConfig aclFilter - } +-- | Translate a config entry into a runtime consumer. For @webhook@, +-- allocates the bounded queue, spawns the delivery worker, and +-- 'link's it to the caller so a worker crash surfaces. +consumerFromConfig :: AlarmsConsumerConfig -> IO AlarmConsumer +consumerFromConfig = \case + AlarmsConsumerLog{aclName, aclEnabled, aclFilter} -> pure ConsumerLog + { clName = aclName + , clEnabled = aclEnabled + , clFilter = maybe emptyAlarmFilter filterFromConfig aclFilter + } + AlarmsConsumerWebhook{acwName, acwEnabled, acwFilter, acwUrl, acwBearerToken, acwQueueCap, acwMaxBackoff} -> do + let cap = fromIntegral (max 1 (fromMaybe 1000 acwQueueCap)) + maxBackoff = max 1 (fromMaybe 60 acwMaxBackoff) + queue <- newTBQueueIO cap + manager <- if "https://" `Text.isPrefixOf` acwUrl + then newManager tlsManagerSettings + else newManager defaultManagerSettings + worker <- async (runWebhookWorker acwName acwUrl acwBearerToken maxBackoff manager queue) + link worker + pure ConsumerWebhook + { cwName = acwName + , cwEnabled = acwEnabled + , cwFilter = maybe emptyAlarmFilter filterFromConfig acwFilter + , cwQueue = queue + , cwWorker = worker + } dispatch :: Trace IO TracerTrace -> AlarmConsumer -> AlarmEvent -> IO () -dispatch tracer consumer@ConsumerLog{clEnabled, clFilter} ev - | clEnabled && matchesFilter clFilter ev = - traceWith tracer TracerAlarmDispatched - { ttAlarmDispatchedConsumer = clName consumer - , ttAlarmDispatchedSource = unAlarmSource (source ev) - , ttAlarmDispatchedRuleId = unRuleId (ruleId ev) - , ttAlarmDispatchedSeverity = severity ev - , ttAlarmDispatchedSummary = summary ev - } - | otherwise = pure () +dispatch tracer consumer ev = case consumer of + ConsumerLog{clName, clEnabled, clFilter} + | clEnabled && matchesFilter clFilter ev -> traced clName + | otherwise -> pure () + ConsumerWebhook{cwName, cwEnabled, cwFilter, cwQueue} + | cwEnabled && matchesFilter cwFilter ev -> do + enqueued <- atomically $ do + full <- isFullTBQueue cwQueue + if full then pure False + else writeTBQueue cwQueue ev >> pure True + if enqueued + then traced cwName + else logLocal + ("webhook queue full [" <> cwName <> "]; dropped eventId " + <> eventId ev) + | otherwise -> pure () + where + traced :: Text -> IO () + traced consumerName = + traceWith tracer TracerAlarmDispatched + { ttAlarmDispatchedConsumer = consumerName + , ttAlarmDispatchedSource = unAlarmSource (source ev) + , ttAlarmDispatchedRuleId = unRuleId (ruleId ev) + , ttAlarmDispatchedSeverity = severity ev + , ttAlarmDispatchedSummary = summary ev + } + +-------------------------------------------------------------------------------- +-- Webhook worker +-------------------------------------------------------------------------------- + +runWebhookWorker :: Text -> Text -> Maybe Text -> Int -> Manager -> TBQueue AlarmEvent -> IO () +runWebhookWorker name url mBearer maxBackoff manager queue = forever $ do + ev <- atomically (readTBQueue queue) + deliverWithRetry name url mBearer maxBackoff manager ev 1 + +deliverWithRetry :: Text -> Text -> Maybe Text -> Int -> Manager -> AlarmEvent -> Int -> IO () +deliverWithRetry name url mBearer maxBackoff manager ev backoff = do + outcome <- attempt url mBearer manager ev + case outcome of + Delivered -> pure () + PermanentFailure reason -> logLocal + ("webhook [" <> name <> "] dropped " <> eventId ev + <> " (permanent: " <> reason <> ")") + Transient reason -> do + logLocal ("webhook [" <> name <> "] retrying " <> eventId ev + <> " in " <> tshow backoff <> "s (" <> reason <> ")") + threadDelay (backoff * 1_000_000) + deliverWithRetry name url mBearer maxBackoff manager ev (min maxBackoff (backoff * 2)) + +data Outcome = Delivered | PermanentFailure !Text | Transient !Text + +attempt :: Text -> Maybe Text -> Manager -> AlarmEvent -> IO Outcome +attempt url mBearer manager ev = do + reqR <- try @SomeException (parseRequest (Text.unpack url)) + case reqR of + Left e -> pure (PermanentFailure ("cannot parse URL: " <> tshow e)) + Right base -> do + let req = base + { method = "POST" + , requestBody = RequestBodyLBS (encode ev) + , requestHeaders = ("Content-Type", "application/json") : bearerHdr mBearer + } + respR <- try @HttpException (httpLbs req manager) + case respR of + Left e -> pure (Transient ("network error: " <> tshow e)) + Right resp -> pure (classify (statusCode (responseStatus resp))) + where + bearerHdr Nothing = [] + bearerHdr (Just t) = [("Authorization", "Bearer " <> BSC.pack (Text.unpack t))] + +classify :: Int -> Outcome +classify code + | code == 200 || code == 201 = Delivered + | code == 429 = Transient "429 rate-limited" + | code >= 500 && code < 600 = Transient ("server " <> tshow code) + | code >= 400 && code < 500 = PermanentFailure ("client " <> tshow code) + | otherwise = Transient ("unexpected " <> tshow code) + +-------------------------------------------------------------------------------- +-- Local logging (never re-enters the alarm pipeline) +-------------------------------------------------------------------------------- + +logLocal :: Text -> IO () +logLocal = TIO.hPutStrLn stderr + +tshow :: Show a => a -> Text +tshow = Text.pack . show diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs index d5bd75e45be..65e7b502551 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs @@ -20,6 +20,7 @@ module Cardano.Tracer.Handlers.Alarms.Registry , traceHistoryRead , lookupProducerCredential , lookupReaderCredential + , openProducerCredential ) where import Cardano.Logging.Types (TraceObject) @@ -45,26 +46,35 @@ import Data.Time.Clock (UTCTime, getCurrentTime) import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) data AlarmRegistry = AlarmRegistry - { arStore :: !AlarmStoreHandle - , arConsumers :: ![AlarmConsumer] -- ^ static, fixed at startup - , arTraceRules :: ![TraceAlarmRule] - , arTimeseriesRules :: ![TimeseriesAlarmRule] - , arAuth :: !AuthTables - , arTracer :: !(Trace IO TracerTrace) + { arStore :: !AlarmStoreHandle + , arConsumers :: ![AlarmConsumer] -- ^ static, fixed at startup + , arTraceRules :: ![TraceAlarmRule] + , arTimeseriesRules :: ![TimeseriesAlarmRule] + , arTimeseriesStatePath :: !(Maybe FilePath) + , arAuth :: !AuthTables + , arTracer :: !(Trace IO TracerTrace) } newAlarmRegistry :: Trace IO TracerTrace -> AlarmsConfig -> IO AlarmRegistry -newAlarmRegistry tracer AlarmsConfig{alRetention, alConsumers, alAuthentication, alTraceRules, alTimeseriesRules} = do - store <- newAlarmStore (fromMaybe emptyRetention alRetention) - authTable <- loadCredentials alAuthentication - tsRules <- traverse timeseriesRuleFromConfig (fromMaybe [] alTimeseriesRules) +newAlarmRegistry tracer AlarmsConfig + { alRetention, alConsumers, alAuthentication + , alTraceRules, alTimeseriesRules, alTimeseriesStatePath + } = do + store <- newAlarmStore (fromMaybe emptyRetention alRetention) + authT <- loadCredentials alAuthentication + consumers <- traverse consumerFromConfig alConsumers + tsRules <- traverse timeseriesRuleFromConfig (fromMaybe [] alTimeseriesRules) + for_ alTimeseriesStatePath \path -> do + persisted <- loadTimeseriesState path + seedRuleStates persisted tsRules pure AlarmRegistry - { arStore = store - , arConsumers = map consumerFromConfig alConsumers - , arTraceRules = map traceRuleFromConfig (fromMaybe [] alTraceRules) - , arTimeseriesRules = tsRules - , arAuth = authTable - , arTracer = tracer + { arStore = store + , arConsumers = consumers + , arTraceRules = map traceRuleFromConfig (fromMaybe [] alTraceRules) + , arTimeseriesRules = tsRules + , arTimeseriesStatePath = alTimeseriesStatePath + , arAuth = authT + , arTracer = tracer } where emptyRetention = AlarmsRetentionConfig Nothing Nothing @@ -158,6 +168,8 @@ evaluateTimeseriesRule registry tsHandle now rule = do Just samples -> do reqs <- evaluateOnce rule now samples for_ reqs \req -> acceptEvent registry timeseriesAlarmSource req + for_ (arTimeseriesStatePath registry) \path -> + saveTimeseriesState path (arTimeseriesRules registry) where failed :: Text -> IO () failed reason = @@ -177,3 +189,9 @@ lookupProducerCredential registry = lookupProducer (arAuth registry) lookupReaderCredential :: AlarmRegistry -> Text -> Maybe ReaderCredential lookupReaderCredential registry = lookupReader (arAuth registry) + +-- | The open (tokenless) producer credential, if one is configured. Only +-- for use when an ingress request carries no @Authorization@ header at +-- all -- see 'Cardano.Tracer.Handlers.Alarms.Server.handleIngress'. +openProducerCredential :: AlarmRegistry -> Maybe ProducerCredential +openProducerCredential registry = openProducer (arAuth registry) diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs index 13593e4dfaf..62f55dd68b1 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs @@ -128,9 +128,26 @@ alarmsApp cfg registry request send = case (requestMethod request, pathInfo requ ("GET", ["alarms", "v1", "events"]) -> handleHistory registry request send _ -> send notFound +-- | Resolve the producer credential for an ingress request. +-- +-- * A request carrying an @Authorization@ header is resolved via the usual +-- token lookup, exactly as before -- an unknown\/garbage token is +-- rejected outright, it never falls back to the open producer. Silently +-- treating a mistyped token as "anonymous" would mask real client +-- misconfiguration. +-- * A request with no @Authorization@ header at all falls back to the +-- configured open producer, if any (see +-- 'Cardano.Tracer.Handlers.Alarms.Auth.openProducer' and 'wellFormed'\'s +-- @allowInsecure@\/single-open-producer checks). +resolveProducerCredential :: AlarmRegistry -> Request -> Maybe ProducerCredential +resolveProducerCredential registry request = + case lookup hAuthorization (requestHeaders request) of + Just _ -> bearerToken request >>= lookupProducerCredential registry + Nothing -> openProducerCredential registry + handleIngress :: AlarmsConfig -> AlarmRegistry -> Application handleIngress cfg registry request send = - case bearerToken request >>= lookupProducerCredential registry of + case resolveProducerCredential registry request of Nothing -> send unauthorized Just (ProducerCredential src) -> do readLimitedBody (maxEventBytes cfg) request >>= \case diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs index 391d576df07..361726e60c7 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs @@ -1,3 +1,6 @@ +{-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE DerivingStrategies #-} {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} @@ -26,6 +29,7 @@ module Cardano.Tracer.Handlers.Alarms.TimeseriesRules , tarRuleId , tarQuery , tarEvaluateEvery + , tarSeriesState , timeseriesAlarmSource , timeseriesRuleFromConfig , SamplePoint (..) @@ -33,6 +37,10 @@ module Cardano.Tracer.Handlers.Alarms.TimeseriesRules , evaluateOnce , ruleRequests , SeriesState (..) + , SeriesKey (..) + , saveTimeseriesState + , loadTimeseriesState + , seedRuleStates ) where import Cardano.Logging.Types (SeverityS) @@ -41,8 +49,10 @@ import Cardano.Timeseries.Domain.Instant (Instant (..)) import Cardano.Tracer.Configuration (AlarmsTimeseriesRuleConfig (..)) import Cardano.Tracer.Handlers.Alarms.Types -import Data.Aeson (object, (.=)) -import Data.IORef (IORef, atomicModifyIORef', newIORef) +import Control.Exception (SomeException, try) +import Data.Aeson (FromJSON, ToJSON, decodeFileStrict', encodeFile, object, (.=)) +import Data.Foldable (for_) +import Data.IORef (IORef, atomicModifyIORef', atomicWriteIORef, newIORef, readIORef) import Data.Map.Strict (Map) import qualified Data.Map.Strict as Map import Data.Maybe (fromMaybe) @@ -52,6 +62,9 @@ import qualified Data.Text as Text import Data.Time.Clock (UTCTime, diffUTCTime) import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) import Data.Word (Word64) +import GHC.Generics (Generic) +import System.Directory (doesFileExist, renameFile) +import System.FilePath ((<.>)) -- | Runtime form of a timeseries rule. The mutable 'IORef' holds the -- per-series state machines — inactive, pending-with-since, or @@ -73,6 +86,7 @@ data TimeseriesAlarmRule = TimeseriesAlarmRule -- the same key. newtype SeriesKey = SeriesKey { unSeriesKey :: Text } deriving stock (Eq, Ord, Show) + deriving newtype (FromJSON, ToJSON) -- | Per-series state. data SeriesState @@ -84,7 +98,8 @@ data SeriesState | Active !UTCTime -- ^ Sample stayed truthy for @for@ seconds and has been published -- at least once; the field is @publishedAt@ (last publish time). - deriving stock (Eq, Show) + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) -- | One boolean sample decoded from a query response, tagged with its -- labels (the source of 'SeriesKey'). @@ -260,3 +275,45 @@ evaluateOnce rule now samples = let (newState, reqs) = ruleRequests rule now samples prev in (newState, reqs) +-------------------------------------------------------------------------------- +-- Persistence +-- +-- One combined JSON file: rule id -> series key -> SeriesState. Atomic +-- rename on write; missing/parse-errored file treated as "no state" per +-- the concept doc's cold-start fallback. +-------------------------------------------------------------------------------- + +type PersistedState = Map Text (Map Text SeriesState) + +saveTimeseriesState :: FilePath -> [TimeseriesAlarmRule] -> IO () +saveTimeseriesState path rules = do + entries <- traverse snapshot rules + let persisted :: PersistedState + persisted = Map.fromList entries + tmp = path <.> "tmp" + encodeFile tmp persisted + renameFile tmp path + where + snapshot :: TimeseriesAlarmRule -> IO (Text, Map Text SeriesState) + snapshot rule = do + m <- readIORef (tarSeriesState rule) + pure (unRuleId (tarRuleId rule), Map.mapKeys unSeriesKey m) + +loadTimeseriesState :: FilePath -> IO PersistedState +loadTimeseriesState path = do + exists <- doesFileExist path + if not exists + then pure Map.empty + else do + result <- try (decodeFileStrict' path) :: IO (Either SomeException (Maybe PersistedState)) + case result of + Right (Just m) -> pure m + _ -> pure Map.empty + +seedRuleStates :: PersistedState -> [TimeseriesAlarmRule] -> IO () +seedRuleStates persisted rules = + for_ rules \rule -> + case Map.lookup (unRuleId (tarRuleId rule)) persisted of + Nothing -> pure () + Just m -> atomicWriteIORef (tarSeriesState rule) (Map.mapKeys SeriesKey m) + diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs index d62bcfc38a9..47571b9dcee 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs @@ -17,20 +17,27 @@ module Cardano.Tracer.Test.Alarms.Tests import Cardano.Logging (DetailLevel (..), SeverityF (..), SeverityS (..), TraceObject (..)) import Cardano.Tracer.Configuration +import Cardano.Tracer.Handlers.Alarms.Auth (ProducerCredential (..), loadCredentials, + lookupProducer, openProducer) import Cardano.Tracer.Handlers.Alarms.Registry import Cardano.Tracer.Handlers.Alarms.Store import Cardano.Tracer.Handlers.Alarms.Types import Cardano.Tracer.MetaTrace (TraceBundle (..), mkTraceBundle) import Control.Concurrent.Async (forConcurrently) +import Control.Exception (bracket) import Data.Aeson (decode, encode) import qualified Data.List as List +import Data.List.NonEmpty (NonEmpty (..)) import Data.Map.Strict (Map) import qualified Data.Map.Strict as Map import Data.Text (Text) import qualified Data.Text as Text +import qualified Data.Text.IO as TIO import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime) import Data.Time.Clock.POSIX (posixSecondsToUTCTime, utcTimeToPOSIXSeconds) +import System.Directory (getTemporaryDirectory, removeFile) +import System.IO (hClose, openTempFile) import Test.Tasty import Test.Tasty.QuickCheck hiding (labels) @@ -67,6 +74,13 @@ tests = testGroup "Test.Alarms" , testProperty "matches in one window collapse to one alarm" propTraceRuleSuppression , testProperty "matches in a new window raise a fresh alarm" propTraceRuleNewWindow ] + , testGroup "producer auth" + [ testProperty "open producer resolves for headerless requests" propOpenProducerHonored + , testProperty "unrecognised token is rejected despite open producer" propWrongTokenRejectedWithOpenProducer + , testProperty "wellFormed accepts a single open producer with allowInsecure" propOpenProducerAllowedWhenWellFormed + , testProperty "wellFormed rejects more than one open producer" propRejectsMultipleOpenProducers + , testProperty "wellFormed rejects an open producer without allowInsecure" propRejectsOpenProducerWithoutAllowInsecure + ] ] -------------------------------------------------------------------------------- @@ -370,6 +384,7 @@ testAlarmsConfig = AlarmsConfig } ] , alTimeseriesRules = Nothing + , alTimeseriesStatePath = Nothing } newTestRegistry :: IO AlarmRegistry @@ -432,3 +447,150 @@ propTraceRuleNewWindow = ioProperty $ do checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error (addUTCTime 60 boundary)] events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter pure (length events === 2) + +-------------------------------------------------------------------------------- +-- Producer auth properties +-------------------------------------------------------------------------------- + +-- | Write @tokenText@ to a fresh temp file for the duration of @action@, +-- mirroring how a real token-bearing producer's @tokenFile@ is read by +-- 'loadCredentials'. Removed afterwards regardless of outcome. +withTokenFile :: Text -> (FilePath -> IO a) -> IO a +withTokenFile tokenText = bracket acquire removeFile + where + acquire = do + tmpDir <- getTemporaryDirectory + (path, hdl) <- openTempFile tmpDir "alarm-producer-token" + TIO.hPutStr hdl tokenText + hClose hdl + pure path + +-- | An "open" (tokenless) producer config: 'pcTokenFile' is 'Nothing'. +openProducerConfigWithSource :: Text -> AlarmSource -> ProducerCredentialConfig +openProducerConfigWithSource name src = ProducerCredentialConfig + { pcName = name + , pcTokenFile = Nothing + , pcSource = unAlarmSource src + } + +-- | 'loadCredentials' resolves a configured open (tokenless) producer into +-- 'AuthTables', and 'Auth.openProducer'\/'Registry.openProducerCredential' +-- surface it -- this is what a headerless ingress request is attributed +-- to (see 'Cardano.Tracer.Handlers.Alarms.Server.resolveProducerCredential'). +propOpenProducerHonored :: Property +propOpenProducerHonored = forAll genShortText \name -> forAll genSource \src -> ioProperty do + let authConfig = AlarmsAuthConfig + { aacProducers = [openProducerConfigWithSource name src] + , aacReaders = [] + } + tables <- loadCredentials authConfig + pure (openProducer tables === Just (ProducerCredential src)) + +-- | An unrecognised bearer token must still be rejected (no producer +-- credential resolves) even when an open producer is configured +-- alongside a token producer -- the open-producer fallback is only for +-- requests carrying no @Authorization@ header at all, never for a +-- present-but-wrong one. A correctly-presented token still resolves, and +-- the open producer remains available for headerless requests, so all +-- three coexist correctly in the same tables. +propWrongTokenRejectedWithOpenProducer :: Property +propWrongTokenRejectedWithOpenProducer = + forAll genShortText \tokenProducerName -> + forAll genSource \tokenSrc -> + forAll genSource \openSrc -> + forAll genShortText \goodToken -> + forAll (genShortText `suchThat` (/= goodToken)) \badToken -> + ioProperty $ withTokenFile goodToken \tokenFile -> do + let authConfig = AlarmsAuthConfig + { aacProducers = + [ ProducerCredentialConfig + { pcName = tokenProducerName + , pcTokenFile = Just tokenFile + , pcSource = unAlarmSource tokenSrc + } + , openProducerConfigWithSource "open" openSrc + ] + , aacReaders = [] + } + tables <- loadCredentials authConfig + pure $ conjoin + [ counterexample "correct token must resolve to its own producer" + (lookupProducer tables goodToken === Just (ProducerCredential tokenSrc)) + , counterexample "unrecognised token must be rejected outright, not fall back to open" + (lookupProducer tables badToken === Nothing) + , counterexample "open producer must remain available for headerless requests" + (openProducer tables === Just (ProducerCredential openSrc)) + ] + +-------------------------------------------------------------------------------- +-- Config validation ('wellFormed') properties +-------------------------------------------------------------------------------- + +-- | A minimal, otherwise-valid 'TracerConfig' wrapping the given +-- 'AlarmsConfig' -- only the fields 'wellFormed' actually inspects +-- (network, logging, the three service endpoints, alarms) need be +-- non-degenerate. +tracerConfigWithAlarms :: AlarmsConfig -> TracerConfig +tracerConfigWithAlarms alarmsCfg = TracerConfig + { networkMagic = 42 + , network = AcceptAt (LocalPipe "cardano-tracer-test-alarms.sock") + , loRequestNum = Nothing + , ekgRequestFreq = Nothing + , hasEKG = Nothing + , hasPrometheus = Nothing + , hasTimeseries = Nothing + , alarms = Just alarmsCfg + , tlsCertificate = Nothing + , hasForwarding = Nothing + , logging = LoggingParams "cardano-tracer-test-logs" FileMode ForHuman :| [] + , rotation = Nothing + , verbosity = Nothing + , metricsNoSuffix = Nothing + , metricsHelp = Nothing + , resourceFreq = Nothing + , ekgRequestFull = Nothing + , prometheusLabels = Nothing + } + +-- | 'wellFormed' rejects the config, with an error mentioning @substr@. +expectRejectedContaining :: String -> TracerConfig -> Property +expectRejectedContaining substr cfg = case wellFormed cfg of + Left msg -> counterexample msg (substr `List.isInfixOf` msg) + Right () -> counterexample "expected wellFormed to reject this config, but it was accepted" False + +openProducerConfig :: Text -> ProducerCredentialConfig +openProducerConfig name = ProducerCredentialConfig + { pcName = name, pcTokenFile = Nothing, pcSource = name } + +-- | Sanity control: a single open producer with 'alAllowInsecure' set is +-- accepted -- without this, the two rejection properties below could +-- pass vacuously for an unrelated reason. +propOpenProducerAllowedWhenWellFormed :: Property +propOpenProducerAllowedWhenWellFormed = forAll genShortText \name -> + let cfg = tracerConfigWithAlarms $ testAlarmsConfig + { alAuthentication = AlarmsAuthConfig [openProducerConfig name] [] + } + in wellFormed cfg === Right () + +-- | Two open (tokenless) producers are ambiguous -- an unauthenticated +-- request could not be attributed to either -- so 'wellFormed' rejects +-- the config outright, regardless of 'alAllowInsecure'. +propRejectsMultipleOpenProducers :: Property +propRejectsMultipleOpenProducers = + forAll genShortText \n1 -> forAll (genShortText `suchThat` (/= n1)) \n2 -> + expectRejectedContaining "more than one open" $ + tracerConfigWithAlarms testAlarmsConfig + { alAuthentication = AlarmsAuthConfig [openProducerConfig n1, openProducerConfig n2] [] + } + +-- | A single open producer still requires an explicit +-- @allowInsecure: true@ acknowledgement; 'Nothing' and 'Just False' are +-- both rejected. +propRejectsOpenProducerWithoutAllowInsecure :: Property +propRejectsOpenProducerWithoutAllowInsecure = + forAll genShortText \name -> forAll (elements [Nothing, Just False]) \allowInsecure -> + expectRejectedContaining "requires allowInsecure" $ + tracerConfigWithAlarms testAlarmsConfig + { alAllowInsecure = allowInsecure + , alAuthentication = AlarmsAuthConfig [openProducerConfig name] [] + } diff --git a/cardano-tracer/test/alarms-smoke.hs b/cardano-tracer/test/alarms-smoke.hs new file mode 100644 index 00000000000..cbc0c334274 --- /dev/null +++ b/cardano-tracer/test/alarms-smoke.hs @@ -0,0 +1,13 @@ +-- | Cabal wrapper for the alarm-subsystem smoke test. Locates +-- @test/alarms-smoke.sh@ via 'getDataFileName' and runs it under @bash@; +-- non-zero exit propagates through 'callProcess' as an exception, which +-- the test framework reports as a test failure. +module Main where + +import Paths_cardano_tracer (getDataFileName) +import System.Process (callProcess) + +main :: IO () +main = do + script <- getDataFileName "test/alarms-smoke.sh" + callProcess "bash" [script] diff --git a/cardano-tracer/test/alarms-smoke.sh b/cardano-tracer/test/alarms-smoke.sh new file mode 100755 index 00000000000..3bb223a5531 --- /dev/null +++ b/cardano-tracer/test/alarms-smoke.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Level-2 smoke test for the alarm HTTP surface. Level 1 (property tests +# in test/Cardano/Tracer/Test/Alarms/) is run separately with +# `cabal test cardano-tracer`. +set -euo pipefail + +work=$(mktemp -d) +prod=secret-producer +rdr=secret-reader +url=http://127.0.0.1:3210/alarms/v1/events + +printf '%s' "$prod" > "$work/p.token" +printf '%s' "$rdr" > "$work/r.token" + +cat > "$work/tracer.yaml" <"$work/tracer.log" & +pid=$! +trap 'kill "$pid" 2>/dev/null; rm -rf "$work"' EXIT + +for _ in {1..60}; do + curl -sSo /dev/null -m1 "$url" 2>/dev/null && break + sleep 0.25 +done + +body='{"sourceEventId":"s1","raisedAt":"2026-08-11T12:00:00Z","ruleId":"cg","severity":"warning","summary":"hi"}' +ap="Authorization: Bearer $prod" +ar="Authorization: Bearer $rdr" +ct="Content-Type: application/json" + +check() { # check + local got exp=$1; shift + got=$(curl -sSo "$work/out" -w '%{http_code}' "$@") + [[ $got == "$exp" ]] || { echo "expected $exp, got $got: $(cat "$work/out")"; exit 1; } +} + +check 201 -H "$ap" -H "$ct" -d "$body" "$url" ; grep -q '"created":true' "$work/out" +check 200 -H "$ap" -H "$ct" -d "$body" "$url" ; grep -q '"created":false' "$work/out" +check 200 -H "$ar" "$url" ; grep -q '"eventId"' "$work/out" +check 401 "$url" +check 401 -H "Authorization: Bearer wrong" "$url" +check 401 -H "$ar" -H "$ct" -d "$body" "$url" +check 400 -H "$ap" -H "$ct" -d '{no' "$url" + +echo OK From 4b21b51f97c7aa2248bd457a6065cf7521710369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Baldur=20Bl=C3=B6ndal?= Date: Thu, 29 Oct 2026 12:35:28 +0000 Subject: [PATCH 8/8] cardano-testnet: add alarms = Nothing to TracerConfig construction --- ...29_000000_cardano-testnet_baldur_alarms_strict_field.yml | 6 ++++++ cardano-testnet/src/Testnet/CardanoTracer.hs | 1 + .../test/Cardano/Tracer/Test/Alarms/TimeseriesTests.hs | 1 + 3 files changed, 8 insertions(+) create mode 100644 cardano-testnet/.changes/20261029_000000_cardano-testnet_baldur_alarms_strict_field.yml diff --git a/cardano-testnet/.changes/20261029_000000_cardano-testnet_baldur_alarms_strict_field.yml b/cardano-testnet/.changes/20261029_000000_cardano-testnet_baldur_alarms_strict_field.yml new file mode 100644 index 00000000000..a6e4f7e680d --- /dev/null +++ b/cardano-testnet/.changes/20261029_000000_cardano-testnet_baldur_alarms_strict_field.yml @@ -0,0 +1,6 @@ +description: | + Add `alarms = Nothing` to the local `TracerConfig` construction so the project compiles against the new strict `alarms` field in `cardano-tracer`. +kind: +- bugfix +pr: 6664 +project: cardano-testnet diff --git a/cardano-testnet/src/Testnet/CardanoTracer.hs b/cardano-testnet/src/Testnet/CardanoTracer.hs index 04fa177bcb8..d1b5d713880 100644 --- a/cardano-testnet/src/Testnet/CardanoTracer.hs +++ b/cardano-testnet/src/Testnet/CardanoTracer.hs @@ -77,6 +77,7 @@ mkConfig CardanoTracerConf { testnetMagic, logFormat, prometheusIP } port logFil , resourceFreq = Nothing , ekgRequestFull = Nothing , prometheusLabels = Nothing + , alarms = Nothing } -- | Data for working with a running @cardano-tracer@. diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/TimeseriesTests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/TimeseriesTests.hs index d283d263914..439d7972a4d 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/TimeseriesTests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/TimeseriesTests.hs @@ -98,6 +98,7 @@ mkAlarmsConfig rule = AlarmsConfig , alConsumers = [] , alTraceRules = Nothing , alTimeseriesRules = Just [rule] + , alTimeseriesStatePath = Nothing } -- | Evaluation timestamp of round @k@: 30 simulated seconds per round