From a4ae8113c03dffb9c5ec2f00ead5a1800e902a8e Mon Sep 17 00:00:00 2001 From: Tripura Repalle Date: Tue, 25 Aug 2026 10:41:33 -0400 Subject: [PATCH] feat(grafana): add UTxO RPC stale alert, dashboard panel, and manage #infra-alerts contact point Adds staleness monitoring for UTxO RPC via its backing Dolos chain tip. UTxO RPC is a stateless proxy with no tip of its own, so it uses dolos_slot (the same Dolos tip metric already used in kupo.json): changes(dolos_slot[15m]) < 1 while Dolos is up means the tip is frozen and UTxO RPC is serving stale chain data. Adds grafana/alerts/demeter/utxorpc.json (group "UTxO RPC", folder "Demeter", rule "UTxO RPC backing Dolos is stale", for 10m, receiver #infra-alerts) and a "Dolos Slot Advancement (changes / 15m)" panel (id 20) to the UTxO RPC dashboard. Also manages the shared #infra-alerts contact point in Terraform (grafana-contact-points.tf) so alert rule groups no longer fail with 'receiver #infra-alerts does not exist'. The Slack webhook URL is read from the sops-encrypted config.yaml (grafana.cloud.slack_webhook_url); the grafana_alerts module now depends_on the contact point so it is created first. Signed-off-by: Tripura Repalle --- grafana-alerts.tf | 3 + grafana-contact-points.tf | 15 +++ grafana/alerts/demeter/utxorpc.json | 137 +++++++++++++++++++++++++ grafana/dashboards/shared/utxorpc.json | 103 +++++++++++++++++++ 4 files changed, 258 insertions(+) create mode 100644 grafana-contact-points.tf create mode 100644 grafana/alerts/demeter/utxorpc.json diff --git a/grafana-alerts.tf b/grafana-alerts.tf index e75d01d..1d8aa14 100644 --- a/grafana-alerts.tf +++ b/grafana-alerts.tf @@ -8,4 +8,7 @@ module "grafana_alerts" { folder_title = each.value.grafana_title folder_uid = each.value.folder_uid datasource_uids = module.grafana_data_sources.uids + + # Rule groups reference the "#infra-alerts" receiver, which must exist first. + depends_on = [grafana_contact_point.infra_alerts] } diff --git a/grafana-contact-points.tf b/grafana-contact-points.tf new file mode 100644 index 0000000..2e10ff0 --- /dev/null +++ b/grafana-contact-points.tf @@ -0,0 +1,15 @@ +# Manages the "#infra-alerts" contact point that every Demeter alert rule group +# routes to (notification_settings.receiver). Grafana rejects rule groups whose +# receiver does not exist, so this must be provisioned before the alerts. +# +# The Slack incoming webhook URL is a secret and is read from the sops-encrypted +# config.yaml (grafana.cloud.slack_webhook_url), matching how the other Grafana +# credentials (auth, sm_access_token) are stored. Add it with: +# sops config.yaml # then under grafana.cloud add: slack_webhook_url: +resource "grafana_contact_point" "infra_alerts" { + name = "#infra-alerts" + + slack { + url = try(local.env_vars.grafana.cloud.slack_webhook_url, "") + } +} diff --git a/grafana/alerts/demeter/utxorpc.json b/grafana/alerts/demeter/utxorpc.json new file mode 100644 index 0000000..7896ce7 --- /dev/null +++ b/grafana/alerts/demeter/utxorpc.json @@ -0,0 +1,137 @@ +{ + "apiVersion": 1, + "groups": [ + { + "orgId": 1, + "name": "UTxO RPC", + "folder": "Demeter", + "interval": "1m", + "rules": [ + { + "uid": "bkl0urpcstale", + "title": "UTxO RPC backing Dolos is stale", + "condition": "C", + "for": "10m", + "data": [ + { + "refId": "A", + "relativeTimeRange": { + "from": 900, + "to": 0 + }, + "datasourceUid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}", + "model": { + "datasource": { + "type": "prometheus", + "uid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}" + }, + "editorMode": "code", + "expr": "changes(dolos_slot{pod=~\"dolos-.*\"}[15m])", + "instant": true, + "intervalMs": 1000, + "legendFormat": "__auto", + "maxDataPoints": 43200, + "range": false, + "refId": "A" + } + }, + { + "refId": "B", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 0, + 0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "A", + "hide": false, + "reducer": "last", + "refId": "B", + "type": "reduce" + } + }, + { + "refId": "C", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 1, + 0 + ], + "type": "lt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "B", + "hide": false, + "refId": "C", + "type": "threshold" + } + } + ], + "noDataState": "OK", + "execErrState": "KeepLast", + "annotations": { + "description": "The Dolos instance backing UTxO RPC is up but its chain tip (dolos_slot) has not advanced for at least 15 minutes, meaning UTxO RPC is serving stale chain data to clients. This usually indicates a wedged chain-sync session that a pod restart will clear.", + "summary": "{{ range $k, $v := $values -}}\n{{ if (match \"C[0-9]+\" $k) -}}\nPod: {{ $v.Labels.pod }} Dolos slot is not advancing\n{{ end }}\n{{ end }}" + }, + "labels": {}, + "isPaused": false, + "notification_settings": { + "receiver": "#infra-alerts" + } + } + ] + } + ] +} diff --git a/grafana/dashboards/shared/utxorpc.json b/grafana/dashboards/shared/utxorpc.json index 42995fa..6e69317 100644 --- a/grafana/dashboards/shared/utxorpc.json +++ b/grafana/dashboards/shared/utxorpc.json @@ -523,6 +523,109 @@ ], "title": "Requests per customer", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of times the backing Dolos instance's chain tip (dolos_slot) advanced in the last 15 minutes. A healthy, synced Dolos advances continuously; a value of 0 means the tip is frozen and UTxO RPC is serving stale chain data. This is the signal behind the \"UTxO RPC backing Dolos is stale\" alert.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Dolos slot changes / 15m", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "12.3.0-18356121373.patch5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "changes(dolos_slot{pod=~\"dolos-$network.*\"}[15m])", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Dolos Slot Advancement (changes / 15m)", + "type": "timeseries" } ], "refresh": "10s",