Skip to content

feat(grafana): add UTxO RPC stale alert, dashboard panel, and manage infra-alerts contact point #70

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tripura-repalle merged 1 commit into from
Aug 25, 2026
Merged

feat(grafana): add UTxO RPC stale alert, dashboard panel, and manage infra-alerts contact point #70

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions grafana-alerts.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,7 @@ module "grafana_alerts" {
folder_title = each.value.grafana_title
folder_uid = each.value.folder_uid
datasource_uids = module.grafana_data_sources.uids

# Rule groups reference the "#infra-alerts" receiver, which must exist first.
depends_on = [grafana_contact_point.infra_alerts]
}
15 changes: 15 additions & 0 deletions grafana-contact-points.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Manages the "#infra-alerts" contact point that every Demeter alert rule group
# routes to (notification_settings.receiver). Grafana rejects rule groups whose
# receiver does not exist, so this must be provisioned before the alerts.
#
# The Slack incoming webhook URL is a secret and is read from the sops-encrypted
# config.yaml (grafana.cloud.slack_webhook_url), matching how the other Grafana
# credentials (auth, sm_access_token) are stored. Add it with:
# sops config.yaml # then under grafana.cloud add: slack_webhook_url: <url>
resource "grafana_contact_point" "infra_alerts" {
name = "#infra-alerts"

slack {
url = try(local.env_vars.grafana.cloud.slack_webhook_url, "")
}
}
137 changes: 137 additions & 0 deletions grafana/alerts/demeter/utxorpc.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
{
"apiVersion": 1,
"groups": [
{
"orgId": 1,
"name": "UTxO RPC",
"folder": "Demeter",
"interval": "1m",
"rules": [
{
"uid": "bkl0urpcstale",
"title": "UTxO RPC backing Dolos is stale",
"condition": "C",
"for": "10m",
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 900,
"to": 0
},
"datasourceUid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}",
"model": {
"datasource": {
"type": "prometheus",
"uid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}"
},
"editorMode": "code",
"expr": "changes(dolos_slot{pod=~\"dolos-.*\"}[15m])",
"instant": true,
"intervalMs": 1000,
"legendFormat": "__auto",
"maxDataPoints": 43200,
"range": false,
"refId": "A"
}
},
{
"refId": "B",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
0,
0
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": []
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"name": "Expression",
"type": "__expr__",
"uid": "__expr__"
},
"expression": "A",
"hide": false,
"reducer": "last",
"refId": "B",
"type": "reduce"
}
},
{
"refId": "C",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
1,
0
],
"type": "lt"
},
"operator": {
"type": "and"
},
"query": {
"params": []
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"datasource": {
"name": "Expression",
"type": "__expr__",
"uid": "__expr__"
},
"expression": "B",
"hide": false,
"refId": "C",
"type": "threshold"
}
}
],
"noDataState": "OK",
"execErrState": "KeepLast",
"annotations": {
"description": "The Dolos instance backing UTxO RPC is up but its chain tip (dolos_slot) has not advanced for at least 15 minutes, meaning UTxO RPC is serving stale chain data to clients. This usually indicates a wedged chain-sync session that a pod restart will clear.",
"summary": "{{ range $k, $v := $values -}}\n{{ if (match \"C[0-9]+\" $k) -}}\nPod: {{ $v.Labels.pod }} Dolos slot is not advancing\n{{ end }}\n{{ end }}"
},
"labels": {},
"isPaused": false,
"notification_settings": {
"receiver": "#infra-alerts"
}
}
]
}
]
}
103 changes: 103 additions & 0 deletions grafana/dashboards/shared/utxorpc.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,109 @@
],
"title": "Requests per customer",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$datasource"
},
"description": "Number of times the backing Dolos instance's chain tip (dolos_slot) advanced in the last 15 minutes. A healthy, synced Dolos advances continuously; a value of 0 means the tip is frozen and UTxO RPC is serving stale chain data. This is the signal behind the \"UTxO RPC backing Dolos is stale\" alert.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "Dolos slot changes / 15m",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": 0
},
{
"color": "green",
"value": 1
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 26
},
"id": 20,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"min"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "asc"
}
},
"pluginVersion": "12.3.0-18356121373.patch5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$datasource"
},
"editorMode": "code",
"expr": "changes(dolos_slot{pod=~\"dolos-$network.*\"}[15m])",
"legendFormat": "{{pod}}",
"range": true,
"refId": "A"
}
],
"title": "Dolos Slot Advancement (changes / 15m)",
"type": "timeseries"
}
],
"refresh": "10s",
Expand Down
Loading