Skip to content

Feat: kupo dolos checkpoint stale alert #65

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tripura-repalle merged 2 commits into from
Aug 24, 2026
+272 −2
Merged

Feat: kupo dolos checkpoint stale alert #65

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c652923
feat(grafana): add Kupo (Dolos) checkpoint-stale alert and dashboard …
tripura-repalle Aug 24, 2026
0ccb37f
feat(grafana): wire up grafana_alerts module so alert rules are applied
tripura-repalle Aug 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions grafana-alerts.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
module "grafana_alerts" {
for_each = {
for alert in local.env_vars.grafana.alerts :
alert.grafana_title => alert
}
source = "./modules/grafana_alerts"
local_directory = each.value.local_directory
folder_title = each.value.grafana_title
folder_uid = each.value.folder_uid
folders = [
{
title = each.value.grafana_title
uid = each.value.folder_uid
}
]
datasource_uids = module.grafana_data_sources.uids
}
137 changes: 137 additions & 0 deletions grafana/alerts/demeter/kupo-dolos.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
{
"apiVersion": 1,
"groups": [
{
"orgId": 1,
"name": "Kupo Dolos",
"folder": "Demeter",
"interval": "1m",
"rules": [
{
"uid": "bkl0kdlckpstale",
"title": "Kupo (Dolos) checkpoint is stale",
"condition": "C",
"for": "10m",
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 900,
"to": 0
},
"datasourceUid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}",
"model": {
"datasource": {
"type": "prometheus",
"uid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}"
},
"editorMode": "code",
"expr": "changes(kupo_most_recent_checkpoint{pod=~\".*dolos.*\"}[15m]) and on (pod) (kupo_connection_status{pod=~\".*dolos.*\"} == 1)",
"instant": true,
"intervalMs": 1000,
"legendFormat": "__auto",
"maxDataPoints": 43200,
"range": false,
"refId": "A"
}
},
{
"refId": "B",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
0,
0
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": []
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"name": "Expression",
"type": "__expr__",
"uid": "__expr__"
},
"expression": "A",
"hide": false,
"reducer": "last",
"refId": "B",
"type": "reduce"
}
},
{
"refId": "C",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
1,
0
],
"type": "lt"
},
"operator": {
"type": "and"
},
"query": {
"params": []
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"datasource": {
"name": "Expression",
"type": "__expr__",
"uid": "__expr__"
},
"expression": "B",
"hide": false,
"refId": "C",
"type": "threshold"
}
}
],
"noDataState": "OK",
"execErrState": "KeepLast",
"annotations": {
"description": "A Dolos-backed Kupo instance is connected to its Dolos backend but its most recent checkpoint (kupo_most_recent_checkpoint) has not advanced for at least 15 minutes, meaning it is serving a stale index to clients. This usually indicates a wedged chain-sync session between Kupo and Dolos that a pod restart will clear.",
"summary": "{{ range $k, $v := $values -}}\n{{ if (match \"C[0-9]+\" $k) -}}\nPod: {{ $v.Labels.pod }} checkpoint is not advancing\n{{ end }}\n{{ end }}"
},
"labels": {},
"isPaused": false,
"notification_settings": {
"receiver": "#infra-alerts"
}
}
]
}
]
}
116 changes: 116 additions & 0 deletions grafana/dashboards/shared/kupo-dolos.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1726,6 +1726,122 @@
],
"title": "CPU Usage (cores)",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 55
},
"id": 107,
"panels": [],
"title": "Checkpoint Health",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "$datasource"
},
"description": "Number of times each Dolos-backed Kupo instance's most recent checkpoint (kupo_most_recent_checkpoint) advanced in the last 15 minutes. A healthy, syncing instance advances continuously; a value of 0 means the checkpoint is frozen (wedged chain-sync with Dolos) and the pod is serving a stale index. This is the signal behind the \"Kupo (Dolos) checkpoint is stale\" alert.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "checkpoint changes / 15m",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": 0
},
{
"color": "green",
"value": 1
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 56
},
"id": 20,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"min"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "asc"
}
},
"pluginVersion": "12.4.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$datasource"
},
"editorMode": "code",
"expr": "changes(kupo_most_recent_checkpoint{pod=~\".*$network.*\"}[15m]) and on (pod) (kupo_connection_status{pod=~\".*$network.*\"} == 1)",
"legendFormat": "{{pod}}",
"range": true,
"refId": "A"
}
],
"title": "Checkpoint Advancement (changes / 15m)",
"type": "timeseries"
}
],
"preload": false,
Expand Down
4 changes: 2 additions & 2 deletions modules/grafana_alerts/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ resource "grafana_folder" "this" {

resource "grafana_rule_group" "this" {
for_each = {
for file in fileset("${path.root}/../${var.local_directory}", "*.json") :
file => jsondecode(templatefile("${path.root}/../${var.local_directory}/${file}", {
for file in fileset("${path.root}/${var.local_directory}", "*.json") :
file => jsondecode(templatefile("${path.root}/${var.local_directory}/${file}", {
datasource_uid_map = var.datasource_uids
}))
}
Expand Down
Loading