Skip to content

Feat: add submit api stale alert and dashboard pannel #68

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tripura-repalle merged 2 commits into from
Aug 25, 2026
+254 −18
Merged

Feat: add submit api stale alert and dashboard pannel #68

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5b6485e
feat(grafana): add Submit API backing-node stale alert and dashboard …
tripura-repalle Aug 25, 2026
8992604
fix(grafana): create the alert folder in the grafana_alerts module
tripura-repalle Aug 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions grafana-alerts.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,5 @@ module "grafana_alerts" {
local_directory = each.value.local_directory
folder_title = each.value.grafana_title
folder_uid = each.value.folder_uid
folders = [
{
title = each.value.grafana_title
uid = each.value.folder_uid
}
]
datasource_uids = module.grafana_data_sources.uids
}
137 changes: 137 additions & 0 deletions grafana/alerts/demeter/submit-api.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
{
"apiVersion": 1,
"groups": [
{
"orgId": 1,
"name": "Submit API",
"folder": "Demeter",
"interval": "1m",
"rules": [
{
"uid": "bkl0saptipstale",
"title": "Submit API backing node is stale",
"condition": "C",
"for": "10m",
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 900,
"to": 0
},
"datasourceUid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}",
"model": {
"datasource": {
"type": "prometheus",
"uid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}"
},
"editorMode": "code",
"expr": "changes(cardano_node_metrics_blockNum_int{namespace=\"ext-nodes-m1\", pod=~\"node-.*\"}[15m])",
"instant": true,
"intervalMs": 1000,
"legendFormat": "__auto",
"maxDataPoints": 43200,
"range": false,
"refId": "A"
}
},
{
"refId": "B",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
0,
0
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": []
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"name": "Expression",
"type": "__expr__",
"uid": "__expr__"
},
"expression": "A",
"hide": false,
"reducer": "last",
"refId": "B",
"type": "reduce"
}
},
{
"refId": "C",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
1,
0
],
"type": "lt"
},
"operator": {
"type": "and"
},
"query": {
"params": []
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"datasource": {
"name": "Expression",
"type": "__expr__",
"uid": "__expr__"
},
"expression": "B",
"hide": false,
"refId": "C",
"type": "threshold"
}
}
],
"noDataState": "OK",
"execErrState": "KeepLast",
"annotations": {
"description": "The cardano-node backing the Submit API is up but its block height (cardano_node_metrics_blockNum_int) has not advanced for at least 15 minutes. Transactions accepted by the Submit API are being forwarded to a node whose chain tip is frozen, so submissions will not be relayed against a current tip. This usually indicates a wedged node that a pod restart will clear.",
"summary": "{{ range $k, $v := $values -}}\n{{ if (match \"C[0-9]+\" $k) -}}\nPod: {{ $v.Labels.pod }} block height is not advancing\n{{ end }}\n{{ end }}"
},
"labels": {},
"isPaused": false,
"notification_settings": {
"receiver": "#infra-alerts"
}
}
]
}
]
}
103 changes: 103 additions & 0 deletions grafana/dashboards/shared/submit-api.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,109 @@
],
"title": "Txs in Mempool (Node)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$datasource"
},
"description": "Number of times the backing cardano-node's block height (cardano_node_metrics_blockNum_int) advanced in the last 15 minutes. A healthy, synced node advances continuously; a value of 0 means the node's tip is frozen and the Submit API is forwarding transactions to a stale node. This is the signal behind the \"Submit API backing node is stale\" alert.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "block changes / 15m",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": 0
},
{
"color": "green",
"value": 1
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 31
},
"id": 20,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"min"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "asc"
}
},
"pluginVersion": "12.3.0-18356121373.patch5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$datasource"
},
"editorMode": "code",
"expr": "changes(cardano_node_metrics_blockNum_int{namespace=\"ext-nodes-m1\", pod=~\"node-$network.*\"}[15m])",
"legendFormat": "{{pod}}",
"range": true,
"refId": "A"
}
],
"title": "Node Block Advancement (changes / 15m)",
"type": "timeseries"
}
],
"refresh": "10s",
Expand Down
18 changes: 14 additions & 4 deletions modules/grafana_alerts/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,28 @@
locals {
alert_files = fileset("${path.root}/${var.local_directory}", "*.json")
}

# Create the folder that the alert rule groups live in. Grafana requires the
# folder to exist before rule groups can be provisioned into it, so we manage it
# here with the UID defined in config.yaml (mirroring the grafana_dashboard
# module, which also creates its own folder). Only created when the directory
# actually contains alert definitions, so an empty/missing directory is a no-op.
resource "grafana_folder" "this" {
for_each = { for folder in var.folders : folder.title => folder if folder.uid == "" }
title = each.key
count = length(local.alert_files) > 0 ? 1 : 0
title = var.folder_title
uid = var.folder_uid
}

resource "grafana_rule_group" "this" {
for_each = {
for file in fileset("${path.root}/${var.local_directory}", "*.json") :
for file in local.alert_files :
file => jsondecode(templatefile("${path.root}/${var.local_directory}/${file}", {
datasource_uid_map = var.datasource_uids
}))
}

name = each.value["groups"][0]["name"]
folder_uid = var.folder_uid
folder_uid = grafana_folder.this[0].uid
interval_seconds = var.default_interval_seconds

dynamic "rule" {
Expand Down
8 changes: 0 additions & 8 deletions modules/grafana_alerts/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,6 @@ variable "local_directory" {
description = "Local directory containing alert JSON files."
}

variable "folders" {
type = list(object({
title = string
uid = optional(string, "")
}))
description = "List of folders to create in Grafana."
}

variable "folder_title" {
type = string
description = "The title of the Grafana folder to associate with alerts."
Expand Down
Loading