Skip to content

fix(grafana): point Kupo proxy panels at green multi-network proxy #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tripura-repalle merged 2 commits into from
Sep 8, 2026
Merged

fix(grafana): point Kupo proxy panels at green multi-network proxy #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ca1aeb7
fix(grafana): point Kupo proxy panels at green multi-network proxy
tripura-repalle Sep 8, 2026
c6c78e7
feat(alerts): detect fabric credential-sync failure via kupo 401 surge
tripura-repalle Sep 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions grafana/alerts/demeter/kupo-dolos.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,90 @@
"notification_settings": {
"receiver": "#infra-alerts"
}
},
{
"uid": "bkl0kupomisscreds",
"title": "Kupo proxy unauthenticated 401 surge (fabric creds not syncing)",
"condition": "B",
"for": "15m",
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 600,
"to": 0
},
"datasourceUid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}",
"model": {
"datasource": {
"type": "prometheus",
"uid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}"
},
"editorMode": "code",
"expr": "sum(rate(kupo_proxy_http_total_request{consumer=\".\", status_code=\"401\"}[10m]))",
"instant": false,
"intervalMs": 1000,
"legendFormat": "__auto",
"maxDataPoints": 43200,
"range": true,
"refId": "A"
}
},
{
"refId": "B",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
0.1,
0
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": []
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"datasource": {
"name": "Expression",
"type": "__expr__",
"uid": "__expr__"
},
"expression": "A",
"hide": false,
"intervalMs": 1000,
"maxDataPoints": 43200,
"refId": "B",
"type": "threshold"
}
}
],
"noDataState": "OK",
"execErrState": "KeepLast",
"annotations": {
"description": "The Kupo (Dolos) proxy is rejecting a sustained volume of requests with HTTP 401 and consumer=\".\" (no valid Demeter credential presented). Kupo/Dolos itself is healthy; this pattern means newly issued or rotated customer credentials are not reaching the proxy. The usual root cause is the demeter-fabric daemon (fabric-daemon-0, ns dmtr-system) silently losing its Kafka/Redpanda consumer connection so it stops syncing credentials into the cluster while its readiness probe still reports healthy. Check fabric-daemon-0 logs for rdkafka BrokerTransportFailure and 'kubectl rollout restart statefulset/fabric-daemon -n dmtr-system' to resync.",
"summary": "Kupo proxy is returning a sustained rate of unauthenticated (consumer=\".\") 401s ({{ humanize $values.A.Value }}/s over 10m) \u2014 likely demeter-fabric not syncing credentials (Kafka disconnect)."
},
"labels": {},
"isPaused": false,
"notification_settings": {
"receiver": "#infra-alerts"
}
}
]
}
Expand Down
10 changes: 5 additions & 5 deletions grafana/dashboards/shared/kupo-dolos.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1199,7 +1199,7 @@
"uid": "$datasource"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{pod=~\".*$network.*\"}[$__rate_interval])))",
"expr": "histogram_quantile(0.99, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{network=~\".*$network.*\"}[$__rate_interval])))",
"legendFormat": "p99 \u2014 {{pod}}",
"range": true,
"refId": "A"
Expand All @@ -1210,7 +1210,7 @@
"uid": "$datasource"
},
"editorMode": "code",
"expr": "histogram_quantile(0.90, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{pod=~\".*$network.*\"}[$__rate_interval])))",
"expr": "histogram_quantile(0.90, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{network=~\".*$network.*\"}[$__rate_interval])))",
"legendFormat": "p90 \u2014 {{pod}}",
"range": true,
"refId": "B"
Expand All @@ -1221,7 +1221,7 @@
"uid": "$datasource"
},
"editorMode": "code",
"expr": "histogram_quantile(0.50, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{pod=~\".*$network.*\"}[$__rate_interval])))",
"expr": "histogram_quantile(0.50, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{network=~\".*$network.*\"}[$__rate_interval])))",
"legendFormat": "p50 \u2014 {{pod}}",
"range": true,
"refId": "C"
Expand Down Expand Up @@ -1316,7 +1316,7 @@
"uid": "$datasource"
},
"editorMode": "code",
"expr": "sum by (pod) (rate(kupo_proxy_http_total_request{pod=~\".*$network.*\"}[$__rate_interval]))",
"expr": "sum by (pod) (rate(kupo_proxy_http_total_request{exported_instance=~\".*$network.*\"}[$__rate_interval]))",
"legendFormat": "{{pod}}",
"range": true,
"refId": "A"
Expand Down Expand Up @@ -1424,7 +1424,7 @@
"uid": "$datasource"
},
"editorMode": "code",
"expr": "sum by (status_code) (rate(kupo_proxy_http_total_request{pod=~\".*$network.*\", status_code=~\"4..|5..\"}[$__rate_interval]))",
"expr": "sum by (status_code) (rate(kupo_proxy_http_total_request{exported_instance=~\".*$network.*\", status_code=~\"4..|5..\"}[$__rate_interval]))",
"legendFormat": "HTTP {{status_code}}",
"range": true,
"refId": "A"
Expand Down
Loading