From ca1aeb7d561c1e2180b3359a5dd8f9431f6b0ee5 Mon Sep 17 00:00:00 2001 From: Tripura Repalle Date: Tue, 8 Sep 2026 14:44:40 -0400 Subject: [PATCH 1/2] fix(grafana): point Kupo proxy panels at green multi-network proxy The Request Latency, Request Rate and HTTP Error Rate panels in the Kupo/Dolos dashboard filtered by pod=~".*$network.*", which only ever matched the idle per-network proxies (dolos-kupo-proxy-demeter-) and never the green multi-network proxy (dolos-kupo-proxy) that actually serves all Demeter Kupo traffic, leaving the panels flat. The green proxy exposes network via different labels per metric: - kupo_proxy_http_request_duration_seconds_bucket -> network label (cardano-mainnet/preprod/preview) - kupo_proxy_http_total_request -> exported_instance label (dolos-cardano-.dolos.svc.cluster.local:11442) Filter the latency panels by network and the rate/error panels by exported_instance so they reflect real green-proxy traffic. Signed-off-by: Tripura Repalle --- grafana/dashboards/shared/kupo-dolos.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/grafana/dashboards/shared/kupo-dolos.json b/grafana/dashboards/shared/kupo-dolos.json index a24e407..3c30a21 100644 --- a/grafana/dashboards/shared/kupo-dolos.json +++ b/grafana/dashboards/shared/kupo-dolos.json @@ -1199,7 +1199,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{pod=~\".*$network.*\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{network=~\".*$network.*\"}[$__rate_interval])))", "legendFormat": "p99 \u2014 {{pod}}", "range": true, "refId": "A" @@ -1210,7 +1210,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{pod=~\".*$network.*\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{network=~\".*$network.*\"}[$__rate_interval])))", "legendFormat": "p90 \u2014 {{pod}}", "range": true, "refId": "B" @@ -1221,7 +1221,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{pod=~\".*$network.*\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{network=~\".*$network.*\"}[$__rate_interval])))", "legendFormat": "p50 \u2014 {{pod}}", "range": true, "refId": "C" @@ -1316,7 +1316,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (pod) (rate(kupo_proxy_http_total_request{pod=~\".*$network.*\"}[$__rate_interval]))", + "expr": "sum by (pod) (rate(kupo_proxy_http_total_request{exported_instance=~\".*$network.*\"}[$__rate_interval]))", "legendFormat": "{{pod}}", "range": true, "refId": "A" @@ -1424,7 +1424,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status_code) (rate(kupo_proxy_http_total_request{pod=~\".*$network.*\", status_code=~\"4..|5..\"}[$__rate_interval]))", + "expr": "sum by (status_code) (rate(kupo_proxy_http_total_request{exported_instance=~\".*$network.*\", status_code=~\"4..|5..\"}[$__rate_interval]))", "legendFormat": "HTTP {{status_code}}", "range": true, "refId": "A" From c6c78e7369bae34ef15c06ae3e812cc97f9945a0 Mon Sep 17 00:00:00 2001 From: Tripura Repalle Date: Tue, 8 Sep 2026 15:24:54 -0400 Subject: [PATCH 2/2] feat(alerts): detect fabric credential-sync failure via kupo 401 surge Add a Grafana alert to the Kupo Dolos group that fires when the Kupo (Dolos) proxy returns a sustained rate of unauthenticated 401s (consumer=".") above 0.1 req/s for 15m. Kupo/Dolos themselves are healthy in this case; the pattern indicates newly issued or rotated Demeter credentials are not reaching the proxy because the demeter-fabric daemon has silently lost its Kafka/Redpanda consumer connection (readiness stays green while it stops syncing creds). There is no direct fabric Kafka metric exported, so this alerts on the customer-facing symptom, with the fabric root cause and remediation in the annotation. Signed-off-by: Tripura Repalle --- grafana/alerts/demeter/kupo-dolos.json | 84 ++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/grafana/alerts/demeter/kupo-dolos.json b/grafana/alerts/demeter/kupo-dolos.json index 052f0e3..a43f3d5 100644 --- a/grafana/alerts/demeter/kupo-dolos.json +++ b/grafana/alerts/demeter/kupo-dolos.json @@ -130,6 +130,90 @@ "notification_settings": { "receiver": "#infra-alerts" } + }, + { + "uid": "bkl0kupomisscreds", + "title": "Kupo proxy unauthenticated 401 surge (fabric creds not syncing)", + "condition": "B", + "for": "15m", + "data": [ + { + "refId": "A", + "relativeTimeRange": { + "from": 600, + "to": 0 + }, + "datasourceUid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}", + "model": { + "datasource": { + "type": "prometheus", + "uid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}" + }, + "editorMode": "code", + "expr": "sum(rate(kupo_proxy_http_total_request{consumer=\".\", status_code=\"401\"}[10m]))", + "instant": false, + "intervalMs": 1000, + "legendFormat": "__auto", + "maxDataPoints": 43200, + "range": true, + "refId": "A" + } + }, + { + "refId": "B", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 0.1, + 0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "A", + "hide": false, + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "B", + "type": "threshold" + } + } + ], + "noDataState": "OK", + "execErrState": "KeepLast", + "annotations": { + "description": "The Kupo (Dolos) proxy is rejecting a sustained volume of requests with HTTP 401 and consumer=\".\" (no valid Demeter credential presented). Kupo/Dolos itself is healthy; this pattern means newly issued or rotated customer credentials are not reaching the proxy. The usual root cause is the demeter-fabric daemon (fabric-daemon-0, ns dmtr-system) silently losing its Kafka/Redpanda consumer connection so it stops syncing credentials into the cluster while its readiness probe still reports healthy. Check fabric-daemon-0 logs for rdkafka BrokerTransportFailure and 'kubectl rollout restart statefulset/fabric-daemon -n dmtr-system' to resync.", + "summary": "Kupo proxy is returning a sustained rate of unauthenticated (consumer=\".\") 401s ({{ humanize $values.A.Value }}/s over 10m) \u2014 likely demeter-fabric not syncing credentials (Kafka disconnect)." + }, + "labels": {}, + "isPaused": false, + "notification_settings": { + "receiver": "#infra-alerts" + } } ] }