diff --git a/grafana/alerts/demeter/kupo-dolos.json b/grafana/alerts/demeter/kupo-dolos.json index 052f0e3..a43f3d5 100644 --- a/grafana/alerts/demeter/kupo-dolos.json +++ b/grafana/alerts/demeter/kupo-dolos.json @@ -130,6 +130,90 @@ "notification_settings": { "receiver": "#infra-alerts" } + }, + { + "uid": "bkl0kupomisscreds", + "title": "Kupo proxy unauthenticated 401 surge (fabric creds not syncing)", + "condition": "B", + "for": "15m", + "data": [ + { + "refId": "A", + "relativeTimeRange": { + "from": 600, + "to": 0 + }, + "datasourceUid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}", + "model": { + "datasource": { + "type": "prometheus", + "uid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}" + }, + "editorMode": "code", + "expr": "sum(rate(kupo_proxy_http_total_request{consumer=\".\", status_code=\"401\"}[10m]))", + "instant": false, + "intervalMs": 1000, + "legendFormat": "__auto", + "maxDataPoints": 43200, + "range": true, + "refId": "A" + } + }, + { + "refId": "B", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 0.1, + 0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "A", + "hide": false, + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "B", + "type": "threshold" + } + } + ], + "noDataState": "OK", + "execErrState": "KeepLast", + "annotations": { + "description": "The Kupo (Dolos) proxy is rejecting a sustained volume of requests with HTTP 401 and consumer=\".\" (no valid Demeter credential presented). Kupo/Dolos itself is healthy; this pattern means newly issued or rotated customer credentials are not reaching the proxy. The usual root cause is the demeter-fabric daemon (fabric-daemon-0, ns dmtr-system) silently losing its Kafka/Redpanda consumer connection so it stops syncing credentials into the cluster while its readiness probe still reports healthy. Check fabric-daemon-0 logs for rdkafka BrokerTransportFailure and 'kubectl rollout restart statefulset/fabric-daemon -n dmtr-system' to resync.", + "summary": "Kupo proxy is returning a sustained rate of unauthenticated (consumer=\".\") 401s ({{ humanize $values.A.Value }}/s over 10m) \u2014 likely demeter-fabric not syncing credentials (Kafka disconnect)." + }, + "labels": {}, + "isPaused": false, + "notification_settings": { + "receiver": "#infra-alerts" + } } ] } diff --git a/grafana/dashboards/shared/kupo-dolos.json b/grafana/dashboards/shared/kupo-dolos.json index a24e407..3c30a21 100644 --- a/grafana/dashboards/shared/kupo-dolos.json +++ b/grafana/dashboards/shared/kupo-dolos.json @@ -1199,7 +1199,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{pod=~\".*$network.*\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{network=~\".*$network.*\"}[$__rate_interval])))", "legendFormat": "p99 \u2014 {{pod}}", "range": true, "refId": "A" @@ -1210,7 +1210,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{pod=~\".*$network.*\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{network=~\".*$network.*\"}[$__rate_interval])))", "legendFormat": "p90 \u2014 {{pod}}", "range": true, "refId": "B" @@ -1221,7 +1221,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{pod=~\".*$network.*\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (le, pod) (rate(kupo_proxy_http_request_duration_seconds_bucket{network=~\".*$network.*\"}[$__rate_interval])))", "legendFormat": "p50 \u2014 {{pod}}", "range": true, "refId": "C" @@ -1316,7 +1316,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (pod) (rate(kupo_proxy_http_total_request{pod=~\".*$network.*\"}[$__rate_interval]))", + "expr": "sum by (pod) (rate(kupo_proxy_http_total_request{exported_instance=~\".*$network.*\"}[$__rate_interval]))", "legendFormat": "{{pod}}", "range": true, "refId": "A" @@ -1424,7 +1424,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status_code) (rate(kupo_proxy_http_total_request{pod=~\".*$network.*\", status_code=~\"4..|5..\"}[$__rate_interval]))", + "expr": "sum by (status_code) (rate(kupo_proxy_http_total_request{exported_instance=~\".*$network.*\", status_code=~\"4..|5..\"}[$__rate_interval]))", "legendFormat": "HTTP {{status_code}}", "range": true, "refId": "A"