From da062dc50ea1570fae429bd42c20b03c606059e7 Mon Sep 17 00:00:00 2001 From: func25 Date: Wed, 23 Sep 2026 18:12:20 +0700 Subject: [PATCH 1/3] fix --- app/vlinsert/datadog/datadog.go | 6 ++++++ app/vlinsert/elasticsearch/elasticsearch.go | 4 ++++ app/vlinsert/internalinsert/internalinsert.go | 2 ++ app/vlinsert/jsonline/jsonline.go | 4 ++++ app/vlinsert/loki/loki_json.go | 4 ++++ app/vlinsert/loki/loki_protobuf.go | 4 ++++ app/vlinsert/nativeinsert/nativeinsert.go | 2 ++ app/vlinsert/nativeinsert/nativemultitenant/multitenant.go | 2 ++ app/vlinsert/opentelemetry/opentelemetry.go | 2 ++ app/vlinsert/splunk/splunk.go | 2 ++ docs/victorialogs/CHANGELOG.md | 1 + 11 files changed, 33 insertions(+) diff --git a/app/vlinsert/datadog/datadog.go b/app/vlinsert/datadog/datadog.go index 659e119081..945c463f22 100644 --- a/app/vlinsert/datadog/datadog.go +++ b/app/vlinsert/datadog/datadog.go @@ -52,6 +52,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool { var err error ts, err = strconv.ParseInt(tsValue, 10, 64) if err != nil { + v2LogsErrorsTotal.Inc() httpserver.Errorf(w, r, "could not parse dd-message-timestamp header value: %s", err) return true } @@ -62,12 +63,14 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool { cp, err := insertutil.GetCommonParams(r) if err != nil { + v2LogsErrorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return true } if len(cp.StreamFields) == 0 { if err := logstorage.CheckStreamFieldNames(*datadogStreamFields); err != nil { + v2LogsErrorsTotal.Inc() httpserver.Errorf(w, r, "invalid stream field names at -datadog.streamFields=%s: %s", datadogStreamFields, err) return true } @@ -79,6 +82,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool { } if err := insertutil.CanWriteData(); err != nil { + v2LogsErrorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return true } @@ -91,6 +95,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool { return err }) if err != nil { + v2LogsErrorsTotal.Inc() httpserver.Errorf(w, r, "cannot read DataDog protocol data: %s", err) return true } @@ -106,6 +111,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool { var ( v2LogsRequestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/datadog/api/v2/logs"}`) + v2LogsErrorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/datadog/api/v2/logs"}`) v2LogsRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`) ) diff --git a/app/vlinsert/elasticsearch/elasticsearch.go b/app/vlinsert/elasticsearch/elasticsearch.go index fd9e14562e..ac60897df8 100644 --- a/app/vlinsert/elasticsearch/elasticsearch.go +++ b/app/vlinsert/elasticsearch/elasticsearch.go @@ -99,10 +99,12 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool { cp, err := insertutil.GetCommonParams(r) if err != nil { + bulkErrorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return true } if err := insertutil.CanWriteData(); err != nil { + bulkErrorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return true } @@ -112,6 +114,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool { n, err := readBulkRequest(streamName, r.Body, encoding, cp.TimeFields, cp.MsgFields, cp.PreserveJSONKeys, lmp) lmp.MustClose() if err != nil { + bulkErrorsTotal.Inc() httpserver.Errorf(w, r, "cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields) return true } @@ -135,6 +138,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool { var ( bulkRequestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`) + bulkErrorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/elasticsearch/_bulk"}`) bulkRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`) ) diff --git a/app/vlinsert/internalinsert/internalinsert.go b/app/vlinsert/internalinsert/internalinsert.go index aa538bbac7..3e4b6857d8 100644 --- a/app/vlinsert/internalinsert/internalinsert.go +++ b/app/vlinsert/internalinsert/internalinsert.go @@ -37,10 +37,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) { cp, err := insertutil.GetCommonParams(r) if err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } if err := insertutil.CanWriteData(); err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } diff --git a/app/vlinsert/jsonline/jsonline.go b/app/vlinsert/jsonline/jsonline.go index dcff32bf5f..edd3749f34 100644 --- a/app/vlinsert/jsonline/jsonline.go +++ b/app/vlinsert/jsonline/jsonline.go @@ -30,16 +30,19 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) { cp, err := insertutil.GetCommonParams(r) if err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } if err := insertutil.CanWriteData(); err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } wcr, err := writeconcurrencylimiter.GetReader(r.Body) if err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "cannot start reading jsonline request: %s", err) return } @@ -48,6 +51,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) { encoding := r.Header.Get("Content-Encoding") reader, err := protoparserutil.GetUncompressedReader(wcr, encoding) if err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "cannot decode jsonline request: %s", err) return } diff --git a/app/vlinsert/loki/loki_json.go b/app/vlinsert/loki/loki_json.go index bbad760875..5098f5da8b 100644 --- a/app/vlinsert/loki/loki_json.go +++ b/app/vlinsert/loki/loki_json.go @@ -29,10 +29,12 @@ func handleJSON(r *http.Request, w http.ResponseWriter) { cp, err := getCommonParams(r) if err != nil { + errorsJSONTotal.Inc() httpserver.Errorf(w, r, "cannot parse common params from request: %s", err) return } if err := insertutil.CanWriteData(); err != nil { + errorsJSONTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } @@ -46,6 +48,7 @@ func handleJSON(r *http.Request, w http.ResponseWriter) { return err }) if err != nil { + errorsJSONTotal.Inc() httpserver.Errorf(w, r, "cannot read Loki json data: %s", err) return } @@ -61,6 +64,7 @@ func handleJSON(r *http.Request, w http.ResponseWriter) { var ( requestsJSONTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`) + errorsJSONTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/loki/api/v1/push",format="json"}`) requestJSONDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`) ) diff --git a/app/vlinsert/loki/loki_protobuf.go b/app/vlinsert/loki/loki_protobuf.go index cf4344a077..f2a0e5118d 100644 --- a/app/vlinsert/loki/loki_protobuf.go +++ b/app/vlinsert/loki/loki_protobuf.go @@ -19,10 +19,12 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) { cp, err := getCommonParams(r) if err != nil { + errorsProtobufTotal.Inc() httpserver.Errorf(w, r, "cannot parse common params from request: %s", err) return } if err := insertutil.CanWriteData(); err != nil { + errorsProtobufTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } @@ -41,6 +43,7 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) { return err }) if err != nil { + errorsProtobufTotal.Inc() httpserver.Errorf(w, r, "cannot read Loki protobuf data: %s", err) return } @@ -56,6 +59,7 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) { var ( requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`) + errorsProtobufTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/loki/api/v1/push",format="protobuf"}`) requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`) ) diff --git a/app/vlinsert/nativeinsert/nativeinsert.go b/app/vlinsert/nativeinsert/nativeinsert.go index 98156ba25f..40d1b2d4a8 100644 --- a/app/vlinsert/nativeinsert/nativeinsert.go +++ b/app/vlinsert/nativeinsert/nativeinsert.go @@ -39,10 +39,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) { cp, err := insertutil.GetCommonParams(r) if err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } if err := insertutil.CanWriteData(); err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } diff --git a/app/vlinsert/nativeinsert/nativemultitenant/multitenant.go b/app/vlinsert/nativeinsert/nativemultitenant/multitenant.go index ad707bb124..a32d87f69f 100644 --- a/app/vlinsert/nativeinsert/nativemultitenant/multitenant.go +++ b/app/vlinsert/nativeinsert/nativemultitenant/multitenant.go @@ -33,10 +33,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) { cp, err := insertutil.GetCommonParams(r) if err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } if err := insertutil.CanWriteData(); err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } diff --git a/app/vlinsert/opentelemetry/opentelemetry.go b/app/vlinsert/opentelemetry/opentelemetry.go index 308be5ca26..769a83ca10 100644 --- a/app/vlinsert/opentelemetry/opentelemetry.go +++ b/app/vlinsert/opentelemetry/opentelemetry.go @@ -40,10 +40,12 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) { cp, err := insertutil.GetCommonParams(r) if err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "cannot parse common params from request: %s", err) return } if err := insertutil.CanWriteData(); err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } diff --git a/app/vlinsert/splunk/splunk.go b/app/vlinsert/splunk/splunk.go index 832be9a5e5..f1d0072f4d 100644 --- a/app/vlinsert/splunk/splunk.go +++ b/app/vlinsert/splunk/splunk.go @@ -135,10 +135,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) { cp, err := getCommonParams(r) if err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } if err := insertutil.CanWriteData(); err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "%s", err) return } diff --git a/docs/victorialogs/CHANGELOG.md b/docs/victorialogs/CHANGELOG.md index 52e15f586d..79a0b6d984 100644 --- a/docs/victorialogs/CHANGELOG.md +++ b/docs/victorialogs/CHANGELOG.md @@ -59,6 +59,7 @@ according to the following docs: * BUGFIX: [cluster version](https://docs.victoriametrics.com/victorialogs/cluster/): evenly spread rerouted data across available `vlstorage` nodes. Previously, healthy nodes adjacent to unavailable nodes in the `-storageNode` list could receive much more data, resulting in uneven resource usage. See [#1548](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1548). * BUGFIX: [cluster version](https://docs.victoriametrics.com/victorialogs/cluster/): properly cancel queries already running on `vlstorage` when the corresponding query is canceled or times out on `vlselect`. Previously, `vlstorage` could fail to detect the disconnected `vlselect`, causing these queries to continue running and waste CPU and disk IO. See [#1672](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1672). * BUGFIX: [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/) and [querying](https://docs.victoriametrics.com/victorialogs/querying/): properly handle logs containing duplicate [stream field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) names. Previously, [v1.52.0](https://github.com/VictoriaMetrics/VictoriaLogs/releases/tag/v1.52.0) could panic when ingesting such logs in single-node VictoriaLogs, drop them during ingestion in VictoriaLogs cluster, or panic when querying such data written by earlier releases. See [#1603](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1603) and [#1604](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1604). +* BUGFIX: [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/): increment [`vl_http_errors_total`](https://docs.victoriametrics.com/victorialogs/metrics/#vl_http_errors_total) for failed insert requests, such as requests with invalid params or requests rejected because the storage is in read-only mode (e.g. because of lack of free disk space). Also expose the missing `vl_http_errors_total` for Loki, Elasticsearch and DataDog ingestion endpoints. See [#1811](https://github.com/VictoriaMetrics/VictoriaLogs/pull/1811). * BUGFIX: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): fix [`week_range[Sun,Sun]` filter](https://docs.victoriametrics.com/victorialogs/logsql/#week-range-filter) when it is used inside the [`filter` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#filter-pipe). Previously, it could fail to match rows on Sunday. See [#1335](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1335). * BUGFIX: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): properly execute queries returning the last `N` logs (such as queries ending with [`| sort by (_time desc) limit N`](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe)) when they contain pipes writing to the `_time` field, such as `math ... as _time`, `replace (...) at _time` or `extract "<_time>"`. Previously such queries could fail with the `missing _time field in the query results` error or return logs in the wrong order when they were executed on wide time ranges. See [#1727](https://github.com/VictoriaMetrics/VictoriaLogs/pull/1727). * BUGFIX: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): fix [`sort by (_time) limit N`](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe) returning logs out of order when the query pipeline included pipes like [`unpack_json`](https://docs.victoriametrics.com/victorialogs/logsql/#unpack_json-pipe) that overwrite `_time`. See [#1360](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1360). From 2ebc6270c489d084548cdf5092576bc29494e602 Mon Sep 17 00:00:00 2001 From: func25 Date: Tue, 6 Oct 2026 15:38:54 +0700 Subject: [PATCH 2/3] update --- app/vlinsert/opentelemetry/opentelemetry.go | 2 +- app/vlinsert/splunk/splunk.go | 3 ++- docs/victorialogs/metrics.md | 11 ++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/app/vlinsert/opentelemetry/opentelemetry.go b/app/vlinsert/opentelemetry/opentelemetry.go index 769a83ca10..02e30cc4a8 100644 --- a/app/vlinsert/opentelemetry/opentelemetry.go +++ b/app/vlinsert/opentelemetry/opentelemetry.go @@ -59,6 +59,7 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) { return err }) if err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "cannot read OpenTelemetry protocol data: %s", err) return } @@ -88,7 +89,6 @@ func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFie } if err := decodeLogsData(data, pushLogs); err != nil { - errorsTotal.Inc() return fmt.Errorf("cannot decode LogsData request from %d bytes: %w", len(data), err) } return nil diff --git a/app/vlinsert/splunk/splunk.go b/app/vlinsert/splunk/splunk.go index f1d0072f4d..f2848a43e1 100644 --- a/app/vlinsert/splunk/splunk.go +++ b/app/vlinsert/splunk/splunk.go @@ -152,6 +152,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) { return processEvent(data, lmp, cp.TimeFields, cp.MsgFields, cp.PreserveJSONKeys) }) if err != nil { + errorsTotal.Inc() httpserver.Errorf(w, r, "cannot read Splunk request: %s", err) return } @@ -179,8 +180,8 @@ func processEvent(data []byte, lmp insertutil.LogMessageProcessor, timeFields, m n++ } if err := s.Error(); err != nil { - errorsTotal.Add(1) if n > 0 { + errorsTotal.Add(1) logger.Warnf("splunk: failed to parse JSON message #%d: %s", n+1, err) return nil } diff --git a/docs/victorialogs/metrics.md b/docs/victorialogs/metrics.md index a86ea9486d..c13fc2beb4 100644 --- a/docs/victorialogs/metrics.md +++ b/docs/victorialogs/metrics.md @@ -52,7 +52,16 @@ These metrics follow the Prometheus exposition format and can be used for monito - `path`: endpoint path - `format`: request format when applicable (e.g. `protobuf`) -**Description:** Errors encountered while processing requests for the given endpoint. The counter is incremented by endpoint handlers on non-trivial processing errors (e.g. request decoding/parsing failures or query execution errors) and may differ from the number of HTTP responses with error status. For line-oriented ingestion (e.g. `/insert/jsonline`), it is incremented per invalid log line inside a request. +**Description:** The number of errors at the given endpoint. + +For data ingestion endpoints, the counter is incremented in two cases: + +- A request is rejected, for example because of invalid request args, broken request body or because the storage is in read-only mode. +- A request is accepted, but some of its log lines or events are invalid and skipped. In this case the counter is incremented per every skipped log line or event, e.g. at `/insert/jsonline`. + +So the counter can be bigger than the number of failed requests. + +For `/internal/select/*` endpoints, the counter is incremented per every failed query, except for queries canceled by the client. ### vl_http_request_duration_seconds **Type:** Summary From 9f34b728367b6cb697dcb89874adf95ba51da849 Mon Sep 17 00:00:00 2001 From: func25 Date: Tue, 6 Oct 2026 15:45:53 +0700 Subject: [PATCH 3/3] update --- docs/victorialogs/metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/victorialogs/metrics.md b/docs/victorialogs/metrics.md index c13fc2beb4..92aa050507 100644 --- a/docs/victorialogs/metrics.md +++ b/docs/victorialogs/metrics.md @@ -56,7 +56,7 @@ These metrics follow the Prometheus exposition format and can be used for monito For data ingestion endpoints, the counter is incremented in two cases: -- A request is rejected, for example because of invalid request args, broken request body or because the storage is in read-only mode. +- A request is counted at `vl_http_requests_total` and then rejected, for example because of invalid request args, broken request body or because the storage is in read-only mode. Requests rejected earlier, such as requests with unsupported HTTP method, content type or protocol version, are counted neither at `vl_http_requests_total` nor at `vl_http_errors_total`. - A request is accepted, but some of its log lines or events are invalid and skipped. In this case the counter is incremented per every skipped log line or event, e.g. at `/insert/jsonline`. So the counter can be bigger than the number of failed requests.