Feat/kubeletstats psi metrics

.chloggen/kubeletstats-psi-metrics.yaml

@@ -0,0 +1,30 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog)
+component: receiver/kubelet_stats
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add opt-in PSI (Pressure Stall Information) metrics for node, pod, and container CPU/memory/IO pressure.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [50943]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Adds 18 new metrics (disabled by default, stability: development) covering `some` and `full`
+  pressure stall percentages and cumulative nanoseconds for CPU, memory, and IO across node,
+  pod, and container scopes. Requires Linux >= 4.20 with cgroup v2 on the node.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

receiver/kubeletstatsreceiver/documentation.md

@@ -386,6 +386,93 @@ metrics:
     enabled: true
 ```
 
+### container.cpu.pressure.avg
+
+Rolling average percentage of time tasks in the container cgroup were stalled waiting for CPU. Requires Linux >= 4.20 with cgroup v2. Values may transiently exceed 100.
+
+| Unit | Metric Type | Value Type | Stability |
+| ---- | ----------- | ---------- | --------- |
+| % | Gauge | Double | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+| pressure.window | The averaging window for PSI stall percentage metrics. | Str: ``10s``, ``60s``, ``300s`` | Recommended | - |
+
+### container.cpu.pressure.total
+
+Cumulative total time tasks in the container cgroup were stalled waiting for CPU. Requires Linux >= 4.20 with cgroup v2.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability |
+| ---- | ----------- | ---------- | ----------------------- | --------- | --------- |
+| ns | Sum | Int | Cumulative | true | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+
+### container.io.pressure.avg
+
+Rolling average percentage of time tasks in the container cgroup were stalled waiting for IO. Requires Linux >= 4.20 with cgroup v2. Values may transiently exceed 100.
+
+| Unit | Metric Type | Value Type | Stability |
+| ---- | ----------- | ---------- | --------- |
+| % | Gauge | Double | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+| pressure.window | The averaging window for PSI stall percentage metrics. | Str: ``10s``, ``60s``, ``300s`` | Recommended | - |
+
+### container.io.pressure.total
+
+Cumulative total time tasks in the container cgroup were stalled waiting for IO. Requires Linux >= 4.20 with cgroup v2.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability |
+| ---- | ----------- | ---------- | ----------------------- | --------- | --------- |
+| ns | Sum | Int | Cumulative | true | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+
+### container.memory.pressure.avg
+
+Rolling average percentage of time tasks in the container cgroup were stalled waiting for memory. Requires Linux >= 4.20 with cgroup v2. Values may transiently exceed 100.
+
+| Unit | Metric Type | Value Type | Stability |
+| ---- | ----------- | ---------- | --------- |
+| % | Gauge | Double | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+| pressure.window | The averaging window for PSI stall percentage metrics. | Str: ``10s``, ``60s``, ``300s`` | Recommended | - |
+
+### container.memory.pressure.total
+
+Cumulative total time tasks in the container cgroup were stalled waiting for memory. Requires Linux >= 4.20 with cgroup v2.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability |
+| ---- | ----------- | ---------- | ----------------------- | --------- | --------- |
+| ns | Sum | Int | Cumulative | true | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+
 ### container.uptime
 
 The time since the container started
@@ -456,6 +543,35 @@ Container memory utilization as a ratio of the container's requests
 | ---- | ----------- | ---------- | --------- |
 | 1 | Gauge | Double | Development |
 
+### k8s.node.cpu.pressure.avg
+
+Rolling average percentage of time tasks in the node cgroup were stalled waiting for CPU. Requires Linux >= 4.20 with cgroup v2. Values may transiently exceed 100.
+
+| Unit | Metric Type | Value Type | Stability |
+| ---- | ----------- | ---------- | --------- |
+| % | Gauge | Double | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+| pressure.window | The averaging window for PSI stall percentage metrics. | Str: ``10s``, ``60s``, ``300s`` | Recommended | - |
+
+### k8s.node.cpu.pressure.total
+
+Cumulative total time tasks in the node cgroup were stalled waiting for CPU. Requires Linux >= 4.20 with cgroup v2.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability |
+| ---- | ----------- | ---------- | ----------------------- | --------- | --------- |
+| ns | Sum | Int | Cumulative | true | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+
 ### k8s.node.filesystem.inode.count
 
 Total number of inodes in the node's root filesystem.
@@ -472,6 +588,64 @@ Number of free inodes in the node's root filesystem.
 | ---- | ----------- | ---------- | ----------------------- | --------- | --------- |
 | {inode} | Sum | Int | Cumulative | false | Development |
 
+### k8s.node.io.pressure.avg
+
+Rolling average percentage of time tasks in the node cgroup were stalled waiting for IO. Requires Linux >= 4.20 with cgroup v2. Values may transiently exceed 100.
+
+| Unit | Metric Type | Value Type | Stability |
+| ---- | ----------- | ---------- | --------- |
+| % | Gauge | Double | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+| pressure.window | The averaging window for PSI stall percentage metrics. | Str: ``10s``, ``60s``, ``300s`` | Recommended | - |
+
+### k8s.node.io.pressure.total
+
+Cumulative total time tasks in the node cgroup were stalled waiting for IO. Requires Linux >= 4.20 with cgroup v2.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability |
+| ---- | ----------- | ---------- | ----------------------- | --------- | --------- |
+| ns | Sum | Int | Cumulative | true | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+
+### k8s.node.memory.pressure.avg
+
+Rolling average percentage of time tasks in the node cgroup were stalled waiting for memory. Requires Linux >= 4.20 with cgroup v2. Values may transiently exceed 100.
+
+| Unit | Metric Type | Value Type | Stability |
+| ---- | ----------- | ---------- | --------- |
+| % | Gauge | Double | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+| pressure.window | The averaging window for PSI stall percentage metrics. | Str: ``10s``, ``60s``, ``300s`` | Recommended | - |
+
+### k8s.node.memory.pressure.total
+
+Cumulative total time tasks in the node cgroup were stalled waiting for memory. Requires Linux >= 4.20 with cgroup v2.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability |
+| ---- | ----------- | ---------- | ----------------------- | --------- | --------- |
+| ns | Sum | Int | Cumulative | true | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+
 ### k8s.node.system_container.cpu.time
 
 Total cumulative CPU time (sum of all cores) spent by the system container since its creation
@@ -520,6 +694,35 @@ Pod cpu utilization as a ratio of the node's capacity
 | ---- | ----------- | ---------- | --------- |
 | 1 | Gauge | Double | Development |
 
+### k8s.pod.cpu.pressure.avg
+
+Rolling average percentage of time tasks in the pod cgroup were stalled waiting for CPU. Requires Linux >= 4.20 with cgroup v2. Values may transiently exceed 100.
+
+| Unit | Metric Type | Value Type | Stability |
+| ---- | ----------- | ---------- | --------- |
+| % | Gauge | Double | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+| pressure.window | The averaging window for PSI stall percentage metrics. | Str: ``10s``, ``60s``, ``300s`` | Recommended | - |
+
+### k8s.pod.cpu.pressure.total
+
+Cumulative total time tasks in the pod cgroup were stalled waiting for CPU. Requires Linux >= 4.20 with cgroup v2.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability |
+| ---- | ----------- | ---------- | ----------------------- | --------- | --------- |
+| ns | Sum | Int | Cumulative | true | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+
 ### k8s.pod.cpu_limit_utilization
 
 Pod cpu utilization as a ratio of the pod's total container limits. If any container is missing a limit the metric is not emitted.
@@ -536,6 +739,35 @@ Pod cpu utilization as a ratio of the pod's total container requests. If any con
 | ---- | ----------- | ---------- | --------- |
 | 1 | Gauge | Double | Development |
 
+### k8s.pod.io.pressure.avg
+
+Rolling average percentage of time tasks in the pod cgroup were stalled waiting for IO. Requires Linux >= 4.20 with cgroup v2. Values may transiently exceed 100.
+
+| Unit | Metric Type | Value Type | Stability |
+| ---- | ----------- | ---------- | --------- |
+| % | Gauge | Double | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+| pressure.window | The averaging window for PSI stall percentage metrics. | Str: ``10s``, ``60s``, ``300s`` | Recommended | - |
+
+### k8s.pod.io.pressure.total
+
+Cumulative total time tasks in the pod cgroup were stalled waiting for IO. Requires Linux >= 4.20 with cgroup v2.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability |
+| ---- | ----------- | ---------- | ----------------------- | --------- | --------- |
+| ns | Sum | Int | Cumulative | true | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+
 ### k8s.pod.memory.node.utilization
 
 Pod memory utilization as a ratio of the node's capacity
@@ -544,6 +776,35 @@ Pod memory utilization as a ratio of the node's capacity
 | ---- | ----------- | ---------- | --------- |
 | 1 | Gauge | Double | Development |
 
+### k8s.pod.memory.pressure.avg
+
+Rolling average percentage of time tasks in the pod cgroup were stalled waiting for memory. Requires Linux >= 4.20 with cgroup v2. Values may transiently exceed 100.
+
+| Unit | Metric Type | Value Type | Stability |
+| ---- | ----------- | ---------- | --------- |
+| % | Gauge | Double | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+| pressure.window | The averaging window for PSI stall percentage metrics. | Str: ``10s``, ``60s``, ``300s`` | Recommended | - |
+
+### k8s.pod.memory.pressure.total
+
+Cumulative total time tasks in the pod cgroup were stalled waiting for memory. Requires Linux >= 4.20 with cgroup v2.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability |
+| ---- | ----------- | ---------- | ----------------------- | --------- | --------- |
+| ns | Sum | Int | Cumulative | true | Development |
+
+#### Attributes
+
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| pressure.type | The PSI pressure type (some = at least one task stalled; full = all tasks stalled). | Str: ``some``, ``full`` | Recommended | - |
+
 ### k8s.pod.memory_limit_utilization
 
 Pod memory utilization as a ratio of the pod's total container limits. If any container is missing a limit the metric is not emitted.

receiver/kubeletstatsreceiver/internal/kubelet/accumulator.go

@@ -62,6 +62,19 @@ func (a *metricDataAccumulator) nodeStats(s stats.NodeStats) {
 	addMemoryMetrics(a.mbs.NodeMetricsBuilder, metadata.NodeMemoryMetrics, s.Memory, currentTime, resources{}, 0)
 	addFilesystemMetrics(a.mbs.NodeMetricsBuilder, metadata.NodeFilesystemMetrics, s.Fs, currentTime)
 	addNetworkMetrics(a.mbs.NodeMetricsBuilder, metadata.NodeNetworkMetrics, s.Network, currentTime, a.allNetworkInterfaces[NodeMetricGroup])
+	// PSI (Pressure Stall Information) — requires Linux >= 4.20, cgroup v2.
+	// PSI fields are *PSIStats pointers; addPSIMetrics/addIOPSIMetrics are nil-safe.
+	if s.CPU != nil {
+		addPSIMetrics(a.mbs.NodeMetricsBuilder, metadata.NodeCPUPressureMetrics, s.CPU.PSI, currentTime)
+	}
+	if s.Memory != nil {
+		addPSIMetrics(a.mbs.NodeMetricsBuilder, metadata.NodeMemoryPressureMetrics, s.Memory.PSI, currentTime)
+	}
+	if s.IO != nil {
+		addPSIMetrics(a.mbs.NodeMetricsBuilder, metadata.NodeIOPressureMetrics, s.IO.PSI, currentTime)
+	}
+	// TODO(#ISSUE): add PSI for system containers (s.SystemContainers[i].CPU.PSI / Memory.PSI / IO).
+	// System containers include etcd, kube-apiserver, etc. Open a GitHub issue and replace #ISSUE.
 	// todo s.Runtime.ImageFs
 	rb := a.mbs.NodeMetricsBuilder.NewResourceBuilder()
 	rb.SetK8sNodeName(s.NodeName)
@@ -97,6 +110,16 @@ func (a *metricDataAccumulator) podStats(s *stats.PodStats) {
 	addMemoryMetrics(a.mbs.PodMetricsBuilder, metadata.PodMemoryMetrics, s.Memory, currentTime, a.metadata.podResources[s.PodRef.UID], a.metadata.nodeInfo.MemoryCapacity)
 	addFilesystemMetrics(a.mbs.PodMetricsBuilder, metadata.PodFilesystemMetrics, s.EphemeralStorage, currentTime)
 	addNetworkMetrics(a.mbs.PodMetricsBuilder, metadata.PodNetworkMetrics, s.Network, currentTime, a.allNetworkInterfaces[PodMetricGroup])
+	// PSI — nil-safe; absent on cgroup v1 / Windows nodes.
+	if s.CPU != nil {
+		addPSIMetrics(a.mbs.PodMetricsBuilder, metadata.PodCPUPressureMetrics, s.CPU.PSI, currentTime)
+	}
+	if s.Memory != nil {
+		addPSIMetrics(a.mbs.PodMetricsBuilder, metadata.PodMemoryPressureMetrics, s.Memory.PSI, currentTime)
+	}
+	if s.IO != nil {
+		addPSIMetrics(a.mbs.PodMetricsBuilder, metadata.PodIOPressureMetrics, s.IO.PSI, currentTime)
+	}
 
 	rb := a.mbs.PodMetricsBuilder.NewResourceBuilder()
 	rb.SetK8sPodUID(s.PodRef.UID)
@@ -134,6 +157,16 @@ func (a *metricDataAccumulator) containerStats(sPod *stats.PodStats, s *stats.Co
 
 	addEphemeralStorageMetrics(a.mbs.ContainerMetricsBuilder, metadata.ContainerEphemeralStorageMetrics, s.Rootfs, metadata.AttributeFsTypeRootfs, currentTime)
 	addEphemeralStorageMetrics(a.mbs.ContainerMetricsBuilder, metadata.ContainerEphemeralStorageMetrics, s.Logs, metadata.AttributeFsTypeLogs, currentTime)
+	// PSI — nil-safe; absent on cgroup v1 / Windows nodes.
+	if s.CPU != nil {
+		addPSIMetrics(a.mbs.ContainerMetricsBuilder, metadata.ContainerCPUPressureMetrics, s.CPU.PSI, currentTime)
+	}
+	if s.Memory != nil {
+		addPSIMetrics(a.mbs.ContainerMetricsBuilder, metadata.ContainerMemoryPressureMetrics, s.Memory.PSI, currentTime)
+	}
+	if s.IO != nil {
+		addPSIMetrics(a.mbs.ContainerMetricsBuilder, metadata.ContainerIOPressureMetrics, s.IO.PSI, currentTime)
+	}
 
 	a.m = append(a.m, a.mbs.ContainerMetricsBuilder.Emit(
 		metadata.WithStartTimeOverride(pcommon.NewTimestampFromTime(s.StartTime.Time)),