feat(monitoring): expose Ironic lifecycle metrics

playbooks/openstack.yml

@@ -126,6 +126,11 @@
       tags:
         - neutron
 
+    - role: ironic
+      when: atmosphere_ironic_enabled | default(false) | bool
+      tags:
+        - ironic
+
     # NOTE(mnaser): This is disabled out of the box until we have a native way
     #               of configuring it with a pre-configured backend out of the
     #               box.

releasenotes/notes/add-ironic-lifecycle-monitoring-6e8c435ea28d2087.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Ironic API and node lifecycle metrics can now be enabled in the OpenStack
+    exporter with ``openstack_exporter_baremetal_enabled``. The collector,
+    dashboard, recording rules, and alerts require
+    ``atmosphere_ironic_enabled`` and remain disabled by default.

releasenotes/notes/add-opt-in-ironic-openstack-role-8ad2a10d2f06bb02.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    The standard OpenStack deployment playbook can now deploy the Ironic role
+    when ``atmosphere_ironic_enabled`` is set to ``true``. The option defaults
+    to ``false`` so existing deployments remain unchanged.

roles/ironic/README.md

@@ -1 +1,14 @@
 # `ironic`
+
+The Ironic role deploys the OpenStack Bare Metal service. It is available from
+the standard `playbooks/openstack.yml` workflow but remains disabled by
+default.
+
+Enable it in inventory variables:
+
+```yaml
+atmosphere_ironic_enabled: true
+```
+
+The playbook condition uses `default(false)`, so inventories that do not define
+the variable retain the existing deployment behavior.

roles/kube_prometheus_stack/files/dashboards/ironic.json

@@ -0,0 +1,253 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "red",
+                  "text": "Unavailable"
+                },
+                "1": {
+                  "color": "green",
+                  "text": "Available"
+                }
+              },
+              "type": "value"
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "openstack_ironic_up",
+          "instant": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Ironic API",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "openstack:ironic_nodes:count",
+          "instant": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Total nodes",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "openstack:ironic_active_nodes:count",
+          "instant": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Active nodes",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "openstack:ironic_maintenance_nodes:count",
+          "instant": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Maintenance nodes",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 5
+      },
+      "id": 5,
+      "options": {
+        "cellHeight": "sm",
+        "showHeader": true
+      },
+      "targets": [
+        {
+          "expr": "openstack_ironic_node",
+          "format": "table",
+          "instant": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Node lifecycle state",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "job": true,
+              "instance": true
+            },
+            "indexByName": {
+              "name": 0,
+              "id": 1,
+              "provision_state": 2,
+              "power_state": 3,
+              "maintenance": 4,
+              "resource_class": 5,
+              "console_enabled": 6
+            }
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 39,
+  "tags": [
+    "openstack",
+    "ironic"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timezone": "browser",
+  "title": "OpenStack Ironic",
+  "uid": "openstack-ironic",
+  "version": 1
+}

roles/kube_prometheus_stack/files/jsonnet/ironic.libsonnet

@@ -0,0 +1,84 @@
+{
+  prometheusRules+:: {
+    groups: [
+      {
+        name: 'ironic-recording',
+        rules: [
+          {
+            record: 'openstack:ironic_nodes:count',
+            expr: 'sum(openstack_ironic_node)',
+          },
+          {
+            record: 'openstack:ironic_nodes_by_provision_state:count',
+            expr: 'sum by (provision_state) (openstack_ironic_node)',
+          },
+          {
+            record: 'openstack:ironic_active_nodes:count',
+            expr: 'sum(openstack_ironic_node{provision_state="active"})',
+          },
+          {
+            record: 'openstack:ironic_maintenance_nodes:count',
+            expr: 'sum(openstack_ironic_node{maintenance="true"})',
+          },
+        ],
+      },
+    ],
+  },
+  prometheusAlerts+:: {
+    groups: [
+      {
+        name: 'ironic-lifecycle',
+        rules: [
+          {
+            alert: 'IronicAPIUnavailable',
+            expr: '(openstack_ironic_up == 0) or absent(openstack_ironic_up)',
+            'for': '5m',
+            labels: {
+              severity: 'P3',
+            },
+            annotations: {
+              summary: 'Ironic API metrics are unavailable',
+              description: 'The Ironic API collector has been unavailable for more than 5 minutes.',
+            },
+          },
+          {
+            alert: 'IronicNodeProvisioningFailed',
+            expr: 'openstack_ironic_node{provision_state=~"deploy failed|error"} == 1',
+            'for': '5m',
+            labels: {
+              severity: 'P4',
+            },
+            annotations: {
+              summary: 'Ironic node provisioning failed',
+              description: 'Ironic node {{ $labels.name }} ({{ $labels.id }}) is in {{ $labels.provision_state }}.',
+            },
+          },
+          {
+            alert: 'IronicNodeProvisioningStalled',
+            expr: 'openstack_ironic_node{provision_state=~"deploying|wait call-back|deleting"} == 1',
+            'for': '2h',
+            labels: {
+              severity: 'P4',
+            },
+            annotations: {
+              summary: 'Ironic node provisioning is stalled',
+              description: 'Ironic node {{ $labels.name }} ({{ $labels.id }}) has remained in {{ $labels.provision_state }} for more than 2 hours.',
+            },
+          },
+          {
+            alert: 'IronicActiveNodePowerStateUnexpected',
+            expr: 'openstack_ironic_node{provision_state="active",power_state!="power on"} == 1',
+            'for': '15m',
+            labels: {
+              severity: 'P3',
+            },
+            annotations: {
+              summary: 'Active Ironic node has an unexpected power state',
+              description: 'Active Ironic node {{ $labels.name }} ({{ $labels.id }}) has reported {{ $labels.power_state }} for more than 15 minutes.',
+            },
+          },
+        ],
+      },
+    ],
+  },
+}

roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet

@@ -104,6 +104,7 @@ local mixins = {
   },
   coredns: (import 'coredns.libsonnet'),
   geneve: (import 'geneve.libsonnet'),
+  ironic: (import 'ironic.libsonnet'),
   kube: (import 'vendor/github.com/kubernetes-monitoring/kubernetes-mixin/mixin.libsonnet') + {
     _config+:: {
       kubeApiserverSelector: 'job="apiserver"',