diff --git a/playbooks/openstack.yml b/playbooks/openstack.yml index 871875e302..309f3ebcfa 100644 --- a/playbooks/openstack.yml +++ b/playbooks/openstack.yml @@ -126,6 +126,11 @@ tags: - neutron + - role: ironic + when: atmosphere_ironic_enabled | default(false) | bool + tags: + - ironic + # NOTE(mnaser): This is disabled out of the box until we have a native way # of configuring it with a pre-configured backend out of the # box. diff --git a/releasenotes/notes/add-ironic-lifecycle-monitoring-6e8c435ea28d2087.yaml b/releasenotes/notes/add-ironic-lifecycle-monitoring-6e8c435ea28d2087.yaml new file mode 100644 index 0000000000..815a3c1bcd --- /dev/null +++ b/releasenotes/notes/add-ironic-lifecycle-monitoring-6e8c435ea28d2087.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Ironic API and node lifecycle metrics can now be enabled in the OpenStack + exporter with ``openstack_exporter_baremetal_enabled``. The collector, + dashboard, recording rules, and alerts require + ``atmosphere_ironic_enabled`` and remain disabled by default. diff --git a/releasenotes/notes/add-opt-in-ironic-openstack-role-8ad2a10d2f06bb02.yaml b/releasenotes/notes/add-opt-in-ironic-openstack-role-8ad2a10d2f06bb02.yaml new file mode 100644 index 0000000000..19aa00440a --- /dev/null +++ b/releasenotes/notes/add-opt-in-ironic-openstack-role-8ad2a10d2f06bb02.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + The standard OpenStack deployment playbook can now deploy the Ironic role + when ``atmosphere_ironic_enabled`` is set to ``true``. The option defaults + to ``false`` so existing deployments remain unchanged. diff --git a/roles/ironic/README.md b/roles/ironic/README.md index 03dfba09bb..c061228f1a 100644 --- a/roles/ironic/README.md +++ b/roles/ironic/README.md @@ -1 +1,14 @@ # `ironic` + +The Ironic role deploys the OpenStack Bare Metal service. It is available from +the standard `playbooks/openstack.yml` workflow but remains disabled by +default. + +Enable it in inventory variables: + +```yaml +atmosphere_ironic_enabled: true +``` + +The playbook condition uses `default(false)`, so inventories that do not define +the variable retain the existing deployment behavior. diff --git a/roles/kube_prometheus_stack/files/dashboards/ironic.json b/roles/kube_prometheus_stack/files/dashboards/ironic.json new file mode 100644 index 0000000000..2d1460b5f5 --- /dev/null +++ b/roles/kube_prometheus_stack/files/dashboards/ironic.json @@ -0,0 +1,253 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "Unavailable" + }, + "1": { + "color": "green", + "text": "Available" + } + }, + "type": "value" + } + ] + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "openstack_ironic_up", + "instant": true, + "refId": "A" + } + ], + "title": "Ironic API", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "openstack:ironic_nodes:count", + "instant": true, + "refId": "A" + } + ], + "title": "Total nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "openstack:ironic_active_nodes:count", + "instant": true, + "refId": "A" + } + ], + "title": "Active nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "openstack:ironic_maintenance_nodes:count", + "instant": true, + "refId": "A" + } + ], + "title": "Maintenance nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 5, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "targets": [ + { + "expr": "openstack_ironic_node", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "title": "Node lifecycle state", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "job": true, + "instance": true + }, + "indexByName": { + "name": 0, + "id": 1, + "provision_state": 2, + "power_state": 3, + "maintenance": 4, + "resource_class": 5, + "console_enabled": 6 + } + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "openstack", + "ironic" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "browser", + "title": "OpenStack Ironic", + "uid": "openstack-ironic", + "version": 1 +} diff --git a/roles/kube_prometheus_stack/files/jsonnet/ironic.libsonnet b/roles/kube_prometheus_stack/files/jsonnet/ironic.libsonnet new file mode 100644 index 0000000000..c062438f79 --- /dev/null +++ b/roles/kube_prometheus_stack/files/jsonnet/ironic.libsonnet @@ -0,0 +1,84 @@ +{ + prometheusRules+:: { + groups: [ + { + name: 'ironic-recording', + rules: [ + { + record: 'openstack:ironic_nodes:count', + expr: 'sum(openstack_ironic_node)', + }, + { + record: 'openstack:ironic_nodes_by_provision_state:count', + expr: 'sum by (provision_state) (openstack_ironic_node)', + }, + { + record: 'openstack:ironic_active_nodes:count', + expr: 'sum(openstack_ironic_node{provision_state="active"})', + }, + { + record: 'openstack:ironic_maintenance_nodes:count', + expr: 'sum(openstack_ironic_node{maintenance="true"})', + }, + ], + }, + ], + }, + prometheusAlerts+:: { + groups: [ + { + name: 'ironic-lifecycle', + rules: [ + { + alert: 'IronicAPIUnavailable', + expr: '(openstack_ironic_up == 0) or absent(openstack_ironic_up)', + 'for': '5m', + labels: { + severity: 'P3', + }, + annotations: { + summary: 'Ironic API metrics are unavailable', + description: 'The Ironic API collector has been unavailable for more than 5 minutes.', + }, + }, + { + alert: 'IronicNodeProvisioningFailed', + expr: 'openstack_ironic_node{provision_state=~"deploy failed|error"} == 1', + 'for': '5m', + labels: { + severity: 'P4', + }, + annotations: { + summary: 'Ironic node provisioning failed', + description: 'Ironic node {{ $labels.name }} ({{ $labels.id }}) is in {{ $labels.provision_state }}.', + }, + }, + { + alert: 'IronicNodeProvisioningStalled', + expr: 'openstack_ironic_node{provision_state=~"deploying|wait call-back|deleting"} == 1', + 'for': '2h', + labels: { + severity: 'P4', + }, + annotations: { + summary: 'Ironic node provisioning is stalled', + description: 'Ironic node {{ $labels.name }} ({{ $labels.id }}) has remained in {{ $labels.provision_state }} for more than 2 hours.', + }, + }, + { + alert: 'IronicActiveNodePowerStateUnexpected', + expr: 'openstack_ironic_node{provision_state="active",power_state!="power on"} == 1', + 'for': '15m', + labels: { + severity: 'P3', + }, + annotations: { + summary: 'Active Ironic node has an unexpected power state', + description: 'Active Ironic node {{ $labels.name }} ({{ $labels.id }}) has reported {{ $labels.power_state }} for more than 15 minutes.', + }, + }, + ], + }, + ], + }, +} diff --git a/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet b/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet index f617ca6e6e..3673d48704 100644 --- a/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet +++ b/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet @@ -104,6 +104,7 @@ local mixins = { }, coredns: (import 'coredns.libsonnet'), geneve: (import 'geneve.libsonnet'), + ironic: (import 'ironic.libsonnet'), kube: (import 'vendor/github.com/kubernetes-monitoring/kubernetes-mixin/mixin.libsonnet') + { _config+:: { kubeApiserverSelector: 'job="apiserver"', diff --git a/roles/kube_prometheus_stack/files/jsonnet/tests.yml b/roles/kube_prometheus_stack/files/jsonnet/tests.yml index dc9804011d..c8418fb7df 100644 --- a/roles/kube_prometheus_stack/files/jsonnet/tests.yml +++ b/roles/kube_prometheus_stack/files/jsonnet/tests.yml @@ -2,6 +2,101 @@ # SPDX-License-Identifier: Apache-2.0 tests: + - interval: 1m + input_series: + - series: 'openstack_ironic_up' + values: '1x10' + alert_rule_test: + - eval_time: 6m + alertname: IronicAPIUnavailable + exp_alerts: [] + + - interval: 1m + alert_rule_test: + - eval_time: 6m + alertname: IronicAPIUnavailable + exp_alerts: + - exp_labels: + severity: P3 + exp_annotations: + summary: "Ironic API metrics are unavailable" + description: "The Ironic API collector has been unavailable for more than 5 minutes." + + - interval: 1m + input_series: + - series: 'openstack_ironic_node{id="node-enroll",name="enrolling",provision_state="enroll fail",power_state="",maintenance="false"}' + values: '1x10' + alert_rule_test: + - eval_time: 6m + alertname: IronicNodeProvisioningFailed + exp_alerts: [] + + - interval: 1m + input_series: + - series: 'openstack_ironic_node{id="node-failed",name="failed-node",provision_state="deploy failed",power_state="power off",maintenance="false"}' + values: '1x10' + alert_rule_test: + - eval_time: 6m + alertname: IronicNodeProvisioningFailed + exp_alerts: + - exp_labels: + id: node-failed + name: failed-node + provision_state: deploy failed + power_state: power off + maintenance: "false" + severity: P4 + exp_annotations: + summary: "Ironic node provisioning failed" + description: "Ironic node failed-node (node-failed) is in deploy failed." + + - interval: 1m + input_series: + - series: 'openstack_ironic_node{id="node-stalled",name="stalled-node",provision_state="wait call-back",power_state="power on",maintenance="false"}' + values: '1x130' + alert_rule_test: + - eval_time: 121m + alertname: IronicNodeProvisioningStalled + exp_alerts: + - exp_labels: + id: node-stalled + name: stalled-node + provision_state: wait call-back + power_state: power on + maintenance: "false" + severity: P4 + exp_annotations: + summary: "Ironic node provisioning is stalled" + description: "Ironic node stalled-node (node-stalled) has remained in wait call-back for more than 2 hours." + + - interval: 1m + input_series: + - series: 'openstack_ironic_node{id="node-active",name="active-node",provision_state="active",power_state="power on",maintenance="false"}' + values: '1x130' + alert_rule_test: + - eval_time: 121m + alertname: IronicNodeProvisioningStalled + exp_alerts: [] + + - interval: 1m + input_series: + - series: 'openstack_ironic_node{id="node-off",name="powered-off-node",provision_state="active",power_state="power off",maintenance="false"}' + values: '1x20' + alert_rule_test: + - eval_time: 16m + alertname: IronicActiveNodePowerStateUnexpected + exp_alerts: + - exp_labels: + id: node-off + name: powered-off-node + provision_state: active + power_state: power off + maintenance: "false" + severity: P3 + exp_annotations: + summary: "Active Ironic node has an unexpected power state" + description: "Active Ironic node powered-off-node (node-off) has reported power off for more than 15 minutes." + # NodeNetworkMulticast - should NOT fire for sustained traffic below the # capacity-planning threshold. - interval: 1m diff --git a/roles/kube_prometheus_stack/tasks/main.yml b/roles/kube_prometheus_stack/tasks/main.yml index 4a28621164..32eb981731 100644 --- a/roles/kube_prometheus_stack/tasks/main.yml +++ b/roles/kube_prometheus_stack/tasks/main.yml @@ -332,6 +332,14 @@ state: present - name: rbd-details state: present + - name: ironic + state: >- + {{ + 'present' + if (atmosphere_ironic_enabled | default(false) | bool + and openstack_exporter_baremetal_enabled | default(false) | bool) + else 'absent' + }} tags: - kube-prometheus-stack-dashboards diff --git a/roles/kube_prometheus_stack/vars/main.yml b/roles/kube_prometheus_stack/vars/main.yml index 9c3559fca4..5100e1a2e1 100644 --- a/roles/kube_prometheus_stack/vars/main.yml +++ b/roles/kube_prometheus_stack/vars/main.yml @@ -1,3 +1,20 @@ +_kube_prometheus_stack_compiled_rules: "{{ lookup('vexxhost.atmosphere.jsonnet', 'jsonnet/rules.jsonnet') }}" +_kube_prometheus_stack_ironic_lifecycle_enabled: >- + {{ + atmosphere_ironic_enabled | default(false) | bool + and openstack_exporter_baremetal_enabled | default(false) | bool + }} +_kube_prometheus_stack_rules: >- + {{ + _kube_prometheus_stack_compiled_rules + if _kube_prometheus_stack_ironic_lifecycle_enabled | bool + else + (_kube_prometheus_stack_compiled_rules + | dict2items + | rejectattr('key', 'equalto', 'ironic') + | items2dict) + }} + _kube_prometheus_stack_helm_values: defaultRules: rules: @@ -763,7 +780,7 @@ _kube_prometheus_stack_helm_values: {{ kube_prometheus_stack_node_exporter_config | to_nice_yaml | indent(4) }} certificate-template.yml: | {{ kube_prometheus_stack_node_exporter_tls_template | to_nice_yaml | indent(4) }} - additionalPrometheusRulesMap: "{{ lookup('vexxhost.atmosphere.jsonnet', 'jsonnet/rules.jsonnet') }}" + additionalPrometheusRulesMap: "{{ _kube_prometheus_stack_rules }}" extraManifests: - apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/roles/openstack_exporter/README.md b/roles/openstack_exporter/README.md index b8f63489b3..27a22d8321 100644 --- a/roles/openstack_exporter/README.md +++ b/roles/openstack_exporter/README.md @@ -1 +1,15 @@ # `openstack_exporter` + +## Ironic lifecycle metrics + +Ironic metrics remain disabled unless both the Ironic service and its exporter +collector are enabled: + +```yaml +atmosphere_ironic_enabled: true +openstack_exporter_baremetal_enabled: true +``` + +Enabling the collector while Ironic is disabled fails validation. When either +option is false, the role keeps `--disable-service.baremetal` in the effective +exporter arguments. diff --git a/roles/openstack_exporter/defaults/main.yml b/roles/openstack_exporter/defaults/main.yml index 21e84e185f..f721dd0c9a 100644 --- a/roles/openstack_exporter/defaults/main.yml +++ b/roles/openstack_exporter/defaults/main.yml @@ -13,6 +13,8 @@ # under the License. # Base arguments for the OpenStack exporter. +openstack_exporter_baremetal_enabled: false + openstack_exporter_args: - --endpoint-type - internal diff --git a/roles/openstack_exporter/tasks/main.yml b/roles/openstack_exporter/tasks/main.yml index f039055be7..a33bf7953a 100644 --- a/roles/openstack_exporter/tasks/main.yml +++ b/roles/openstack_exporter/tasks/main.yml @@ -12,6 +12,30 @@ # License for the specific language governing permissions and limitations # under the License. +- name: Validate Ironic lifecycle monitoring configuration + ansible.builtin.assert: + that: + - atmosphere_ironic_enabled | default(false) | bool + fail_msg: >- + openstack_exporter_baremetal_enabled requires + atmosphere_ironic_enabled=true. + when: openstack_exporter_baremetal_enabled | bool + +- name: Build effective OpenStack exporter arguments + ansible.builtin.set_fact: + _openstack_exporter_effective_args: >- + {{ + (openstack_exporter_args + | reject('equalto', '--disable-service.baremetal') + | list) + if (atmosphere_ironic_enabled | default(false) | bool + and openstack_exporter_baremetal_enabled | bool) + else + (openstack_exporter_args + if '--disable-service.baremetal' in openstack_exporter_args + else openstack_exporter_args + ['--disable-service.baremetal']) + }} + - name: Deploy service kubernetes.core.k8s: state: present @@ -67,7 +91,7 @@ - name: openstack-exporter image: "{{ atmosphere_images['prometheus_openstack_exporter'] | vexxhost.kubernetes.docker_image('ref') }}" args: >- - {{ openstack_exporter_args + + {{ _openstack_exporter_effective_args + (openstack_exporter_neutron_disabled_metrics if atmosphere_network_backend == 'ovn' else ['--disable-service.network']) }} diff --git a/roles/openstack_helm_endpoints/tasks/main.yml b/roles/openstack_helm_endpoints/tasks/main.yml index 9170fd0e8a..0264d4c1e6 100644 --- a/roles/openstack_helm_endpoints/tasks/main.yml +++ b/roles/openstack_helm_endpoints/tasks/main.yml @@ -76,7 +76,7 @@ ansible.builtin.set_fact: openstack_helm_endpoints: | {{ openstack_helm_endpoints | combine(lookup('vars', '_openstack_helm_endpoints_' + service), recursive=True) }} - loop: "{{ openstack_helm_endpoints_list }}" + loop: "{{ openstack_helm_endpoints_list | default([], true) }}" loop_control: loop_var: service diff --git a/tests/unit/test_ironic_playbook.py b/tests/unit/test_ironic_playbook.py new file mode 100644 index 0000000000..9d6d0a76e4 --- /dev/null +++ b/tests/unit/test_ironic_playbook.py @@ -0,0 +1,47 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import yaml + + +def test_ironic_role_is_opt_in_and_follows_neutron(): + repository_root = Path(__file__).parents[2] + plays = yaml.safe_load( + (repository_root / "playbooks" / "openstack.yml").read_text() + ) + + openstack_play = next( + play + for play in plays + if play.get("hosts") == "controllers[0]" + and any(role.get("role") == "neutron" for role in play.get("roles", [])) + ) + roles = openstack_play["roles"] + role_names = [role["role"] for role in roles] + ironic = roles[role_names.index("ironic")] + + assert role_names.index("ironic") > role_names.index("neutron") + assert ironic["when"] == ("atmosphere_ironic_enabled | default(false) | bool") + assert ironic["tags"] == ["ironic"] + + +def test_disabled_ironic_role_cannot_expand_an_empty_endpoint_loop(): + repository_root = Path(__file__).parents[2] + tasks = yaml.safe_load( + ( + repository_root + / "roles" + / "openstack_helm_endpoints" + / "tasks" + / "main.yml" + ).read_text() + ) + task = next( + task + for task in tasks + if task.get("name") == "Generate OpenStack-Helm endpoints" + ) + + assert task["loop"] == ("{{ openstack_helm_endpoints_list | default([], true) }}")