From c634fa6fb4bc043faab58324bf28f01047d65d7d Mon Sep 17 00:00:00 2001 From: Sagi Zigmon Date: Sun, 9 Aug 2026 18:54:28 +0300 Subject: [PATCH] feat: add DPF network validation checks for DPU deployments Add two new rules for OpenShift clusters with NVIDIA BlueField DPUs: - DpuBondLacpHealth: verifies LACP bond health on DPU ports - OvnGeneveTunnelLocalIp: verifies Geneve tunnel local_ip matches node IP Includes BondBase extraction, dpf profile, and full test coverage. Assisted-by: Claude Code (Claude Opus 4.6) --- .../domains/network_domain.py | 5 +- .../rules/network/bond_base.py | 29 ++ .../rules/network/dpf_validations.py | 274 ++++++++++++ .../network/node_connectivity_validations.py | 18 +- src/profiles/profiles.yaml | 2 + tests/domains/test_network_domain.py | 8 +- tests/rules/network/test_dpf_validations.py | 403 ++++++++++++++++++ 7 files changed, 722 insertions(+), 17 deletions(-) create mode 100644 src/in_cluster_checks/rules/network/bond_base.py create mode 100644 src/in_cluster_checks/rules/network/dpf_validations.py create mode 100644 tests/rules/network/test_dpf_validations.py diff --git a/src/in_cluster_checks/domains/network_domain.py b/src/in_cluster_checks/domains/network_domain.py index ad592ce..4cffe9f 100644 --- a/src/in_cluster_checks/domains/network_domain.py +++ b/src/in_cluster_checks/domains/network_domain.py @@ -9,6 +9,7 @@ from in_cluster_checks.core.domain import RuleDomain from in_cluster_checks.rules.network.dns_validations import VerifyDnsReachability +from in_cluster_checks.rules.network.dpf_validations import DpuBondLacpHealth, OvnGeneveTunnelLocalIp from in_cluster_checks.rules.network.nmstate_validations import VerifyAllNNCPsAvailable from in_cluster_checks.rules.network.node_connectivity_validations import ( AreAllNodesConnected, @@ -40,7 +41,7 @@ class NetworkValidationDomain(RuleDomain): """ Network rule domain for OpenShift. - Validates network health including OVS, OVN-Kubernetes, Whereabouts, and node connectivity. + Validates network health including OVS, OVN-Kubernetes, DPF, Whereabouts, and node connectivity. """ def domain_name(self) -> str: @@ -68,6 +69,8 @@ def get_rule_classes(self) -> List[type]: NodesHaveOvnkubeNodePod, LogicalSwitchNodeValidator, MTUOverlayInterfaces, + DpuBondLacpHealth, + OvnGeneveTunnelLocalIp, VerifyAllNNCPsAvailable, WhereaboutsDuplicateIPAddresses, WhereaboutsMissingPodrefs, diff --git a/src/in_cluster_checks/rules/network/bond_base.py b/src/in_cluster_checks/rules/network/bond_base.py new file mode 100644 index 0000000..97d15af --- /dev/null +++ b/src/in_cluster_checks/rules/network/bond_base.py @@ -0,0 +1,29 @@ +""" +Base class for network bond validation rules. + +Provides common functionality for rules that validate bonded network interfaces. +""" + +from in_cluster_checks.core.rule import PrerequisiteResult, Rule + + +class BondBase(Rule): + """ + Base class for bond-related validation rules. + + Provides common prerequisite checking for rules that require + bond interfaces to be configured on the node. + """ + + BONDING_PATH = "/proc/net/bonding" + + def is_prerequisite_fulfilled(self) -> PrerequisiteResult: + """ + Check if bond interfaces exist on this node. + + Returns: + PrerequisiteResult indicating if bonding is configured + """ + if self.file_utils.is_dir_exist(self.BONDING_PATH): + return PrerequisiteResult.met() + return PrerequisiteResult.not_met("No bond interfaces configured") diff --git a/src/in_cluster_checks/rules/network/dpf_validations.py b/src/in_cluster_checks/rules/network/dpf_validations.py new file mode 100644 index 0000000..9145f8c --- /dev/null +++ b/src/in_cluster_checks/rules/network/dpf_validations.py @@ -0,0 +1,274 @@ +""" +DPF (DPU Platform Framework) validation checks for OpenShift clusters +with NVIDIA BlueField DPUs. +""" + +import re +from typing import Dict, List, Optional + +from in_cluster_checks.core.exceptions import UnExpectedSystemOutput +from in_cluster_checks.core.rule import PrerequisiteResult, Rule, RuleResult +from in_cluster_checks.rules.network.bond_base import BondBase +from in_cluster_checks.utils.enums import Objectives +from in_cluster_checks.utils.safe_cmd_string import SafeCmdString + + +class DpuBondLacpHealth(BondBase): + """Verify LACP bond health on DPU ports. + + Checks that bond interfaces using 802.3ad (LACP) mode have all slave + interfaces UP, are in the same aggregator, and are not in churned state. + A degraded bond means traffic flows through a single port, reducing + available bandwidth without any Kubernetes-visible failure. + """ + + objective_hosts = [Objectives.ALL_NODES] + supported_profiles = {"dpf"} + unique_name = "dpu_bond_lacp_health" + title = "Verify LACP bond health on DPU ports" + links = ["https://github.com/RedHatInsights/incluster-checks/wiki/" "DPF---LACP-bond-health-on-DPU-ports"] + + def _parse_bond_info(self, bond_name: str) -> Dict: + """Parse /proc/net/bonding/ for LACP-specific fields. + + Args: + bond_name: Name of the bond interface. + + Returns: + Dictionary with bond mode, MII status, and per-slave details. + """ + bond_file = f"/proc/net/bonding/{bond_name}" + lines = self.file_utils.get_lines_in_file(bond_file) + if lines is None: + return {"error": f"Cannot read bond {bond_name}"} + + content = "\n".join(lines) + sections = [s.strip() for s in content.split("\n\n") if s.strip()] + + info: Dict = {"mode": "", "mii_status": "", "slaves": []} + + for section in sections: + section_lines = [line.strip() for line in section.split("\n") if line.strip()] + + if any(line.startswith("Slave Interface:") for line in section_lines): + slave = self._parse_slave_section(section_lines) + if slave: + info["slaves"].append(slave) + else: + for line in section_lines: + if line.startswith("Bonding Mode:"): + info["mode"] = line.split(":", 1)[1].strip() + elif line.startswith("MII Status:"): + info["mii_status"] = line.split(":", 1)[1].strip() + + return info + + def _parse_slave_section(self, section_lines: List[str]) -> Optional[Dict]: + """Parse a slave interface section. + + Args: + section_lines: Lines from a slave section. + + Returns: + Dictionary with slave details, or None if parsing fails. + """ + slave: Dict = { + "name": "", + "mii_status": "", + "speed": "", + "aggregator_id": "", + "actor_churn": "", + "partner_churn": "", + } + + for line in section_lines: + if line.startswith("Slave Interface:"): + slave["name"] = line.split(":", 1)[1].strip() + elif line.startswith("MII Status:"): + slave["mii_status"] = line.split(":", 1)[1].strip() + elif line.startswith("Speed:"): + slave["speed"] = line.split(":", 1)[1].strip() + elif line.startswith("Aggregator ID:"): + slave["aggregator_id"] = line.split(":", 1)[1].strip() + elif line.startswith("Actor Churn State:"): + slave["actor_churn"] = line.split(":", 1)[1].strip() + elif line.startswith("Partner Churn State:"): + slave["partner_churn"] = line.split(":", 1)[1].strip() + + return slave if slave["name"] else None + + def run_rule(self) -> RuleResult: + """Check LACP bond health on all bond interfaces.""" + bond_names_list = self.file_utils.list_files(self.BONDING_PATH) + if bond_names_list is None: + raise UnExpectedSystemOutput( + ip=self.get_host_ip(), + cmd=f"ls {self.BONDING_PATH}", + output="Failed to list bond interfaces", + ) + + bond_names = bond_names_list + all_issues: List[str] = [] + all_passed: List[str] = [] + + for bond_name in bond_names: + info = self._parse_bond_info(bond_name) + + if "error" in info: + all_issues.append(f"{bond_name}: {info['error']}") + continue + + if "802.3ad" not in info["mode"]: + continue + + if info["mii_status"] != "up": + all_issues.append(f"{bond_name}: bond MII status is {info['mii_status']}") + continue + + if len(info["slaves"]) < 2: + all_issues.append(f"{bond_name}: only {len(info['slaves'])} slave(s), " "expected 2+ for LACP") + continue + + down_slaves = [s for s in info["slaves"] if s["mii_status"] != "up"] + if down_slaves: + names = ", ".join(s["name"] for s in down_slaves) + all_issues.append(f"{bond_name}: slave(s) down: {names}") + + missing_agg = [s["name"] for s in info["slaves"] if not s["aggregator_id"]] + if missing_agg: + all_issues.append( + f"{bond_name}: missing aggregator_id on slave(s): " f"{', '.join(missing_agg)}, LACP not negotiated" + ) + else: + agg_ids = set(s["aggregator_id"] for s in info["slaves"]) + if len(agg_ids) > 1: + details = ", ".join(f"{s['name']}=agg{s['aggregator_id']}" for s in info["slaves"]) + all_issues.append( + f"{bond_name}: slaves in different aggregators " f"({details}), LACP not fully negotiated" + ) + + churned: List[str] = [] + for s in info["slaves"]: + if s["actor_churn"] and s["actor_churn"] != "none": + churned.append(f"{s['name']} actor={s['actor_churn']}") + if s["partner_churn"] and s["partner_churn"] != "none": + churned.append(f"{s['name']} partner={s['partner_churn']}") + if churned: + all_issues.append(f"{bond_name}: LACP churn detected: {', '.join(churned)}") + + if not down_slaves and not missing_agg and not churned: + agg_ids = set(s["aggregator_id"] for s in info["slaves"]) + if len(agg_ids) == 1: + slave_info = ", ".join(f"{s['name']} ({s['speed']})" for s in info["slaves"]) + all_passed.append(f"{bond_name}: LACP healthy, " f"{len(info['slaves'])} slaves UP ({slave_info})") + + if not all_issues and not all_passed: + return RuleResult.skip("No LACP (802.3ad) bonds found") + + if all_issues: + msg = "LACP bond issues detected:\n" + "\n".join(f" - {i}" for i in all_issues) + if all_passed: + msg += "\nHealthy bonds:\n" + "\n".join(f" - {p}" for p in all_passed) + return RuleResult.failed(msg) + + return RuleResult.passed("All LACP bonds healthy:\n" + "\n".join(f" - {p}" for p in all_passed)) + + +class OvnGeneveTunnelLocalIp(Rule): + """Verify OVN Geneve tunnel local_ip matches the node's Kubernetes InternalIP. + + OVN programs Geneve tunnels between nodes using each node's IP as the + local_ip in the tunnel options. If the node's IP changes (e.g. + during + interface migration) without a corresponding OVN restart, the tunnels + retain the old local_ip, causing silent inter-node connectivity loss + while the node remains Ready. + """ + + objective_hosts = [Objectives.ALL_NODES] + supported_profiles = {"dpf"} + unique_name = "ovn_geneve_tunnel_local_ip" + title = "Verify OVN Geneve tunnel local_ip matches node InternalIP" + links = ["https://github.com/RedHatInsights/incluster-checks/wiki/" "DPF---OVN-Geneve-tunnel-local_ip"] + + def _get_ovs_show(self) -> Optional[str]: + """Run ovs-vsctl show and return output, or None on failure.""" + try: + return self.get_output_from_run_cmd(SafeCmdString("ovs-vsctl show")) + except Exception: + return None + + def _extract_geneve_local_ips(self, ovs_output: str) -> List[str]: + """Extract local_ip values only from Geneve tunnel interface blocks. + + Args: + ovs_output: Output from ovs-vsctl show. + + Returns: + Unique list of local_ip values found in Geneve interface blocks. + """ + local_ips = [] + in_geneve_block = False + for line in ovs_output.splitlines(): + stripped = line.strip() + if stripped == "type: geneve": + in_geneve_block = True + elif stripped.startswith("type:"): + in_geneve_block = False + elif in_geneve_block and "local_ip=" in stripped: + match = re.search(r'local_ip="([^"]+)"', stripped) + if match and match.group(1) not in local_ips: + local_ips.append(match.group(1)) + in_geneve_block = False + return local_ips + + def is_prerequisite_fulfilled(self) -> PrerequisiteResult: + """Check if OVS has any Geneve tunnel interfaces.""" + ovs_output = self._get_ovs_show() + if ovs_output is None: + return PrerequisiteResult.not_met("Cannot access OVS") + if "geneve" not in ovs_output: + return PrerequisiteResult.not_met("No Geneve tunnels configured") + return PrerequisiteResult.met() + + def _get_node_ip(self) -> Optional[str]: + """Get this node's primary IP from the nodeip-configuration file.""" + try: + content = self.file_utils.read_file("/run/nodeip-configuration/primary-ip") + return content.strip() if content.strip() else None + except UnExpectedSystemOutput: + return None + + def run_rule(self) -> RuleResult: + """Check that Geneve tunnel local_ip matches node InternalIP.""" + ovs_output = self._get_ovs_show() + if ovs_output is None: + raise UnExpectedSystemOutput( + ip=self.get_host_ip(), + cmd="ovs-vsctl show", + output="Failed to run ovs-vsctl show", + ) + + local_ips = list(set(self._extract_geneve_local_ips(ovs_output))) + if not local_ips: + return RuleResult.skip("No Geneve local_ip found in OVS output") + + node_ip = self._get_node_ip() + if not node_ip: + return RuleResult.skip("Cannot determine node primary IP") + + if len(local_ips) > 1: + return RuleResult.failed( + f"Multiple different local_ip values in Geneve tunnels: " + f"{sorted(local_ips)}. OVS configuration may be inconsistent." + ) + + geneve_local_ip = local_ips[0] + if geneve_local_ip != node_ip: + return RuleResult.failed( + f"Geneve tunnel local_ip ({geneve_local_ip}) does not match " + f"node primary IP ({node_ip}). " + f"Inter-node pod connectivity may be broken." + ) + + return RuleResult.passed(f"Geneve tunnel local_ip ({geneve_local_ip}) matches node primary IP") diff --git a/src/in_cluster_checks/rules/network/node_connectivity_validations.py b/src/in_cluster_checks/rules/network/node_connectivity_validations.py index 327a46b..e871faa 100644 --- a/src/in_cluster_checks/rules/network/node_connectivity_validations.py +++ b/src/in_cluster_checks/rules/network/node_connectivity_validations.py @@ -9,7 +9,8 @@ import re from in_cluster_checks.core.operations import DataCollector -from in_cluster_checks.core.rule import OrchestratorRule, PrerequisiteResult, Rule, RuleResult +from in_cluster_checks.core.rule import OrchestratorRule, RuleResult +from in_cluster_checks.rules.network.bond_base import BondBase from in_cluster_checks.rules.network.ovs_base import OvsOperatorBase from in_cluster_checks.utils.enums import Objectives from in_cluster_checks.utils.safe_cmd_string import SafeCmdString @@ -51,7 +52,7 @@ def run_rule(self) -> RuleResult: return RuleResult.passed(f"All {len(self._node_executors)} nodes are connected") -class VerifyBondedInterfacesUp(Rule): +class VerifyBondedInterfacesUp(BondBase): """ Check if bonded network interfaces are up. @@ -64,19 +65,6 @@ class VerifyBondedInterfacesUp(Rule): unique_name = "check_if_bonded_interfaces_are_up" title = "Check if bonded interfaces are up" - BONDING_PATH = "/proc/net/bonding" - - def is_prerequisite_fulfilled(self) -> PrerequisiteResult: - """ - Check if bonding directory exists. - - Returns: - PrerequisiteResult indicating if bonding is configured - """ - if self.file_utils.is_dir_exist(self.BONDING_PATH): - return PrerequisiteResult.met() - return PrerequisiteResult.not_met("Bonding directory does not exist - no bonded interfaces configured") - def run_rule(self) -> RuleResult: """ Verify all bonded interfaces are up. diff --git a/src/profiles/profiles.yaml b/src/profiles/profiles.yaml index 4258b39..4b815ab 100644 --- a/src/profiles/profiles.yaml +++ b/src/profiles/profiles.yaml @@ -15,6 +15,8 @@ profiles: # --- Layer 3: Platforms & Hardware --- gpu: include: [ai-base] # GPU implies AI-base rules + dpf: + include: [general] # DPF (DPU Platform Framework) deployments with NVIDIA BlueField-3 DPUs spectrum-x: description: "NVIDIA Spectrum-X networking platform for AI factories" include: [general] diff --git a/tests/domains/test_network_domain.py b/tests/domains/test_network_domain.py index c684782..6ddd9ce 100644 --- a/tests/domains/test_network_domain.py +++ b/tests/domains/test_network_domain.py @@ -8,6 +8,10 @@ from in_cluster_checks.domains.network_domain import NetworkValidationDomain from in_cluster_checks.rules.network.dns_validations import VerifyDnsReachability +from in_cluster_checks.rules.network.dpf_validations import ( + DpuBondLacpHealth, + OvnGeneveTunnelLocalIp, +) from in_cluster_checks.rules.network.nmstate_validations import VerifyAllNNCPsAvailable from in_cluster_checks.rules.network.node_connectivity_validations import ( AreAllNodesConnected, @@ -69,7 +73,7 @@ def test_get_rule_classes(self): rules = domain.get_rule_classes() assert isinstance(rules, list) - assert len(rules) == 18 + assert len(rules) == 20 assert OvsInterfaceAndPortFound in rules assert OvsPhysicalPortHealthCheck in rules assert OvsBridgeInterfaceHealthCheck in rules @@ -87,6 +91,8 @@ def test_get_rule_classes(self): assert WhereaboutsMissingPodrefs in rules assert WhereaboutsMissingAllocations in rules assert WhereaboutsExistingAllocations in rules + assert DpuBondLacpHealth in rules + assert OvnGeneveTunnelLocalIp in rules assert VerifyDnsReachability in rules def test_verify_runs_validators(self): diff --git a/tests/rules/network/test_dpf_validations.py b/tests/rules/network/test_dpf_validations.py new file mode 100644 index 0000000..ce15319 --- /dev/null +++ b/tests/rules/network/test_dpf_validations.py @@ -0,0 +1,403 @@ +"""Unit tests for DPF validation checks.""" + +import pytest + +from in_cluster_checks.rules.network.dpf_validations import DpuBondLacpHealth, OvnGeneveTunnelLocalIp +from tests.pytest_tools.test_operator_base import CmdOutput +from tests.pytest_tools.test_rule_base import RuleScenarioParams, RuleTestBase + + +BOND_LACP_HEALTHY = """Ethernet Channel Bonding Driver: v5.14.0-570.64.1.el9_6.x86_64 + +Bonding Mode: IEEE 802.3ad Dynamic link aggregation +Transmit Hash Policy: layer3+4 (1) +MII Status: up +MII Polling Interval (ms): 100 +Up Delay (ms): 0 +Down Delay (ms): 0 + +802.3ad info +LACP active: on +LACP rate: slow +Min links: 0 +Aggregator selection policy (ad_select): stable + +Slave Interface: ens7f0np0 +MII Status: up +Speed: 200000 Mbps +Duplex: full +Link Failure Count: 0 +Permanent HW addr: c4:70:bd:c2:c1:68 +Slave queue ID: 0 +Aggregator ID: 1 +Actor Churn State: none +Partner Churn State: none + +Slave Interface: ens7f1np1 +MII Status: up +Speed: 200000 Mbps +Duplex: full +Link Failure Count: 0 +Permanent HW addr: c4:70:bd:c2:c1:69 +Slave queue ID: 0 +Aggregator ID: 1 +Actor Churn State: none +Partner Churn State: none +""" + +BOND_LACP_SLAVE_DOWN = """Ethernet Channel Bonding Driver: v5.14.0-570.64.1.el9_6.x86_64 + +Bonding Mode: IEEE 802.3ad Dynamic link aggregation +Transmit Hash Policy: layer3+4 (1) +MII Status: up +MII Polling Interval (ms): 100 + +802.3ad info +LACP active: on + +Slave Interface: ens7f0np0 +MII Status: up +Speed: 200000 Mbps +Duplex: full +Link Failure Count: 0 +Aggregator ID: 1 +Actor Churn State: none +Partner Churn State: none + +Slave Interface: ens7f1np1 +MII Status: down +Speed: Unknown +Duplex: Unknown +Link Failure Count: 1 +Aggregator ID: 2 +Actor Churn State: churned +Partner Churn State: churned +""" + +BOND_LACP_DIFFERENT_AGGREGATORS = """Ethernet Channel Bonding Driver: v5.14.0-570.64.1.el9_6.x86_64 + +Bonding Mode: IEEE 802.3ad Dynamic link aggregation +Transmit Hash Policy: layer3+4 (1) +MII Status: up + +802.3ad info +LACP active: on + +Slave Interface: ens7f0np0 +MII Status: up +Speed: 200000 Mbps +Duplex: full +Aggregator ID: 1 +Actor Churn State: none +Partner Churn State: churned + +Slave Interface: ens7f1np1 +MII Status: up +Speed: 200000 Mbps +Duplex: full +Aggregator ID: 2 +Actor Churn State: none +Partner Churn State: churned +""" + +BOND_LACP_SINGLE_SLAVE = """Ethernet Channel Bonding Driver: v5.14.0-570.64.1.el9_6.x86_64 + +Bonding Mode: IEEE 802.3ad Dynamic link aggregation +Transmit Hash Policy: layer3+4 (1) +MII Status: up + +802.3ad info +LACP active: on + +Slave Interface: ens7f0np0 +MII Status: up +Speed: 200000 Mbps +Duplex: full +Aggregator ID: 1 +Actor Churn State: none +Partner Churn State: none +""" + +BOND_ACTIVE_BACKUP = """Ethernet Channel Bonding Driver: v5.14.0-570.64.1.el9_6.x86_64 + +Bonding Mode: fault-tolerance (active-backup) +Primary Slave: None +Currently Active Slave: ens7f0np0 +MII Status: up + +Slave Interface: ens7f0np0 +MII Status: up +Speed: 200000 Mbps + +Slave Interface: ens7f1np1 +MII Status: up +Speed: 200000 Mbps +""" + + +class TestDpuBondLacpHealth(RuleTestBase): + """Tests for DpuBondLacpHealth validator.""" + + tested_type = DpuBondLacpHealth + + scenario_prerequisite_not_fulfilled = [ + RuleScenarioParams( + "no_bonding_directory", + cmd_input_output_dict={ + "test -d /proc/net/bonding": CmdOutput("", return_code=1), + }, + ) + ] + + scenario_prerequisite_fulfilled = [ + RuleScenarioParams( + "bonding_directory_exists", + cmd_input_output_dict={ + "test -d /proc/net/bonding": CmdOutput(""), + }, + ) + ] + + scenario_passed = [ + RuleScenarioParams( + scenario_title="lacp_bond_healthy_two_slaves", + cmd_input_output_dict={ + "ls /proc/net/bonding": CmdOutput("bond0"), + "cat /proc/net/bonding/bond0": CmdOutput(BOND_LACP_HEALTHY), + }, + ), + ] + + scenario_failed = [ + RuleScenarioParams( + scenario_title="lacp_slave_down", + cmd_input_output_dict={ + "ls /proc/net/bonding": CmdOutput("bond0"), + "cat /proc/net/bonding/bond0": CmdOutput(BOND_LACP_SLAVE_DOWN), + }, + failed_msg="LACP bond issues detected:\n - bond0: slave(s) down: ens7f1np1\n - bond0: slaves in different aggregators (ens7f0np0=agg1, ens7f1np1=agg2), LACP not fully negotiated\n - bond0: LACP churn detected: ens7f1np1 actor=churned, ens7f1np1 partner=churned", + ), + RuleScenarioParams( + scenario_title="lacp_different_aggregators", + cmd_input_output_dict={ + "ls /proc/net/bonding": CmdOutput("bond0"), + "cat /proc/net/bonding/bond0": CmdOutput(BOND_LACP_DIFFERENT_AGGREGATORS), + }, + failed_msg="LACP bond issues detected:\n - bond0: slaves in different aggregators (ens7f0np0=agg1, ens7f1np1=agg2), LACP not fully negotiated\n - bond0: LACP churn detected: ens7f0np0 partner=churned, ens7f1np1 partner=churned", + ), + RuleScenarioParams( + scenario_title="lacp_single_slave", + cmd_input_output_dict={ + "ls /proc/net/bonding": CmdOutput("bond0"), + "cat /proc/net/bonding/bond0": CmdOutput(BOND_LACP_SINGLE_SLAVE), + }, + failed_msg="LACP bond issues detected:\n - bond0: only 1 slave(s), expected 2+ for LACP", + ), + ] + + scenario_not_applicable = [] + + scenario_skip = [ + RuleScenarioParams( + scenario_title="non_lacp_bond_only_active_backup", + cmd_input_output_dict={ + "ls /proc/net/bonding": CmdOutput("bond0"), + "cat /proc/net/bonding/bond0": CmdOutput(BOND_ACTIVE_BACKUP), + }, + ), + ] + + @pytest.mark.parametrize("scenario_params", scenario_skip) + def test_scenario_skip(self, scenario_params, tested_object): + """Test that non-LACP bonds result in SKIP.""" + from in_cluster_checks.utils.enums import Status + + self._init_validation_object(tested_object, scenario_params) + with self._apply_patches(scenario_params, tested_object): + result = tested_object.run_rule() + assert result.status == Status.SKIP, ( + f"Expected SKIP for scenario: {scenario_params.scenario_title}, got {result.status}" + ) + + @pytest.mark.parametrize("scenario_params", scenario_prerequisite_not_fulfilled) + def test_prerequisite_not_fulfilled(self, scenario_params, tested_object): + """Test that prerequisite is not fulfilled when no bond interfaces exist.""" + RuleTestBase.test_prerequisite_not_fulfilled(self, scenario_params, tested_object) + + @pytest.mark.parametrize("scenario_params", scenario_prerequisite_fulfilled) + def test_prerequisite_fulfilled(self, scenario_params, tested_object): + """Test that prerequisite is fulfilled when bond interfaces exist.""" + RuleTestBase.test_prerequisite_fulfilled(self, scenario_params, tested_object) + + @pytest.mark.parametrize("scenario_params", scenario_passed) + def test_scenario_passed(self, scenario_params, tested_object): + """Test that healthy LACP bonds pass.""" + RuleTestBase.test_scenario_passed(self, scenario_params, tested_object) + + @pytest.mark.parametrize("scenario_params", scenario_failed) + def test_scenario_failed(self, scenario_params, tested_object): + """Test that degraded LACP bonds are detected and reported.""" + RuleTestBase.test_scenario_failed(self, scenario_params, tested_object) + + @pytest.mark.parametrize("scenario_params", scenario_not_applicable) + def test_scenario_not_applicable(self, scenario_params, tested_object): + """Test that non-applicable scenarios are handled correctly.""" + RuleTestBase.test_scenario_not_applicable(self, scenario_params, tested_object) + + +OVS_SHOW_WITH_GENEVE = """Bridge br-int + Port ovn-abc123-0 + Interface ovn-abc123-0 + type: geneve + options: {csum="true", key=flow, local_ip="10.6.135.202", remote_ip="10.6.135.236"} + Port ovn-def456-0 + Interface ovn-def456-0 + type: geneve + options: {csum="true", key=flow, local_ip="10.6.135.202", remote_ip="10.6.135.225"} +""" + +OVS_SHOW_STALE_LOCAL_IP = """Bridge br-int + Port ovn-abc123-0 + Interface ovn-abc123-0 + type: geneve + options: {csum="true", key=flow, local_ip="10.6.135.1", remote_ip="10.6.135.236"} +""" + +OVS_SHOW_MULTI_LOCAL_IP = """Bridge br-int + Port ovn-abc123-0 + Interface ovn-abc123-0 + type: geneve + options: {csum="true", key=flow, local_ip="10.6.135.202", remote_ip="10.6.135.236"} + Port ovn-abc124-0 + Interface ovn-abc124-0 + type: geneve + options: {csum="true", key=flow, local_ip="10.6.135.1", remote_ip="10.6.135.225"} +""" + +OVS_SHOW_NO_GENEVE = """Bridge br-ex + Port br-ex + Interface br-ex + type: internal +""" + + +class TestOvnGeneveTunnelLocalIp(RuleTestBase): + """Tests for OvnGeneveTunnelLocalIp validator.""" + + tested_type = OvnGeneveTunnelLocalIp + + scenario_prerequisite_not_fulfilled = [ + RuleScenarioParams( + "ovs_not_accessible", + cmd_input_output_dict={ + "ovs-vsctl show": CmdOutput("", return_code=1), + }, + ), + RuleScenarioParams( + "no_geneve_tunnels", + cmd_input_output_dict={ + "ovs-vsctl show": CmdOutput(OVS_SHOW_NO_GENEVE), + }, + ), + ] + + scenario_prerequisite_fulfilled = [ + RuleScenarioParams( + "geneve_tunnels_present", + cmd_input_output_dict={ + "ovs-vsctl show": CmdOutput(OVS_SHOW_WITH_GENEVE), + }, + ), + ] + + scenario_passed = [ + RuleScenarioParams( + scenario_title="local_ip_matches_node_ip", + cmd_input_output_dict={ + "ovs-vsctl show": CmdOutput(OVS_SHOW_WITH_GENEVE), + "cat /run/nodeip-configuration/primary-ip": CmdOutput("10.6.135.202"), + }, + ), + ] + + scenario_failed = [ + RuleScenarioParams( + scenario_title="multiple_local_ips_inconsistent", + cmd_input_output_dict={ + "ovs-vsctl show": CmdOutput(OVS_SHOW_MULTI_LOCAL_IP), + "cat /run/nodeip-configuration/primary-ip": CmdOutput("10.6.135.202"), + }, + failed_msg=( + "Multiple different local_ip values in Geneve tunnels: ['10.6.135.1', '10.6.135.202']. " + "OVS configuration may be inconsistent." + ), + ), + RuleScenarioParams( + scenario_title="local_ip_stale_after_ip_change", + cmd_input_output_dict={ + "ovs-vsctl show": CmdOutput(OVS_SHOW_STALE_LOCAL_IP), + "cat /run/nodeip-configuration/primary-ip": CmdOutput("10.6.156.21"), + }, + failed_msg=( + "Geneve tunnel local_ip (10.6.135.1) does not match " + "node primary IP (10.6.156.21). " + "Inter-node pod connectivity may be broken." + ), + ), + ] + + scenario_skip = [ + RuleScenarioParams( + scenario_title="cannot_determine_node_ip", + cmd_input_output_dict={ + "ovs-vsctl show": CmdOutput(OVS_SHOW_WITH_GENEVE), + "cat /run/nodeip-configuration/primary-ip": CmdOutput("", return_code=1), + }, + ), + ] + + scenario_unexpected_system_output = [ + RuleScenarioParams( + scenario_title="ovs_fails_after_prerequisite_passed", + cmd_input_output_dict={ + "ovs-vsctl show": CmdOutput("", return_code=1), + }, + ), + ] + + @pytest.mark.parametrize("scenario_params", scenario_prerequisite_not_fulfilled) + def test_prerequisite_not_fulfilled(self, scenario_params, tested_object): + """Test that prerequisite is not fulfilled when OVS is unavailable or has no Geneve tunnels.""" + RuleTestBase.test_prerequisite_not_fulfilled(self, scenario_params, tested_object) + + @pytest.mark.parametrize("scenario_params", scenario_prerequisite_fulfilled) + def test_prerequisite_fulfilled(self, scenario_params, tested_object): + """Test that prerequisite is fulfilled when Geneve tunnels are present.""" + RuleTestBase.test_prerequisite_fulfilled(self, scenario_params, tested_object) + + @pytest.mark.parametrize("scenario_params", scenario_passed) + def test_scenario_passed(self, scenario_params, tested_object): + """Test that matching local_ip and node IP passes.""" + RuleTestBase.test_scenario_passed(self, scenario_params, tested_object) + + @pytest.mark.parametrize("scenario_params", scenario_failed) + def test_scenario_failed(self, scenario_params, tested_object): + """Test that stale or inconsistent local_ip values are detected.""" + RuleTestBase.test_scenario_failed(self, scenario_params, tested_object) + + @pytest.mark.parametrize("scenario_params", scenario_unexpected_system_output) + def test_scenario_unexpected_system_output(self, scenario_params, tested_object): + """Test that ovs-vsctl failure in run_rule raises UnExpectedSystemOutput.""" + RuleTestBase.test_scenario_unexpected_system_output(self, scenario_params, tested_object) + + @pytest.mark.parametrize("scenario_params", scenario_skip) + def test_scenario_skip_geneve(self, scenario_params, tested_object): + """Test that missing node IP results in SKIP.""" + from in_cluster_checks.utils.enums import Status + + self._init_validation_object(tested_object, scenario_params) + with self._apply_patches(scenario_params, tested_object): + result = tested_object.run_rule() + assert result.status == Status.SKIP, ( + f"Expected SKIP for scenario: {scenario_params.scenario_title}, got {result.status}" + )