Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions src/in_cluster_checks/domains/k8s_domain.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from typing import List

from in_cluster_checks.core.domain import RuleDomain
from in_cluster_checks.rules.k8s.k8s_validations import ( # AllPodsReadyAndRunning disabled (PDRIVE-806)
from in_cluster_checks.rules.k8s.k8s_validations import (
AllDeploymentsAvailable,
AllStatefulsetsReady,
CheckDeploymentsReplicaStatus,
Expand Down Expand Up @@ -53,7 +53,6 @@ def get_rule_classes(self) -> List[type]:
List of Rule classes
"""
return [
# AllPodsReadyAndRunning, # Disabled: replaced by InfraPodsReadyAndRunning (PDRIVE-806)
InfraPodsReadyAndRunning,
NodesAreReady,
NodesCpuAndMemoryStatus,
Expand Down
91 changes: 2 additions & 89 deletions src/in_cluster_checks/rules/k8s/k8s_validations.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,98 +15,11 @@
from in_cluster_checks.utils.enums import Objectives, Status


class AllPodsReadyAndRunning(OrchestratorRule):
"""Verify all pods are ready and in running state across all namespaces."""

objective_hosts = [Objectives.ORCHESTRATOR]
unique_name = "all_pods_are_running_all_namespaces"
title = "Verify all pods are ready and on running state"

def run_rule(self):
"""Check if all pods across all namespaces are ready and running."""
ready_pods, not_running_pods = self._get_pods_lists()

if len(ready_pods) == 0:
return RuleResult.failed("Did not get any pods from 'oc get pods --all-namespaces'")

if not_running_pods:
message = "Not all pods are running\n"
message += "Following pods are not running or partially not ready:\n"

# Format not-ready pods with details
for pod_info in not_running_pods:
namespace = pod_info["namespace"]
pod_name = pod_info["name"]
status = pod_info["status"]
ready = pod_info["ready"]
message += f" {namespace}/{pod_name} - Ready: {ready}, Status: {status}\n"

return RuleResult.failed(message)

return RuleResult.passed()

def _get_pods_lists(self):
"""
Get lists of ready and not-ready pods.

Returns:
tuple: (ready_pods_list, not_running_pods_list)
Each list contains dicts with pod information
"""
ready_pods = []
not_running_pods = []

# Use helper method from OrchestratorRule (logs command automatically)
pod_objects = self.oc_api.get_all_pods(all_namespaces=True, timeout=45)

if not pod_objects:
return [], []

for pod in pod_objects:
pod_data = pod.as_dict()
namespace = pod_data["metadata"]["namespace"]
pod_name = pod_data["metadata"]["name"]
status_dict = pod_data.get("status", {})

# Get phase (Running, Pending, Failed, etc.)
phase = status_dict.get("phase", "Unknown")

# Skip Completed jobs
if phase == "Succeeded":
continue

# Get container statuses
container_statuses = status_dict.get("containerStatuses", [])

# Calculate ready containers
total_containers = len(container_statuses)
ready_containers = sum(1 for c in container_statuses if c.get("ready", False))
ready_str = f"{ready_containers}/{total_containers}"

pod_info = {
"namespace": namespace,
"name": pod_name,
"status": phase,
"ready": ready_str,
}

# Check if pod is not running or not all containers are ready
if phase != "Running":
not_running_pods.append(pod_info)
elif ready_containers != total_containers:
not_running_pods.append(pod_info)
else:
ready_pods.append(pod_info)

return ready_pods, not_running_pods


class InfraPodsReadyAndRunning(OrchestratorRule):
"""Verify pods in infrastructure namespaces are ready and running.

Scoped replacement for AllPodsReadyAndRunning that only checks critical
OpenShift infrastructure namespaces, avoiding scale/memory issues and
false positives from user workload pods.
Checks critical OpenShift infrastructure namespaces only, avoiding
scale/memory issues and false positives from user workload pods.
"""

objective_hosts = [Objectives.ORCHESTRATOR]
Expand Down
77 changes: 1 addition & 76 deletions tests/rules/k8s/test_k8s_validations.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
"""
Tests for K8s/OpenShift validations.

Adapted from HealthChecks test patterns for AllPodsReadyAndRunning.
"""
"""Tests for K8s/OpenShift validations."""

import json
import logging
Expand All @@ -15,7 +11,6 @@
from in_cluster_checks.core.exceptions import UnExpectedSystemOutput
from in_cluster_checks.rules.k8s.k8s_validations import (
AllDeploymentsAvailable,
AllPodsReadyAndRunning,
AllStatefulsetsReady,
InfraPodsReadyAndRunning,
CheckDeploymentsReplicaStatus,
Expand All @@ -36,76 +31,6 @@
from tests.pytest_tools.test_rule_base import RuleScenarioParams, RuleTestBase


def create_mock_pod(namespace, name, phase, ready_containers, total_containers):
"""Create a mock pod object."""
mock_pod = Mock()
container_statuses = [{"ready": i < ready_containers} for i in range(total_containers)]
mock_pod.as_dict.return_value = {
"metadata": {"namespace": namespace, "name": name},
"status": {
"phase": phase,
"containerStatuses": container_statuses,
},
}
return mock_pod


class TestAllPodsReadyAndRunning:
"""Test AllPodsReadyAndRunning rule."""

@pytest.fixture
def tested_object(self):
"""Create instance of AllPodsReadyAndRunning for testing."""
return AllPodsReadyAndRunning(host_executor=Mock(), node_executors={})

def test_all_pods_running_and_ready(self, tested_object):
"""Test when all pods are running and ready."""
tested_object.oc_api.get_all_pods = Mock(
return_value=[
create_mock_pod("default", "pod1", "Running", 2, 2),
create_mock_pod("kube-system", "pod2", "Running", 1, 1),
]
)

result = tested_object.run_rule()
assert result.status == Status.PASSED

def test_some_pods_not_running(self, tested_object):
"""Test when some pods are not in Running state."""
tested_object.oc_api.get_all_pods = Mock(
return_value=[
create_mock_pod("default", "running-pod", "Running", 1, 1),
create_mock_pod("default", "pending-pod", "Pending", 0, 1),
]
)

result = tested_object.run_rule()
assert result.status == Status.FAILED
assert "pending-pod" in result.message
assert "Pending" in result.message

def test_completed_pods_ignored(self, tested_object):
"""Test that completed/succeeded pods are ignored."""
tested_object.oc_api.get_all_pods = Mock(
return_value=[
create_mock_pod("default", "running-pod", "Running", 1, 1),
create_mock_pod("default", "completed-job", "Succeeded", 0, 1),
]
)

result = tested_object.run_rule()
# Should pass because completed jobs are ignored
assert result.status == Status.PASSED

def test_no_pods_found(self, tested_object):
"""Test when no pods are found in the cluster."""
tested_object.oc_api.get_all_pods = Mock(return_value=[])

result = tested_object.run_rule()
assert result.status == Status.FAILED
assert "Did not get any pods" in result.message


def create_mock_infra_pod(
namespace, name, phase, ready_containers, total_containers, owner_kind=None, creation_timestamp=None,
finished_at=None,
Expand Down
Loading