diff --git a/.github/workflows/api-parity-audit.yml b/.github/workflows/api-parity-audit.yml index 62c0c19117..1e89a2dacf 100644 --- a/.github/workflows/api-parity-audit.yml +++ b/.github/workflows/api-parity-audit.yml @@ -33,8 +33,8 @@ on: required: false default: false type: boolean - schedule: - - cron: '0 6 * * 1' # Weekly on Monday 6am UTC + # schedule: + # - cron: '0 6 * * 1' # Weekly on Monday 6am UTC — disabled to avoid interfering with other runs concurrency: group: ${{ github.workflow }} diff --git a/.github/workflows/cross-cluster-restore.yml b/.github/workflows/cross-cluster-restore.yml new file mode 100644 index 0000000000..afeea05221 --- /dev/null +++ b/.github/workflows/cross-cluster-restore.yml @@ -0,0 +1,238 @@ +name: Cross-Cluster Restore Test (Docker) +run-name: "Cross-Cluster Restore | ${{ inputs.SBCLI_BRANCH || 'main' }} | ${{ inputs.MNODES || '192.168.10.210' }} | ${{ inputs.BOOTSTRAP_DATA_CHUNKS || '1' }}+${{ inputs.BOOTSTRAP_PARITY_CHUNKS || '2' }}" + +on: + workflow_dispatch: + inputs: + # ========================= + # Node inputs + # ========================= + STORAGE_NODE_IPS: + description: "Space-separated storage node IPs (split evenly between 2 clusters; min 2*(NDCS+NPCS) nodes)" + required: true + type: string + default: "192.168.10.201 192.168.10.202 192.168.10.203 192.168.10.204" + + MNODES: + description: "Management node IP (shared by both clusters)" + required: true + type: string + default: "192.168.10.210" + + # ========================= + # Bootstrap params + # ========================= + BOOTSTRAP_DATA_CHUNKS: + description: "Data chunks per stripe (NDCS) — min nodes per cluster = NDCS + NPCS" + required: true + type: string + default: "1" + + BOOTSTRAP_PARITY_CHUNKS: + description: "Parity chunks per stripe (NPCS)" + required: true + type: string + default: "2" + + BOOTSTRAP_JOURNAL_PARTITION: + description: "Journal partition index" + required: false + type: string + default: "1" + + BOOTSTRAP_HA_JM_COUNT: + description: "HA journal manager count" + required: false + type: string + default: "4" + + BOOTSTRAP_DATA_NIC: + description: "Data NIC interface" + required: false + type: string + default: "eth1" + + BOOTSTRAP_MAX_SUBSYS: + description: "Max NVMe-oF subsystems per node" + required: false + type: string + default: "300" + + # ========================= + # Cluster / lab + # ========================= + SBCLI_BRANCH: + description: "sbcli repo branch to clone for tests" + required: false + type: string + default: "main" + + CUSTOM_IMAGES: + description: 'Image overrides: spdk="..." docker="..."' + required: false + type: string + default: 'spdk="simplyblock/spdk:main-latest" docker="simplyblock/simplyblock:main"' + + CLIENTNODES: + description: "Space-separated client/FIO node IPs" + required: false + type: string + default: "192.168.10.165 192.168.10.166" + + SSH_USER: + description: "SSH user for all nodes" + required: false + type: string + default: "root" + + KEY_PATH: + description: "SSH private key path on runner" + required: false + type: string + default: "/home/ec2-user/.ssh/simplyblock-us-east-2.pem" + + NFS_MOUNTPOINT: + description: "NFS mount path" + required: false + type: string + default: "/mnt/nfs_share" + + EXTRA_CLUSTER_ARGS: + description: "Additional args for sbcli cluster create" + required: false + type: string + + EXTRA_SN_ARGS: + description: "Additional args for sbcli storage-node add-node" + required: false + type: string + + send_slack_notification: + description: "Send Slack notification on completion" + required: false + type: boolean + default: true + +concurrency: + group: simplyblock-lab-cross-cluster-${{ inputs.MNODES || '192.168.10.210' }} + cancel-in-progress: true + +jobs: + # ============================================================ + # Job 1: Validate node count and split IPs between 2 clusters + # ============================================================ + validate-and-split: + name: Validate nodes & split for 2 clusters + runs-on: [self-hosted] + outputs: + cluster1_nodes: ${{ steps.split.outputs.cluster1_nodes }} + cluster2_nodes: ${{ steps.split.outputs.cluster2_nodes }} + total_nodes: ${{ steps.split.outputs.total_nodes }} + c1_count: ${{ steps.split.outputs.c1_count }} + c2_count: ${{ steps.split.outputs.c2_count }} + + steps: + - name: Validate node count and split IPs + id: split + shell: bash + run: | + set -euo pipefail + + # Parse input IPs into array + read -ra IPS <<< "${{ inputs.STORAGE_NODE_IPS }}" + TOTAL=${#IPS[@]} + + NDCS=${{ inputs.BOOTSTRAP_DATA_CHUNKS }} + NPCS=${{ inputs.BOOTSTRAP_PARITY_CHUNKS }} + MIN_PER_CLUSTER=$((NDCS + NPCS)) + MIN_TOTAL=$((MIN_PER_CLUSTER * 2)) + + echo "=== Cross-Cluster Node Validation ===" + echo "Total storage node IPs provided: ${TOTAL}" + echo "NDCS=${NDCS}, NPCS=${NPCS}" + echo "Min nodes per cluster (n+k): ${MIN_PER_CLUSTER}" + echo "Min total nodes needed: ${MIN_TOTAL}" + + if [[ ${TOTAL} -lt ${MIN_TOTAL} ]]; then + echo "ERROR: Need at least ${MIN_TOTAL} nodes for 2 clusters with ${NDCS}+${NPCS} configuration, but only got ${TOTAL}." + echo "Provide at least ${MIN_PER_CLUSTER} nodes per cluster." + exit 1 + fi + + # Split evenly: first half → cluster-1, second half → cluster-2 + HALF=$((TOTAL / 2)) + + C1_IPS="" + C2_IPS="" + for i in "${!IPS[@]}"; do + if [[ $i -lt $HALF ]]; then + C1_IPS="${C1_IPS:+${C1_IPS} }${IPS[$i]}" + else + C2_IPS="${C2_IPS:+${C2_IPS} }${IPS[$i]}" + fi + done + + C1_COUNT=$(echo "${C1_IPS}" | wc -w) + C2_COUNT=$(echo "${C2_IPS}" | wc -w) + + echo "" + echo "=== Split Result ===" + echo "Cluster-1 nodes (${C1_COUNT}): ${C1_IPS}" + echo "Cluster-2 nodes (${C2_COUNT}): ${C2_IPS}" + + # Validate both halves have enough nodes + if [[ ${C1_COUNT} -lt ${MIN_PER_CLUSTER} ]]; then + echo "ERROR: Cluster-1 has ${C1_COUNT} nodes but needs at least ${MIN_PER_CLUSTER}" + exit 1 + fi + if [[ ${C2_COUNT} -lt ${MIN_PER_CLUSTER} ]]; then + echo "ERROR: Cluster-2 has ${C2_COUNT} nodes but needs at least ${MIN_PER_CLUSTER}" + exit 1 + fi + + echo "cluster1_nodes=${C1_IPS}" >> "$GITHUB_OUTPUT" + echo "cluster2_nodes=${C2_IPS}" >> "$GITHUB_OUTPUT" + echo "total_nodes=${TOTAL}" >> "$GITHUB_OUTPUT" + echo "c1_count=${C1_COUNT}" >> "$GITHUB_OUTPUT" + echo "c2_count=${C2_COUNT}" >> "$GITHUB_OUTPUT" + + # ============================================================ + # Job 2: Bootstrap cluster-1 + run cross-cluster restore test + # + # Uses e2e-bootstrap.yml which: + # - Cleans all nodes (STORAGE_PRIVATE_IPS + NEW_NODE_IPS) + # - Bootstraps cluster-1 from STORAGE_PRIVATE_IPS + # - Passes NEW_NODE_IPS to the test as spare nodes + # - TestBackupCrossClusterRestore auto-bootstraps cluster-2 + # from spare nodes (installs sbcli, creates cluster, adds + # nodes, activates, creates pool) + # ============================================================ + cross-cluster-restore: + name: Bootstrap C1 + Cross-Cluster Restore Test + needs: validate-and-split + uses: ./.github/workflows/e2e-bootstrap.yml + with: + STORAGE_PRIVATE_IPS: ${{ needs.validate-and-split.outputs.cluster1_nodes }} + NEW_NODE_IPS: ${{ needs.validate-and-split.outputs.cluster2_nodes }} + MNODES: ${{ inputs.MNODES }} + API_INVOKE_URL: "http://${{ inputs.MNODES }}/" + BASTION_IP: ${{ inputs.MNODES }} + GRAFANA_ENDPOINT: "http://${{ inputs.MNODES }}/grafana" + TEST_CLASS: "TestBackupCrossClusterRestore" + CLUSTER_SECURITY: "backup" + SBCLI_BRANCH: ${{ inputs.SBCLI_BRANCH || 'main' }} + CUSTOM_IMAGES: ${{ inputs.CUSTOM_IMAGES || 'spdk="simplyblock/spdk:main-latest" docker="simplyblock/simplyblock:main"' }} + CLIENTNODES: ${{ inputs.CLIENTNODES || '192.168.10.165 192.168.10.166' }} + SSH_USER: ${{ inputs.SSH_USER || 'root' }} + KEY_PATH: ${{ inputs.KEY_PATH || '/home/ec2-user/.ssh/simplyblock-us-east-2.pem' }} + NFS_MOUNTPOINT: ${{ inputs.NFS_MOUNTPOINT || '/mnt/nfs_share' }} + BOOTSTRAP_MAX_SUBSYS: ${{ inputs.BOOTSTRAP_MAX_SUBSYS || '300' }} + BOOTSTRAP_DATA_CHUNKS: ${{ inputs.BOOTSTRAP_DATA_CHUNKS || '1' }} + BOOTSTRAP_PARITY_CHUNKS: ${{ inputs.BOOTSTRAP_PARITY_CHUNKS || '2' }} + BOOTSTRAP_JOURNAL_PARTITION: ${{ inputs.BOOTSTRAP_JOURNAL_PARTITION || '1' }} + BOOTSTRAP_HA_JM_COUNT: ${{ inputs.BOOTSTRAP_HA_JM_COUNT || '4' }} + BOOTSTRAP_DATA_NIC: ${{ inputs.BOOTSTRAP_DATA_NIC || 'eth1' }} + EXTRA_CLUSTER_ARGS: ${{ inputs.EXTRA_CLUSTER_ARGS || '' }} + EXTRA_SN_ARGS: ${{ inputs.EXTRA_SN_ARGS || '' }} + send_slack_notification: ${{ inputs.send_slack_notification }} + secrets: inherit diff --git a/.github/workflows/e2e-bootstrap-k8s.yml b/.github/workflows/e2e-bootstrap-k8s.yml deleted file mode 100755 index efefb8523c..0000000000 --- a/.github/workflows/e2e-bootstrap-k8s.yml +++ /dev/null @@ -1,1037 +0,0 @@ -name: Bootstrap K8s Cluster + Run E2E Tests -run-name: "K8s E2E Bootstrap | ${{ inputs.SBCLI_BRANCH }} | ${{ inputs.K3S_MNODES }}" - -on: - workflow_dispatch: - inputs: - # ========================= - # Cluster / Lab inputs - # ========================= - K3S_MNODES: - description: "K3s master node IP (also used as API endpoint)" - required: true - default: "192.168.10.81" - - STORAGE_PRIVATE_IPS: - description: "Space-separated storage node IPs (K8s worker nodes)" - required: true - default: "192.168.10.82 192.168.10.83 192.168.10.84" - - API_INVOKE_URL: - description: "Cluster API URL (default: http:///)" - required: false - default: "" - - GRAFANA_ENDPOINT: - description: "Grafana endpoint" - required: false - default: "" - - SBCLI_BRANCH: - description: "sbcli repo branch for test code" - required: true - default: "main" - - K8S_CHARTS_BRANCH: - description: "sbcli branch that contains simplyblock_core/scripts/charts" - required: false - default: "main-sfam-2359" - - K8S_NAMESPACE: - description: "Kubernetes namespace for simplyblock" - required: false - default: "simplyblock" - - # ========================= - # SSH / client inputs - # ========================= - SSH_USER: - description: "SSH user for all nodes" - required: true - default: "root" - - KEY_PATH: - description: "SSH private key path on runner" - required: true - default: "/home/ec2-user/.ssh/simplyblock-us-east-2.pem" - - CLIENTNODES: - description: "Space-separated client node IPs" - required: true - default: "192.168.10.165 192.168.10.166" - - # ========================= - # Cleanup inputs - # ========================= - NFS_MOUNTPOINT: - description: "NFS mountpoint to unmount everywhere" - required: true - default: "/mnt/nfs_share" - - # ========================= - # Bootstrap params - # ========================= - BOOTSTRAP_DATA_NIC: - description: "Data NIC on storage nodes" - required: true - default: "eth1" - - EXTRA_HELM_ARGS: - description: "Extra --set flags for helm upgrade --install (space-separated)" - required: false - default: "" - - # ========================= - # E2E test inputs - # ========================= - TEST_CLASS: - description: "E2E test class name (empty = run all)" - required: false - default: "" - - NDCS: - description: "ndcs passed to e2e.py" - required: false - default: "1" - - NPCS: - description: "npcs passed to e2e.py" - required: false - default: "1" - - RUN_HA: - description: "Enable HA tests" - type: boolean - required: false - default: false - - CUSTOM_IMAGES: - description: "Image overrides: set spdk and/or docker values, leave as \"\" to skip." - default: 'spdk="" docker=""' - required: false - - USE_EXISTING_CLUSTER: - description: "Use existing K8s cluster (skip k3s bootstrap). Kubeconfig is read from K8S_KUBECONFIG secret." - type: boolean - required: false - default: false - -concurrency: - group: simplyblock-lab-k8s-e2e-${{ inputs.K3S_MNODES }} - cancel-in-progress: false - -jobs: - bootstrap-and-e2e-k8s: - name: K8s Pre-clean -> Bootstrap -> Helm -> E2E - runs-on: [self-hosted] - timeout-minutes: 1440 - - env: - # Cluster/lab env - K3S_MNODES: ${{ inputs.K3S_MNODES }} - MNODES: ${{ inputs.K3S_MNODES }} - STORAGE_PRIVATE_IPS: ${{ inputs.STORAGE_PRIVATE_IPS }} - BASTION_SERVER: ${{ inputs.K3S_MNODES }} - BASTION_IP: ${{ inputs.K3S_MNODES }} - K8S_NAMESPACE: ${{ inputs.K8S_NAMESPACE || 'simplyblock' }} - GRAFANA_ENDPOINT: ${{ inputs.GRAFANA_ENDPOINT || '' }} - SBCLI_CMD: "sbctl" - SBCLI_BRANCH: ${{ inputs.SBCLI_BRANCH }} - K8S_CHARTS_BRANCH: ${{ inputs.K8S_CHARTS_BRANCH || 'main-sfam-2359' }} - BOOTSTRAP_DATA_NIC: ${{ inputs.BOOTSTRAP_DATA_NIC }} - EXTRA_HELM_ARGS: ${{ inputs.EXTRA_HELM_ARGS || '' }} - - # SSH/client env - SSH_USER: ${{ inputs.SSH_USER }} - KEY_PATH: ${{ inputs.KEY_PATH }} - CLIENTNODES: ${{ inputs.CLIENTNODES }} - CLIENT_IP: ${{ inputs.CLIENTNODES }} - - # Cleanup - NFS_MOUNTPOINT: ${{ inputs.NFS_MOUNTPOINT }} - - # E2E - TEST_CLASS: ${{ inputs.TEST_CLASS || '' }} - NDCS: ${{ inputs.NDCS || '1' }} - NPCS: ${{ inputs.NPCS || '1' }} - CUSTOM_IMAGES: ${{ inputs.CUSTOM_IMAGES || 'spdk="" docker=""' }} - - # Secrets - SSH_PASSWORD: ${{ secrets.SSH_PASSWORD }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }} - MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }} - SUPABASE_ANON_KEY: ${{ secrets.SUPABASE_ANON_KEY }} - - # Filled during run - API_BASE_URL: "" - API_INVOKE_URL: "" - CLUSTER_ID: "" - CLUSTER_SECRET: "" - - steps: - - name: Runner diagnostics - shell: bash - run: | - set -euxo pipefail - uname -a; whoami; pwd - python3 --version || true - git --version - kubectl version --client 2>/dev/null || echo "kubectl not yet installed" - helm version 2>/dev/null || echo "helm not yet installed" - - - name: Clear stale test artifacts - shell: bash - run: rm -f sbcli/e2e/output.log || true - - # ============================================================ - # TOOL INSTALL - # ============================================================ - - name: Install kubectl (if missing) - shell: bash - run: | - set -euxo pipefail - if command -v kubectl >/dev/null 2>&1; then echo "kubectl present"; exit 0; fi - curl -LO "https://dl.k8s.io/release/$(curl -Ls https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - chmod +x kubectl && sudo mv kubectl /usr/local/bin/kubectl - kubectl version --client - - - name: Install helm (if missing) - shell: bash - run: | - set -euxo pipefail - if command -v helm >/dev/null 2>&1; then echo "helm present"; exit 0; fi - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - helm version - - - name: Install prereqs (sshpass) - shell: bash - run: | - set -euxo pipefail - if command -v sshpass >/dev/null 2>&1; then exit 0; fi - if command -v apt-get >/dev/null 2>&1; then - sudo apt-get update -y && sudo apt-get install -y sshpass - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y epel-release || true && sudo yum install -y sshpass - elif command -v dnf >/dev/null 2>&1; then - sudo dnf install -y sshpass - else - echo "ERROR: Cannot install sshpass" && exit 1 - fi - - # ============================================================ - # IMAGE OVERRIDES - # ============================================================ - - name: Parse CUSTOM_IMAGES overrides - shell: bash - run: | - set -euxo pipefail - custom="${{ inputs.CUSTOM_IMAGES }}" - for item in $custom; do - key="${item%%=*}"; value="${item#*=}"; value="${value//\"/}" - if [[ -z "$value" ]]; then echo "Skipping $key (empty)"; continue; fi - case "$key" in - spdk) echo "SIMPLY_BLOCK_SPDK_ULTRA_IMAGE=$value" >> "$GITHUB_ENV" ;; - docker) echo "SIMPLY_BLOCK_DOCKER_IMAGE=$value" >> "$GITHUB_ENV" ;; - *) echo "Unknown image key: $key (ignored)" ;; - esac - done - - # ============================================================ - # KEY SETUP - # ============================================================ - - name: Resolve and validate SSH KEY_PATH - shell: bash - run: | - set -euxo pipefail - kp="${KEY_PATH}" - kp="${kp%\"}"; kp="${kp#\"}"; kp="${kp%\'}"; kp="${kp#\'}" - [[ "$kp" == .ssh/* ]] && kp="${HOME}/${kp}" - [[ "$kp" == ~/* ]] && kp="${HOME}/${kp#~/}" - echo "KEY_PATH=$kp" >> "$GITHUB_ENV" - test -f "$kp" || (echo "ERROR: SSH key not found at $kp" && exit 1) - chmod 600 "$kp" || true - - - name: Export KEY_NAME from KEY_PATH - shell: bash - run: echo "KEY_NAME=$(basename "${KEY_PATH}")" >> "$GITHUB_ENV" - - - name: Validate required secrets - shell: bash - run: | - [[ -n "${SSH_PASSWORD}" ]] || (echo "ERROR: secrets.SSH_PASSWORD required" && exit 1) - - # ============================================================ - # PRE-BOOTSTRAP CLEANUP - # ============================================================ - - name: Pre-clean — kill fio/tmux, unmount NFS - shell: bash - run: | - set -euxo pipefail - run_remote() { - sshpass -p "${SSH_PASSWORD}" ssh \ - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "${SSH_USER}@$1" "bash -s" <<< "$2" - } - run_remote_with_retry() { - local ip="$1" - local script="$2" - local max=5 - for attempt in $(seq 1 $max); do - run_remote "$ip" "$script" && return 0 - echo "Attempt $attempt/$max failed for $ip, retrying in 5s..." - sleep 5 - done - echo "All $max attempts failed for $ip, continuing..." - return 0 - } - targets="$K3S_MNODES $STORAGE_PRIVATE_IPS $CLIENTNODES" - uniq_targets="$(echo "$targets" | tr ' ' '\n' | sed '/^$/d' | sort -u | tr '\n' ' ')" - for ip in $uniq_targets; do - run_remote_with_retry "$ip" "pkill -9 fio || true; pkill -9 tmux || true - mp='${NFS_MOUNTPOINT}' - mountpoint -q \"\$mp\" && umount -f \"\$mp\" || true" - done - - - name: Pre-clean — uninstall k3s / k3s-agent, remove containers - if: ${{ inputs.USE_EXISTING_CLUSTER != true }} - shell: bash - run: | - set -euxo pipefail - run_remote() { - sshpass -p "${SSH_PASSWORD}" ssh \ - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "${SSH_USER}@$1" "bash -s" <<< "$2" - } - echo "---- k3s master: $K3S_MNODES ----" - run_remote "$K3S_MNODES" "set -euxo pipefail - systemctl stop firewalld || true; systemctl stop ufw || true - k3s-uninstall.sh || true - docker stop \$(docker ps -aq) || true - docker rm -f \$(docker ps -aq) || true - docker system prune -af || true - pip uninstall -y '${SBCLI_CMD}' || true - rm -rf /usr/local/bin/sbc* || true" - for ip in $STORAGE_PRIVATE_IPS; do - run_remote "$ip" "set -euxo pipefail - systemctl stop firewalld || true; systemctl stop ufw || true - k3s-agent-uninstall.sh || true - docker stop \$(docker ps -aq) || true - docker rm -f \$(docker ps -aq) || true - docker system prune -af || true - pip uninstall -y '${SBCLI_CMD}' || true - rm -rf /usr/local/bin/sbc* || true" - sleep 10 - done - - - name: Pre-clean — disconnect lvols and unmount /mnt on clients - shell: bash - run: | - set -euxo pipefail - run_remote() { - sshpass -p "${SSH_PASSWORD}" ssh \ - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "${SSH_USER}@$1" "bash -s" <<< "$2" - } - for ip in $CLIENTNODES; do - run_remote "$ip" "set -euxo pipefail - subsystems=\$(nvme list-subsys 2>/dev/null | grep -i lvol | awk '{print \$3}' | cut -d '=' -f 2 || true) - for s in \$subsystems; do nvme disconnect -n \"\$s\" || true; done - mps=\$(mount | grep ' /mnt' | awk '{print \$3}' || true) - for mp in \$mps; do umount -f \"\$mp\" || true; done - dirs=\$(find /mnt -mindepth 1 -type d 2>/dev/null || true) - for d in \$dirs; do rm -rf \"\$d\" || true; done" - done - - - name: Pre-clean — remove /etc/simplyblock - if: ${{ inputs.USE_EXISTING_CLUSTER != true }} - shell: bash - run: | - set -euxo pipefail - run_remote() { - sshpass -p "${SSH_PASSWORD}" ssh \ - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "${SSH_USER}@$1" "bash -s" <<< "$2" - } - for ip in $K3S_MNODES $STORAGE_PRIVATE_IPS; do - run_remote "$ip" "rm -rf /etc/simplyblock || true" - done - - - name: Reboot storage nodes + wait online + disk reset - if: ${{ inputs.USE_EXISTING_CLUSTER != true }} - shell: bash - run: | - set -euxo pipefail - run_remote() { - sshpass -p "${SSH_PASSWORD}" ssh \ - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "${SSH_USER}@$1" "bash -s" <<< "$2" - } - for ip in $STORAGE_PRIVATE_IPS; do - sshpass -p "${SSH_PASSWORD}" ssh -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null "${SSH_USER}@${ip}" \ - "nohup reboot >/dev/null 2>&1 &" || true - done - for ip in $STORAGE_PRIVATE_IPS; do - for i in {1..60}; do - if sshpass -p "${SSH_PASSWORD}" ssh -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \ - "${SSH_USER}@${ip}" "echo online" >/dev/null 2>&1; then - echo "$ip online"; break - fi - sleep 10 - [[ "$i" -lt 60 ]] || (echo "ERROR: $ip did not come online" && exit 1) - done - done - for ip in $STORAGE_PRIVATE_IPS; do - run_remote "$ip" "set -euxo pipefail - for dev in /dev/nvme0n1 /dev/nvme1n1 /dev/nvme2n1 /dev/nvme3n1; do - [[ -b \$dev ]] && parted \$dev --script rm 1 rm 2 rm 3 || true - done - for i in 0 1 2 3; do - dev=/dev/nvme\${i}n1 - [[ -b \$dev ]] && parted -fs \$dev mklabel gpt || true - done" - done - - # ============================================================ - # K3S BOOTSTRAP - # ============================================================ - - name: Clone simplyBlockDeploy (bootstrap repo) - if: ${{ inputs.USE_EXISTING_CLUSTER != true }} - shell: bash - run: | - set -euxo pipefail - rm -rf simplyBlockDeploy - git clone https://github.com/simplyblock-io/simplyBlockDeploy.git simplyBlockDeploy - test -f simplyBlockDeploy/bare-metal/bootstrap-k3s.sh - - - name: Bootstrap K3s cluster (--k8s-snode) - if: ${{ inputs.USE_EXISTING_CLUSTER != true }} - shell: bash - run: | - set -euxo pipefail - cd simplyBlockDeploy/bare-metal - chmod +x ./bootstrap-k3s.sh - extra_args=() - [[ -n "${SIMPLY_BLOCK_SPDK_ULTRA_IMAGE-}" ]] && extra_args+=( --spdk-image "${SIMPLY_BLOCK_SPDK_ULTRA_IMAGE}" ) - [[ -n "${SIMPLY_BLOCK_DOCKER_IMAGE-}" ]] && extra_args+=( --docker-image "${SIMPLY_BLOCK_DOCKER_IMAGE}" ) - [[ -n "${BOOTSTRAP_DATA_NIC:-}" ]] && extra_args+=( --data-nics "${BOOTSTRAP_DATA_NIC}" ) - export K3S_MNODES="${K3S_MNODES}" - export STORAGE_PRIVATE_IPS="${STORAGE_PRIVATE_IPS}" - set +e - ./bootstrap-k3s.sh --k8s-snode "${extra_args[@]}" | tee bootstrap-k3s.log - rc=${PIPESTATUS[0]}; set -e - [[ "$rc" -eq 0 ]] || (echo "ERROR: bootstrap-k3s.sh failed (rc=$rc)" && exit "$rc") - - - name: Copy kubeconfig from K3s master to runner - if: ${{ inputs.USE_EXISTING_CLUSTER != true }} - shell: bash - run: | - set -euxo pipefail - mkdir -p ~/.kube - sshpass -p "${SSH_PASSWORD}" scp \ - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "${SSH_USER}@${K3S_MNODES}:/etc/rancher/k3s/k3s.yaml" ~/.kube/config_k3s - sed -i "s|127.0.0.1|${K3S_MNODES}|g" ~/.kube/config_k3s - sed -i "s|localhost|${K3S_MNODES}|g" ~/.kube/config_k3s - echo "KUBECONFIG=${HOME}/.kube/config_k3s" >> "$GITHUB_ENV" - kubectl get nodes - - - name: Write kubeconfig from secret (existing cluster) - if: ${{ inputs.USE_EXISTING_CLUSTER == true }} - shell: bash - run: | - set -euxo pipefail - mkdir -p ~/.kube - echo "${{ secrets.K8S_KUBECONFIG }}" > ~/.kube/config_k3s - chmod 600 ~/.kube/config_k3s - echo "KUBECONFIG=${HOME}/.kube/config_k3s" >> "$GITHUB_ENV" - kubectl get nodes - - # ============================================================ - # HELM DEPLOY - # ============================================================ - - name: Pre-Helm cleanup (uninstall + delete CRDs/resources) - shell: bash - run: | - set -euxo pipefail - NS="${K8S_NAMESPACE}" - - # Uninstall existing Helm release (ignore error if not installed) - helm uninstall sbcli -n "${NS}" 2>/dev/null || true - - # Force-delete all pods, PVCs, PVs - kubectl -n "${NS}" delete pod --all --force --grace-period=0 2>/dev/null || true - kubectl -n "${NS}" delete pvc --all --force --grace-period=0 2>/dev/null || true - kubectl delete pv --all --force --grace-period=0 2>/dev/null || true - - # Remove finalizers then delete known CRD instances - RESOURCES=( - "simplyblockpool.storage.simplyblock.io simplyblock-pool" - "simplyblockpool.storage.simplyblock.io simplyblock-pool2" - "simplyblocklvol.storage.simplyblock.io simplyblock-lvol" - "simplyblocktask.storage.simplyblock.io simplyblock-task" - "simplyblockdevices.storage.simplyblock.io simplyblock-devices" - "simplyblockdevices.storage.simplyblock.io simplyblock-device-action" - "simplyblockstoragenodes.storage.simplyblock.io simplyblock-node" - "simplyblockstoragenodes.storage.simplyblock.io simplyblock-node2" - "simplyblockstoragenodes.storage.simplyblock.io simplyblock-node-action" - "simplyblockstoragenodesets.storage.simplyblock.io simplyblock-node" - "simplyblockstoragenodesets.storage.simplyblock.io simplyblock-node2" - "simplyblockstoragenodesets.storage.simplyblock.io simplyblock-node-action" - "simplyblockstorageclusters.storage.simplyblock.io simplyblock-cluster" - "simplyblockstorageclusters.storage.simplyblock.io simplyblock-cluster2" - "simplyblockstorageclusters.storage.simplyblock.io simplyblock-cluster-activate" - "simplyblocksnapshotreplications.storage.simplyblock.io simplyblock-snap-replication" - "simplyblocksnapshotreplications.storage.simplyblock.io simplyblock-snap-replication-failback" - ) - for item in "${RESOURCES[@]}"; do - KIND=$(echo "$item" | awk '{print $1}') - NAME=$(echo "$item" | awk '{print $2}') - kubectl -n "${NS}" patch "${KIND}" "${NAME}" \ - --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true - kubectl -n "${NS}" delete "${KIND}" "${NAME}" --ignore-not-found 2>/dev/null || true - done - echo "Pre-Helm cleanup done." - - - name: Build worker node names for Helm - shell: bash - run: | - set -euxo pipefail - # Generate worker1,worker2,...,workerN from the number of STORAGE_PRIVATE_IPS - count=$(echo "${STORAGE_PRIVATE_IPS}" | tr ' ' '\n' | grep -c '\S') - worker_csv=$(seq 1 "${count}" | sed 's/^/worker/' | tr '\n' ',' | sed 's/,$//') - echo "Worker node names: ${worker_csv}" - [[ -n "${worker_csv}" ]] || (echo "ERROR: STORAGE_PRIVATE_IPS is empty" && exit 1) - echo "HELM_WORKER_NODES={${worker_csv}}" >> "$GITHUB_ENV" - - - name: Clone sbcli charts branch - shell: bash - run: | - set -euxo pipefail - rm -rf sbcli-charts - git clone --branch "${K8S_CHARTS_BRANCH}" --single-branch \ - https://github.com/simplyblock-io/sbcli.git sbcli-charts - test -d sbcli-charts/simplyblock_core/scripts/charts - - - name: Deploy simplyblock via Helm - shell: bash - run: | - set -euxo pipefail - cd sbcli-charts/simplyblock_core/scripts/charts - extra_set=() - if [[ -n "${SIMPLY_BLOCK_DOCKER_IMAGE-}" ]]; then - extra_set+=( --set "image.repository=${SIMPLY_BLOCK_DOCKER_IMAGE%%:*}" ) - extra_set+=( --set "image.tag=${SIMPLY_BLOCK_DOCKER_IMAGE##*:}" ) - fi - if [[ -n "${HELM_WORKER_NODES:-}" ]]; then - extra_set+=( --set "storageNodes.workerNodes=${HELM_WORKER_NODES}" ) - fi - [[ -n "${EXTRA_HELM_ARGS:-}" ]] && read -ra extra_caller <<< "${EXTRA_HELM_ARGS}" && extra_set+=( "${extra_caller[@]}" ) - helm upgrade --install sbcli \ - --namespace "${K8S_NAMESPACE}" \ - --create-namespace \ - "${extra_set[@]}" \ - ./ | tee helm-install.log - - - name: Wait for admin-control pod to be Running - shell: bash - run: | - set -euxo pipefail - for i in {1..60}; do - status=$(kubectl get pods -n "${K8S_NAMESPACE}" --no-headers \ - -o custom-columns=:metadata.name,:status.phase \ - | grep simplyblock-admin-control | awk '{print $2}' || true) - [[ "${status}" == "Running" ]] && echo "Admin pod Running." && break - echo "Attempt $i: status=${status:-pending}. Waiting 10s..." - sleep 10 - [[ "$i" -lt 60 ]] || (echo "ERROR: admin pod did not start" && exit 1) - done - kubectl get pods -n "${K8S_NAMESPACE}" - - - name: Wait for cluster to be active - shell: bash - run: | - set -euxo pipefail - admin_pod="$(kubectl get pods -n "${K8S_NAMESPACE}" --no-headers \ - -o custom-columns=:metadata.name | grep simplyblock-admin-control | head -1)" - echo "Admin pod: ${admin_pod}" - for i in {1..120}; do - output="$(kubectl exec -n "${K8S_NAMESPACE}" "${admin_pod}" -- \ - bash -c "${SBCLI_CMD} cluster list" 2>/dev/null || true)" - echo "cluster list output: ${output}" - if echo "${output}" | grep -qi "active"; then - echo "Cluster is active." - break - fi - echo "Attempt $i: cluster not active yet. Waiting 10s..." - sleep 10 - [[ "$i" -lt 120 ]] || (echo "ERROR: cluster did not become active" && exit 1) - done - - # ============================================================ - # CLUSTER CREDENTIALS - # ============================================================ - - name: Resolve API_BASE_URL - shell: bash - run: | - set -euxo pipefail - if [[ -n "${{ inputs.API_INVOKE_URL }}" ]]; then - url="${{ inputs.API_INVOKE_URL }}" - else - url="http://${K3S_MNODES}/" - fi - echo "API_BASE_URL=${url}" >> "$GITHUB_ENV" - echo "API_INVOKE_URL=${url}" >> "$GITHUB_ENV" - echo "API_BASE_URL=${url}" - - - name: Fetch CLUSTER_ID and CLUSTER_SECRET via kubectl exec - shell: bash - run: | - set -euxo pipefail - admin_pod="$(kubectl get pods -n "${K8S_NAMESPACE}" --no-headers \ - -o custom-columns=:metadata.name | grep simplyblock-admin-control | head -1)" - echo "Admin pod: ${admin_pod}" - - cluster_id="$(kubectl exec -n "${K8S_NAMESPACE}" "${admin_pod}" -- \ - bash -c "${SBCLI_CMD} cluster list" \ - | grep -Eo '[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}' | head -1 || true)" - - [[ -n "${cluster_id}" ]] || (echo "ERROR: could not get cluster_id" && exit 1) - - cluster_secret="$(kubectl exec -n "${K8S_NAMESPACE}" "${admin_pod}" -- \ - bash -c "${SBCLI_CMD} cluster get-secret ${cluster_id}" \ - | tr -d '\r' | tail -n 1 | xargs)" - - [[ -n "${cluster_secret}" ]] || (echo "ERROR: could not get cluster_secret" && exit 1) - - echo "CLUSTER_ID=${cluster_id}" >> "$GITHUB_ENV" - echo "CLUSTER_SECRET=${cluster_secret}" >> "$GITHUB_ENV" - echo "Fetched CLUSTER_ID=${cluster_id}" - - # ============================================================ - # E2E TESTS - # ============================================================ - - name: Clone sbcli repo (test code) - shell: bash - run: | - set -euxo pipefail - rm -rf sbcli - wf_branch="${{ github.ref_name }}" - fallback_branch="${SBCLI_BRANCH}" - if git ls-remote --heads https://github.com/simplyblock-io/sbcli.git "$wf_branch" | grep -q "$wf_branch"; then - git clone --branch "$wf_branch" --single-branch https://github.com/simplyblock-io/sbcli.git sbcli - else - git clone --branch "$fallback_branch" --single-branch https://github.com/simplyblock-io/sbcli.git sbcli - fi - test -f sbcli/e2e/e2e.py - - - name: Install Python deps - shell: bash - run: | - set -euxo pipefail - python3 -m pip install --upgrade pip - [[ -f "sbcli/e2e/requirements.txt" ]] && pip install -r sbcli/e2e/requirements.txt || true - - - name: Cleanup logs before e2e - shell: bash - working-directory: sbcli/e2e - run: python3 logs/cleanup.py - - - name: Set RUN_DIR_FILE - shell: bash - run: echo "RUN_DIR_FILE=/tmp/sb_k8s_e2e_run_dir_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}.txt" >> "$GITHUB_ENV" - - - name: Record test start time - shell: bash - run: | - echo "TEST_START_EPOCH=$(date +%s)" >> "$GITHUB_ENV" - echo "TEST_START_HUMAN=$(date -u +'%Y-%m-%d %H:%M:%S UTC')" >> "$GITHUB_ENV" - - - name: Run E2E tests - shell: bash - working-directory: sbcli/e2e - run: | - set -euxo pipefail - testname_args=() - if [[ -n "${TEST_CLASS:-}" ]]; then - testname_args+=( --testname "${TEST_CLASS}" ) - fi - ha_args=() - if [[ "${{ inputs.RUN_HA }}" == "true" ]]; then - ha_args+=( --run_ha True ) - fi - python3 -u e2e.py \ - --ndcs "${NDCS}" \ - --npcs "${NPCS}" \ - --run_k8s True \ - "${testname_args[@]}" \ - "${ha_args[@]}" \ - 2>&1 | tee output.log - - - name: Copy raw output.log to each test's NFS folder - if: always() - shell: bash - run: | - set +e - out_log="sbcli/e2e/output.log" - if [ ! -f "${out_log}" ]; then - echo "No output.log found, skipping" - exit 0 - fi - echo "output.log size: $(stat --printf='%s' "${out_log}" 2>/dev/null || echo '?') bytes" - - mapfile -t log_paths < <( - grep 'Logs Path:' "${out_log}" 2>/dev/null \ - | sed 's/\x1b\[[0-9;]*m//g' \ - | sed 's/.*Logs Path: *//' \ - | sed 's/[[:space:]]*$//' || true - ) - - if [ ${#log_paths[@]} -eq 0 ]; then - echo "No 'Logs Path:' entries found in output.log" - if [ -n "${RUN_BASE_DIR:-}" ] && [ -d "${RUN_BASE_DIR}" ]; then - cp "${out_log}" "${RUN_BASE_DIR}/github_raw_output.log" || true - echo "Saved to ${RUN_BASE_DIR}/github_raw_output.log" - fi - exit 0 - fi - - for path in "${log_paths[@]}"; do - if [ -d "${path}" ]; then - cp "${out_log}" "${path}/github_raw_output.log" || true - echo "Saved output.log -> ${path}/github_raw_output.log" - else - echo "WARN: path does not exist: ${path}" - fi - done - - # ============================================================ - # POST-TEST - # ============================================================ - - name: Post-test cleanup - if: always() - shell: bash - run: | - run_remote() { - ssh -i "${KEY_PATH}" -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null "${SSH_USER}@$1" \ - "bash -s" <<< "$2" || true - } - for ip in $CLIENTNODES; do - run_remote "$ip" "pkill -9 fio || true; pkill -9 tmux || true" - done - - - name: Export RUN_BASE_DIR from RUN_DIR_FILE - if: always() - shell: bash - run: | - test -f "${RUN_DIR_FILE}" || (echo "RUN_DIR_FILE not found — skipping"; exit 0) - RUN_BASE_DIR="$(cat "${RUN_DIR_FILE}" | tr -d '\r\n')" - [[ -n "${RUN_BASE_DIR}" ]] && echo "RUN_BASE_DIR=${RUN_BASE_DIR}" >> "$GITHUB_ENV" || true - - - name: Mark test end time - if: always() - shell: bash - run: | - echo "TEST_END_EPOCH=$(date +%s)" >> "$GITHUB_ENV" - echo "TEST_END_HUMAN=$(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> "$GITHUB_ENV" - - - name: Write Job Summary - if: always() - shell: bash - run: | - out_log="sbcli/e2e/output.log" - start="${TEST_START_EPOCH:-0}"; end="${TEST_END_EPOCH:-0}" - dur_sec=0; [[ "$end" -ge "$start" && "$start" -gt 0 ]] && dur_sec=$((end-start)) - dur_fmt="${dur_sec}s ($(( dur_sec/3600 ))h $(( (dur_sec%3600)/60 ))m)" - failure_summary="(unknown)" - if [[ -f "${out_log}" ]]; then - failure_summary="$(grep -oE 'MultipleExceptions: .+' "${out_log}" | tail -1 | sed 's/\x1b\[[0-9;]*m//g' || true)" - [[ -z "${failure_summary}" ]] && failure_summary="$(grep -E 'RuntimeError:|AssertionError:' "${out_log}" | tail -1 | sed 's/\x1b\[[0-9;]*m//g' || true)" - [[ -z "${failure_summary}" ]] && failure_summary="(no exception found)" - fi - total_cases="$(grep -E 'Total Cases:' "${out_log}" 2>/dev/null | tail -1 | grep -oE '[0-9]+$' || echo '?')" - passed_cnt="$(grep -E 'Passed:' "${out_log}" 2>/dev/null | tail -1 | grep -oE '[0-9]+$' || echo '?')" - failed_cnt="$(grep -E 'Failed:' "${out_log}" 2>/dev/null | tail -1 | grep -oE '[0-9]+$' || echo '?')" - conclusion="SUCCESS"; [[ "${{ job.status }}" != "success" ]] && conclusion="FAILED" - { - echo "## SimplyBlock K8s E2E Run Summary" - echo "**Result:** ${conclusion}" - echo "### Run Info" - echo "- **Test class:** \`${TEST_CLASS:-all}\`" - echo "- **K3s master:** \`${K3S_MNODES}\`" - echo "- **CLUSTER_ID:** \`${CLUSTER_ID}\`" - echo "- **NDCS/NPCS:** \`${NDCS}/${NPCS}\`" - echo "- **Start (UTC):** ${TEST_START_HUMAN:-unknown}" - echo "- **Duration:** ${dur_fmt}" - echo "### Test Results" - echo "- **Total:** ${total_cases} | **Passed:** ${passed_cnt} | **Failed:** ${failed_cnt}" - echo "### Failure Reason" - echo '```'; printf '%s\n' "${failure_summary}"; echo '```' - echo "" - echo "> **Note:** Graylog/OpenSearch logs are currently being collected and will be available in the NFS log directory shortly." - } >> "$GITHUB_STEP_SUMMARY" - - - name: Send Slack Notification - if: always() - shell: bash - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - JOB_STATUS: ${{ job.status }} - SLACK_RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - GITHUB_REF_NAME: ${{ github.ref_name }} - SLACK_WF_NAME: "K8s E2E (Bootstrap)" - run: | - python3 - <<'PYEOF' - import json, os, re, sys, urllib.request - webhook = os.environ.get("SLACK_WEBHOOK_URL","") - if not webhook: print("No SLACK_WEBHOOK_URL"); sys.exit(0) - out_log = "sbcli/e2e/output.log" - total = passed = failed = 0 - if os.path.isfile(out_log): - content = open(out_log).read() - def px(pat): - m = re.search(pat, content); return int(m.group(1)) if m else 0 - total = px(r'Total Cases:\s*(\d+)') - passed = px(r'Passed:\s*(\d+)') - failed = px(r'Failed:\s*(\d+)') - s = int(os.environ.get("TEST_START_EPOCH","0") or "0") - e = int(os.environ.get("TEST_END_EPOCH","0") or "0") - secs = max(0, e-s) if e>=s>0 else 0 - dur = f"{secs//3600}h {(secs%3600)//60}m {secs%60}s" - ok = os.environ.get("JOB_STATUS","") == "success" - icon = ":white_check_mark:" if ok else ":x:" - mention = "" if ok else " " - lines = [ - f"{icon} *SimplyBlock {os.environ.get('SLACK_WF_NAME','Run')}*{mention}", - f"*Status:* {'SUCCESS' if ok else 'FAILURE'} | *Duration:* {dur}", - f"*Branch:* `{os.environ.get('GITHUB_REF_NAME','?')}` | *K3s:* `{os.environ.get('K3S_MNODES','?')}` | *Test:* `{os.environ.get('TEST_CLASS','all') or 'all'}`", - "", - ] - if total > 0: - lines += [f":white_check_mark: *Passed:* {passed}/{total}", f":x: *Failed:* {failed}"] - lines.append(f":link: *Run:* <{os.environ.get('SLACK_RUN_URL','')}|View on GitHub>") - lines.append(f":hourglass_flowing_sand: _Logs are being collected and will be available shortly._") - req = urllib.request.Request(webhook, data=json.dumps({"text":"\n".join(lines)}).encode(), - headers={"Content-Type":"application/json"}) - try: - urllib.request.urlopen(req, timeout=15); print("Slack sent.") - except Exception as exc: - print(f"WARN: Slack failed: {exc}", file=sys.stderr) - PYEOF - - - - name: Collect Graylog/OpenSearch logs - if: '!cancelled()' - timeout-minutes: 480 - shell: bash - run: | - set +e - NAMESPACE="${K8S_NAMESPACE:-simplyblock}" - [ -z "${TEST_START_EPOCH:-}" ] || [ -z "${TEST_END_EPOCH:-}" ] && exit 0 - ELAPSED=$((TEST_END_EPOCH - TEST_START_EPOCH)) - [ "${ELAPSED}" -le 0 ] && exit 0 - - WINDOW_START=$((TEST_START_EPOCH - 3600)) - WINDOW_END=$((TEST_END_EPOCH + 3600)) - - ADMIN_POD="" - for i in $(seq 1 12); do - ADMIN_POD=$(kubectl -n ${NAMESPACE} get pods -l app=simplyblock-admin-control \ - -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true - if [ -n "${ADMIN_POD}" ]; then - PHASE=$(kubectl -n ${NAMESPACE} get pod "${ADMIN_POD}" -o jsonpath='{.status.phase}' 2>/dev/null) || true - [ "${PHASE}" = "Running" ] && break; ADMIN_POD="" - fi - sleep 10 - done - [ -z "${ADMIN_POD}" ] && echo "No admin pod found, skipping Graylog collection" && exit 0 - - # Deploy updated collect_logs.py to admin pod (use /tmp since package dir is read-only) - BUILTIN_SCRIPT="/usr/local/lib/python3.12/site-packages/simplyblock_core/scripts/collect_logs.py" - DEPLOYED_SCRIPT="/tmp/collect_logs.py" - SCRIPT_SRC="sbcli/scripts/collect_logs.py" - if [ -f "${SCRIPT_SRC}" ]; then - kubectl -n ${NAMESPACE} cp "${SCRIPT_SRC}" "${ADMIN_POD}:${DEPLOYED_SCRIPT}" 2>/dev/null || \ - echo "WARN: failed to deploy updated collect_logs.py, using built-in version" - fi - # Determine which script to use - COLLECT_SCRIPT="${BUILTIN_SCRIPT}" - if kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- test -f "${DEPLOYED_SCRIPT}" 2>/dev/null; then - COLLECT_SCRIPT="${DEPLOYED_SCRIPT}" - echo " Using deployed script: ${COLLECT_SCRIPT}" - else - echo " Using built-in script: ${COLLECT_SCRIPT}" - fi - - MGMT_IP=$(kubectl get svc -n ${NAMESPACE} | grep graylog | awk '{print $3}') - OPENSEARCH_IP=$(kubectl get svc opensearch-cluster-master -n ${NAMESPACE} -o jsonpath='{.spec.clusterIP}' 2>/dev/null || echo "") - OUTPUT_DIR="" - if [ -n "${RUN_BASE_DIR:-}" ] && [ -d "${RUN_BASE_DIR}" ]; then - OUTPUT_DIR="${RUN_BASE_DIR}/graylog_collected" - else - OUTPUT_DIR="${NFS_MOUNTPOINT:-/mnt/nfs_share}/graylog_collected-$(date -u '+%Y%m%d-%H%M%S')" - fi - mkdir -p "${OUTPUT_DIR}" 2>/dev/null || true - - epoch_to_iso() { - python3 -c "from datetime import datetime,timezone; print(datetime.fromtimestamp($1,tz=timezone.utc).strftime('%Y-%m-%dT%H:%M:%S'))" - } - - # Build chunk boundaries, then iterate in REVERSE order (newest first) - _CHUNK_STARTS=() - _C=${WINDOW_START} - while [ ${_C} -lt ${WINDOW_END} ]; do - _CHUNK_STARTS+=(${_C}) - _C=$((_C + 3600)) - done - NUM_CHUNKS=${#_CHUNK_STARTS[@]} - - run_collect() { - local iso=$1 mins=$2 outdir=$3 extra_flag=${4:-} - local mgmt_ip_to_use="${MGMT_IP}" - if [[ "${extra_flag}" == *"--use-opensearch"* ]] && [ -n "${OPENSEARCH_IP}" ]; then - mgmt_ip_to_use="${OPENSEARCH_IP}" - fi - kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- \ - python3 "${COLLECT_SCRIPT}" \ - "${iso}" "${mins}" \ - --mode kubernetes --namespace "${NAMESPACE}" \ - --output-dir "${outdir}" \ - ${extra_flag} \ - ${mgmt_ip_to_use:+--mgmt-ip "${mgmt_ip_to_use}"} \ - ${CLUSTER_ID:+--cluster-id "${CLUSTER_ID}"} \ - 2>&1 - local rc=$? - [ $rc -ne 0 ] && return $rc - local has_content - has_content=$(kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- \ - find "${outdir}" -name "*.tar.gz" -size +0 2>/dev/null | head -1) - if [ -z "${has_content}" ]; then - echo " WARN: collect_logs.py succeeded but no .tar.gz output found" - return 1 - fi - } - - collect_adaptive() { - local start_epoch=$1 end_epoch=$2 outdir=$3 extra_flag=${4:-} - local duration=$((end_epoch - start_epoch)) - local mins=$(( (duration + 59) / 60 )) - local iso=$(epoch_to_iso ${start_epoch}) - if run_collect "${iso}" "${mins}" "${outdir}" "${extra_flag}"; then - return 0 - fi - echo " WARN: ${mins}m window failed, retrying with 5-min sub-windows..." - local sub_start=${start_epoch} - while [ ${sub_start} -lt ${end_epoch} ]; do - local sub_end=$((sub_start + 300)) - [ ${sub_end} -gt ${end_epoch} ] && sub_end=${end_epoch} - local sub_mins=$(( ((sub_end - sub_start) + 59) / 60 )) - local sub_iso=$(epoch_to_iso ${sub_start}) - if ! run_collect "${sub_iso}" "${sub_mins}" "${outdir}" "${extra_flag}"; then - echo " WARN: 5-min window at ${sub_iso} failed, retrying with 1-min windows..." - local micro_start=${sub_start} - while [ ${micro_start} -lt ${sub_end} ]; do - local micro_end=$((micro_start + 60)) - [ ${micro_end} -gt ${sub_end} ] && micro_end=${sub_end} - local micro_mins=$(( ((micro_end - micro_start) + 59) / 60 )) - local micro_iso=$(epoch_to_iso ${micro_start}) - run_collect "${micro_iso}" "${micro_mins}" "${outdir}" "${extra_flag}" || \ - echo " WARN: 1-min window at ${micro_iso} also failed" - micro_start=${micro_end} - done - fi - sub_start=${sub_end} - done - } - - PREFER_OPENSEARCH="" - CHUNK=0 - for (( _IDX=NUM_CHUNKS-1; _IDX>=0; _IDX-- )); do - CHUNK=$((CHUNK + 1)) - CHUNK_START=${_CHUNK_STARTS[$_IDX]} - CHUNK_END=$((CHUNK_START + 3600)) - [ ${CHUNK_END} -gt ${WINDOW_END} ] && CHUNK_END=${WINDOW_END} - CHUNK_MINUTES=$(( ((CHUNK_END - CHUNK_START) + 59) / 60 )) - CHUNK_ISO=$(epoch_to_iso ${CHUNK_START}) - echo "--- Chunk ${CHUNK}/${NUM_CHUNKS}: ${CHUNK_ISO} for ${CHUNK_MINUTES}m (newest-first) ---" - POD_OUTPUT_DIR="/tmp/graylog_collect_chunk${CHUNK}" - kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- mkdir -p "${POD_OUTPUT_DIR}" 2>/dev/null || true - if [ -z "${PREFER_OPENSEARCH}" ]; then - echo " Probing OpenSearch availability..." - if run_collect "${CHUNK_ISO}" "${CHUNK_MINUTES}" "${POD_OUTPUT_DIR}" "--use-opensearch"; then - PREFER_OPENSEARCH=true - echo " OpenSearch works — using it for all chunks" - else - PREFER_OPENSEARCH=false - echo " OpenSearch unavailable — using Graylog for all chunks" - kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- rm -rf "${POD_OUTPUT_DIR}" 2>/dev/null || true - kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- mkdir -p "${POD_OUTPUT_DIR}" 2>/dev/null || true - collect_adaptive ${CHUNK_START} ${CHUNK_END} "${POD_OUTPUT_DIR}" || \ - echo "WARN: Graylog also failed for chunk ${CHUNK}" - fi - elif [ "${PREFER_OPENSEARCH}" = "true" ]; then - collect_adaptive ${CHUNK_START} ${CHUNK_END} "${POD_OUTPUT_DIR}" "--use-opensearch" || { - echo "WARN: OpenSearch failed, falling back to Graylog..." - collect_adaptive ${CHUNK_START} ${CHUNK_END} "${POD_OUTPUT_DIR}" || true - } - else - collect_adaptive ${CHUNK_START} ${CHUNK_END} "${POD_OUTPUT_DIR}" || { - echo "WARN: Graylog failed, falling back to OpenSearch..." - collect_adaptive ${CHUNK_START} ${CHUNK_END} "${POD_OUTPUT_DIR}" "--use-opensearch" || true - } - fi - TARBALLS=$(kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- \ - find "${POD_OUTPUT_DIR}" -name "*.tar.gz" -type f 2>/dev/null) || true - if [ -n "${TARBALLS}" ]; then - for TB in ${TARBALLS}; do - kubectl -n ${NAMESPACE} cp "${ADMIN_POD}:${TB}" "${OUTPUT_DIR}/$(basename ${TB})" 2>&1 || true - done - for TB_FILE in "${OUTPUT_DIR}"/*.tar.gz; do - [ -f "${TB_FILE}" ] && tar -xzf "${TB_FILE}" -C "${OUTPUT_DIR}/" 2>/dev/null || true - done - fi - kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- rm -rf "${POD_OUTPUT_DIR}" 2>/dev/null || true - done - echo "=== Log collection complete (${CHUNK} chunks, newest-first): ${OUTPUT_DIR} ===" - - - name: Collect mgmt snapshots via kubectl exec - if: always() - shell: bash - run: | - admin_pod="$(kubectl get pods -n "${K8S_NAMESPACE}" --no-headers \ - -o custom-columns=:metadata.name | grep simplyblock-admin-control | head -1 || true)" - [[ -z "${admin_pod}" ]] && echo "No admin pod; skipping." && exit 0 - run_base="${RUN_BASE_DIR:-/tmp/k8s_e2e_$(date +%s)}" - outdir="${run_base}/mgmt_details/mgmt"; mkdir -p "${outdir}" - for cmd in "cluster list" "pool list" "lvol list" "snapshot list" "sn list"; do - kubectl exec -n "${K8S_NAMESPACE}" "${admin_pod}" -- \ - bash -c "${SBCLI_CMD} ${cmd}" > "${outdir}/${cmd// /_}.txt" 2>&1 || true - done - - - name: Collect K8s pod logs - if: always() - shell: bash - run: | - run_base="${RUN_BASE_DIR:-/tmp/k8s_e2e_$(date +%s)}" - pod_log_dir="${run_base}/k8s_pod_logs_final"; mkdir -p "${pod_log_dir}" - pods="$(kubectl get pods -n "${K8S_NAMESPACE}" --no-headers \ - -o custom-columns=:metadata.name 2>/dev/null || true)" - for pod in $pods; do - kubectl logs -n "${K8S_NAMESPACE}" "${pod}" \ - --all-containers=true --timestamps=true \ - > "${pod_log_dir}/${pod}.log" 2>&1 || true - done - - name: Upload logs (always) - if: always() - uses: actions/upload-artifact@v4 - with: - name: simplyblock-k8s-e2e-logs-${{ github.run_id }} - path: | - simplyBlockDeploy/bare-metal/bootstrap-k3s.log - sbcli-charts/simplyblock_core/scripts/charts/helm-install.log - sbcli/e2e/output.log - sbcli/e2e/logs/** - if-no-files-found: warn diff --git a/.github/workflows/e2e-bootstrap.yml b/.github/workflows/e2e-bootstrap.yml index b00c9fd43b..338ae739d9 100644 --- a/.github/workflows/e2e-bootstrap.yml +++ b/.github/workflows/e2e-bootstrap.yml @@ -21,7 +21,7 @@ on: default: "http://192.168.10.210/grafana" SBCLI_BRANCH: type: string - default: "R25.10-Hotfix" + default: "main" CUSTOM_IMAGES: type: string default: 'spdk="simplyblock/spdk:main-latest" docker="simplyblock/simplyblock:main"' @@ -119,7 +119,7 @@ on: SBCLI_BRANCH: description: "sbcli repo branch to clone for tests" required: true - default: "R25.10-Hotfix" + default: "main" CUSTOM_IMAGES: description: "Image overrides: set spdk and/or docker values, leave as \"\" to skip." @@ -259,7 +259,7 @@ jobs: EXTRA_SN_ARGS: ${{ inputs.EXTRA_SN_ARGS || '' }} GRAFANA_ENDPOINT: ${{ inputs.GRAFANA_ENDPOINT || 'http://192.168.10.210/grafana' }} SBCLI_CMD: "sbctl" - SBCLI_BRANCH: ${{ inputs.SBCLI_BRANCH || 'R25.10-Hotfix' }} + SBCLI_BRANCH: ${{ inputs.SBCLI_BRANCH || 'main' }} # SSH/client env SSH_USER: ${{ inputs.SSH_USER || 'root' }} @@ -844,6 +844,35 @@ jobs: echo "TEST_START_EPOCH=$(date +%s)" >> "$GITHUB_ENV" echo "TEST_START_HUMAN=$(date -u +'%Y-%m-%d %H:%M:%S UTC')" >> "$GITHUB_ENV" + - name: Start MinIO trace logging + if: ${{ env.CLUSTER_SECURITY == 'backup' }} + shell: bash + run: | + set -euxo pipefail + + # 1. Install mc (MinIO Client) + if ! command -v mc &>/dev/null; then + if curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc 2>/dev/null; then + chmod +x /usr/local/bin/mc + else + curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc + chmod +x /tmp/mc + export PATH="/tmp:$PATH" + echo "/tmp" >> "$GITHUB_PATH" + fi + fi + + # 2. Configure mc alias to external MinIO + mc alias set myminio http://192.168.10.164:9000 "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" + + # 3. Start admin trace in background + MINIO_TRACE_LOG="/tmp/minio-trace-${GITHUB_RUN_ID}.log" + echo "MINIO_TRACE_LOG=${MINIO_TRACE_LOG}" >> "$GITHUB_ENV" + nohup mc admin trace -v --all myminio > "${MINIO_TRACE_LOG}" 2>&1 & + echo "MC_TRACE_PID=$!" >> "$GITHUB_ENV" + + echo "=== MinIO trace logging started (PID=$!, log=${MINIO_TRACE_LOG}) ===" + - name: Run e2e tests shell: bash working-directory: sbcli/e2e @@ -1754,6 +1783,25 @@ jobs: ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "rm -rf '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true done echo "=== Log collection complete (${CHUNK} chunks, newest-first): ${OUTPUT_DIR} ===" + - name: Stop MinIO trace logging and save + if: always() + shell: bash + run: | + set +e + if [ -n "${MC_TRACE_PID:-}" ]; then + echo "Stopping MinIO trace (PID=${MC_TRACE_PID})..." + kill "${MC_TRACE_PID}" 2>/dev/null || true + wait "${MC_TRACE_PID}" 2>/dev/null || true + fi + if [ -f "${MINIO_TRACE_LOG:-}" ]; then + LINES=$(wc -l < "${MINIO_TRACE_LOG}") + SIZE=$(du -h "${MINIO_TRACE_LOG}" | cut -f1) + echo "=== MinIO trace log: ${LINES} lines, ${SIZE} ===" + cp "${MINIO_TRACE_LOG}" sbcli/e2e/minio-trace.log 2>/dev/null || true + else + echo "No MinIO trace log found (backup tests may not have run)" + fi + - name: Upload logs (always) if: always() uses: actions/upload-artifact@v4 @@ -1762,6 +1810,7 @@ jobs: path: | simplyBlockDeploy/bare-metal/bootstrap.log sbcli/e2e/output.log + sbcli/e2e/minio-trace.log sbcli/e2e/logs/** if-no-files-found: warn diff --git a/.github/workflows/e2e-only.yml b/.github/workflows/e2e-only.yml index 1b0b743a97..11e5012d30 100755 --- a/.github/workflows/e2e-only.yml +++ b/.github/workflows/e2e-only.yml @@ -30,7 +30,7 @@ on: SBCLI_BRANCH: description: "sbcli repo branch to clone for tests" required: true - default: "R25.10-Hotfix" + default: "main" CLUSTER_ID: description: "Cluster ID to run tests on" diff --git a/.github/workflows/e2e-scheduler.yml b/.github/workflows/e2e-scheduler.yml index 1062f8f2b1..54e54009e0 100644 --- a/.github/workflows/e2e-scheduler.yml +++ b/.github/workflows/e2e-scheduler.yml @@ -1,10 +1,10 @@ name: E2E Scheduler # Runs 4 e2e-bootstrap variants daily (sequentially) unless a stress run is active. -# Run 1: R25.10-Hotfix ndcs=1 npcs=1 storage: .201-.203 -# Run 2: R25.10-Hotfix ndcs=2 npcs=2 storage: .201-.204 -# Run 3: main ndcs=1 npcs=1 storage: .201-.203 -# Run 4: main ndcs=2 npcs=2 storage: .201-.204 +# Run 1: main ndcs=1 npcs=1 storage: .201-.203 +# Run 2: main ndcs=2 npcs=2 storage: .201-.204 +# Run 3: main ndcs=1 npcs=1 storage: .201-.203 +# Run 4: main ndcs=2 npcs=2 storage: .201-.204 # # Infrastructure defaults not listed here are defined in e2e-bootstrap.yml. @@ -84,16 +84,16 @@ jobs: fi # ============================================================ - # RUN 1 — R25.10-Hotfix | ndcs=1 npcs=1 | 3 storage nodes + # RUN 1 — main | ndcs=1 npcs=1 | 3 storage nodes # ============================================================ e2e-run-1: - name: "E2E R25.10-Hotfix 1+1" + name: "E2E main 1+1" needs: check-stress if: needs.check-stress.outputs.stress_running != 'true' uses: ./.github/workflows/e2e-bootstrap.yml secrets: inherit with: - SBCLI_BRANCH: "R25.10-Hotfix" + SBCLI_BRANCH: "main" STORAGE_PRIVATE_IPS: "192.168.10.201 192.168.10.202 192.168.10.203" MNODES: "192.168.10.210" API_INVOKE_URL: "http://192.168.10.210/" @@ -106,16 +106,16 @@ jobs: RUN_LABEL: "run1" # ============================================================ - # RUN 2 — R25.10-Hotfix | ndcs=2 npcs=2 | 4 storage nodes + # RUN 2 — main | ndcs=2 npcs=2 | 4 storage nodes # ============================================================ e2e-run-2: - name: "E2E R25.10-Hotfix 2+2" + name: "E2E main 2+2" needs: [check-stress, e2e-run-1] if: always() && !cancelled() && needs.check-stress.outputs.stress_running != 'true' && needs.e2e-run-1.result != 'skipped' uses: ./.github/workflows/e2e-bootstrap.yml secrets: inherit with: - SBCLI_BRANCH: "R25.10-Hotfix" + SBCLI_BRANCH: "main" STORAGE_PRIVATE_IPS: "192.168.10.201 192.168.10.202 192.168.10.203 192.168.10.204" MNODES: "192.168.10.210" API_INVOKE_URL: "http://192.168.10.210/" diff --git a/.github/workflows/k8s-e2e.yaml b/.github/workflows/k8s-e2e.yaml deleted file mode 100755 index bfca553d94..0000000000 --- a/.github/workflows/k8s-e2e.yaml +++ /dev/null @@ -1,489 +0,0 @@ -name: E2E K8s Tests - -on: - push: - branches: - - pre-release - - sbcli - schedule: - - cron: '0 9 * * *' # Runs every day at 9 AM UTC - workflow_dispatch: - inputs: - simplyBlockDeploy_branch: - description: 'Branch for simplyBlockDeploy' - required: true - default: 'main' - operator_repo_branch: - description: 'Branch for simplyblock-operator repo' - required: true - default: 'main' - testname: - description: 'Name of test to run. Empty to run all' - required: false - default: '' - send_slack_notification: - description: 'Send Slack notification?' - required: false - default: true - type: boolean - ndcs: - description: 'Number of data chunks' - required: false - default: 2 - npcs: - description: 'Number of parity chunks' - required: false - default: 1 - bs: - description: 'Block size' - required: false - default: 4096 - chunk_bs: - description: 'Chunk block size' - required: false - default: 4096 - -jobs: - e2e: - runs-on: self-hosted - concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - steps: - - name: Fix workspace permissions - run: sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE" 2>/dev/null || true - - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set send_slack_notification for scheduled and push events - if: github.event_name == 'schedule' || github.event_name == 'push' - run: echo "send_slack_notification=true" >> $GITHUB_ENV - - - name: Set send_slack_notification for manual workflow_dispatch - if: github.event_name == 'workflow_dispatch' - run: echo "send_slack_notification=${{ github.event.inputs.send_slack_notification }}" >> $GITHUB_ENV - - - uses: actions/setup-go@v5 - with: - go-version: '1.22' - - run: go version - - - name: Install Helm v3.15.4 - run: | - curl -L https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz -o helm-v3.15.4-linux-amd64.tar.gz - tar -zxvf helm-v3.15.4-linux-amd64.tar.gz - sudo mv linux-amd64/helm /usr/local/bin/helm - helm version - - - name: Install kubectl v1.31.0 - run: | - curl -LO "https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl" - chmod +x kubectl - sudo mv kubectl /usr/local/bin/ - kubectl version --client - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - uses: actions/checkout@master - name: Checkout code simplyBlockDeploy - with: - repository: simplyblock-io/simplyBlockDeploy - ref: refs/heads/${{ github.event.inputs.simplyBlockDeploy_branch || 'main'}} - path: 'simplyBlockDeploy' - token: ${{ secrets.GH_ACCESS_KEY_ID_RAUNAK }} - - - uses: actions/checkout@master - name: Checkout simplyblock-operator - with: - repository: simplyblock/simplyblock-operator - ref: refs/heads/${{ github.event.inputs.operator_repo_branch || 'main'}} - path: 'simplyblock-operator' - token: ${{ secrets.GH_ACCESS_KEY_ID_RAUNAK }} - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_wrapper: false - - - name: Initialize Terraform - run: | - cd $GITHUB_WORKSPACE/simplyBlockDeploy - export TFSTATE_BUCKET=simplyblock-terraform-state-bucket - export TFSTATE_KEY=csi - export TFSTATE_REGION=us-east-2 - export TFSTATE_DYNAMODB_TABLE=terraform-up-and-running-locks - - terraform init -reconfigure \ - -backend-config="bucket=${TFSTATE_BUCKET}" \ - -backend-config="key=${TFSTATE_KEY}" \ - -backend-config="region=${TFSTATE_REGION}" \ - -backend-config="dynamodb_table=${TFSTATE_DYNAMODB_TABLE}" \ - -backend-config="encrypt=true" - - - name: Select or create workspace - run: | - cd $GITHUB_WORKSPACE/simplyBlockDeploy - terraform workspace select -or-create ghiaction-sbclie2ek8s - - - name: Validate Terraform Configuration - run: | - cd $GITHUB_WORKSPACE/simplyBlockDeploy - terraform validate - - - name: Set parameters based on branch and input - run: | - echo "SBCLI_CMD=sbctl" >> $GITHUB_ENV - echo "NDCS=${{ github.event.inputs.ndcs || 2 }}" >> $GITHUB_ENV - echo "NPCS=${{ github.event.inputs.npcs || 1 }}" >> $GITHUB_ENV - echo "BS=${{ github.event.inputs.bs || 4096 }}" >> $GITHUB_ENV - echo "CHUNK_BS=${{ github.event.inputs.chunk_bs || 4096 }}" >> $GITHUB_ENV - shell: bash - - - name: Plan Terraform Changes - run: | - cd $GITHUB_WORKSPACE/simplyBlockDeploy - terraform plan \ - -var "mgmt_nodes=1" -var "storage_nodes=4" -var "volumes_per_storage_nodes=1" \ - -var storage_nodes_instance_type="m6g.2xlarge" \ - -var "storage_nodes_arch=arm64" -var "snode_deploy_on_k8s=true" \ - -var "sec_storage_nodes_instance_type=m6g.2xlarge" \ - -var "extra_nodes=1" -var "extra_nodes_instance_type=m6gd.xlarge" \ - -var storage_nodes_ebs_size2=100 -var "region=us-east-2" \ - -var "extra_nodes_arch=arm64" -var "sec_storage_nodes=1" \ - -var "sbcli_cmd=$SBCLI_CMD" -out=tfplan - env: - SBCLI_CMD: sbctl - - - name: Apply Terraform Changes - run: | - cd $GITHUB_WORKSPACE/simplyBlockDeploy - terraform apply tfplan - - - name: Get Terraform Outputs - id: terraform_outputs - run: | - cd $GITHUB_WORKSPACE/simplyBlockDeploy - output_bastion_public_ip=$(terraform output -raw bastion_public_ip) - echo "::set-output name=bastion_public_ip::$output_bastion_public_ip" - output_key_name=$(terraform output -raw key_name) - echo "::set-output name=key_name::$output_key_name" - output_extra_node_public_ip=$(terraform output -raw extra_nodes_public_ips) - echo "::set-output name=extra_node_public_ip::$output_extra_node_public_ip" - output_mgmt_private_ip=$(terraform output -raw mgmt_private_ips) - echo "::set-output name=mgmt_private_ip::$output_mgmt_private_ip" - - - name: Bootstrap Cluster - run: | - cd $GITHUB_WORKSPACE/simplyBlockDeploy - ./bootstrap-cluster.sh --sbcli-cmd "$SBCLI_CMD" \ - --max-lvol 10 --max-snap 10 --max-prov 400G --number-of-devices 1 \ - --distr-ndcs $NDCS \ - --distr-npcs $NPCS \ - --distr-bs $BS \ - --distr-chunk-bs $CHUNK_BS \ - --k8s-snode --ha-type ha - id: bootstrap_cluster - env: - SBCLI_CMD: sbctl - NDCS: ${{ env.NDCS }} - NPCS: ${{ env.NPCS }} - BS: ${{ env.BS }} - CHUNK_BS: ${{ env.CHUNK_BS }} - - - name: Bootstrap k3s - run: | - cd $GITHUB_WORKSPACE/simplyBlockDeploy - ./bootstrap-k3s.sh --k8s-snode - id: bootstrap_k3s - - - name: Configure Kubeconfig - run: | - mkdir -p $HOME/.kube - KUBECONFIG_FILE="$HOME/.kube/config-${{ github.run_id }}" - scp -i "$HOME/.ssh/${{ steps.terraform_outputs.outputs.key_name }}" ec2-user@${{ steps.terraform_outputs.outputs.extra_node_public_ip }}:/etc/rancher/k3s/k3s.yaml "$KUBECONFIG_FILE" - sed -i "s/server: https:\/\/\(localhost\|127.0.0.1\)\(:[0-9]*\)/server: https:\/\/${{ steps.terraform_outputs.outputs.extra_node_public_ip }}\2/" "$KUBECONFIG_FILE" - echo "KUBECONFIG=$KUBECONFIG_FILE" >> $GITHUB_ENV - - - name: Install Helm Chart for simplyblock-operator - run: | - cd $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator/ - echo "Sleeping for 30 seconds before helm install" - helm install spdk-csi ./ \ - --namespace spdk-csi \ - --create-namespace \ - --set csiConfig.simplybk.uuid="${{ steps.bootstrap_cluster.outputs.cluster_id }}" \ - --set csiConfig.simplybk.ip="${{ steps.bootstrap_cluster.outputs.cluster_api_gateway_endpoint }}" \ - --set csiSecret.simplybk.secret="${{ steps.bootstrap_cluster.outputs.cluster_secret }}" \ - --set logicalVolume.pool_name="testing1" \ - --set logicalVolume.snapshot="True" \ - --set image.spdkcsi.tag=latest \ - --set image.simplyblock.tag=main \ - --set logicalVolume.distr_ndcs="${NDCS}" \ - --set logicalVolume.distr_npcs="${NPCS}" \ - --set storagenode.create=true \ - --set logicalVolume.encryption=true - env: - NDCS: ${{ env.NDCS }} - NPCS: ${{ env.NPCS }} - - - - name: Check Cluster Status - run: | - CLUSTER_API_GATEWAY_ENDPOINT=${{ steps.bootstrap_cluster.outputs.cluster_api_gateway_endpoint }} - CLUSTER_UUID=${{ steps.bootstrap_cluster.outputs.cluster_id }} - CLUSTER_SECRET=${{ steps.bootstrap_cluster.outputs.cluster_secret }} - BASTION_IP=${{ steps.terraform_outputs.outputs.bastion_public_ip }} - MGMT_IP=${{ steps.terraform_outputs.outputs.mgmt_private_ip }} - KEY_NAME=${{ steps.terraform_outputs.outputs.key_name }} - - n=0 - until [ "$n" -ge 60 ] - do - response=$(curl -s -X GET "$CLUSTER_API_GATEWAY_ENDPOINT/cluster/$CLUSTER_UUID" \ - -H "Content-Type: application/json" \ - -H "Authorization: $CLUSTER_UUID $CLUSTER_SECRET") - - status=$(echo $response | jq -r '.results[0].status') - - if [ "$status" != "active" ]; then - echo "Cluster status is not active, current status: $status, retrying" - n=$((n+1)) - sleep 10 - else - echo "Cluster status is active" - exit 0 - fi - done - - echo "Cluster is not active after polling, attempting forced activation..." - ssh -i "$HOME/.ssh/${KEY_NAME}" \ - -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - -o ProxyCommand="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i \"$HOME/.ssh/${KEY_NAME}\" -W %h:%p ec2-user@${BASTION_IP}" \ - ec2-user@${MGMT_IP} "sbctl -d cluster activate ${CLUSTER_UUID}" || true - - echo "Waiting 30s for activation to take effect..." - sleep 30 - - response=$(curl -s -X GET "$CLUSTER_API_GATEWAY_ENDPOINT/cluster/$CLUSTER_UUID" \ - -H "Content-Type: application/json" \ - -H "Authorization: $CLUSTER_UUID $CLUSTER_SECRET") - status=$(echo $response | jq -r '.results[0].status') - - if [ "$status" = "active" ]; then - echo "Cluster is now active after forced activation" - exit 0 - fi - - echo "ERROR: Cluster is still not active after forced activation, status: $status" - exit 1 - - - name: Record Test Start Time - run: echo "TEST_START_TIME=$(date +%s)" >> $GITHUB_ENV - - - name: Setup Tests & Run Tests - timeout-minutes: 120 - run: | - cd $GITHUB_WORKSPACE/e2e - sudo apt-get install -y python3.12-venv - python3 -m venv myenv - source myenv/bin/activate - python3 -m pip install -r requirements.txt - echo "Running tests in namespace ${{ steps.get-namespace.outputs.namespace }}" - export CLUSTER_ID=${{ steps.bootstrap_cluster.outputs.cluster_id }} - export CLUSTER_SECRET=${{ steps.bootstrap_cluster.outputs.cluster_secret }} - export CLUSTER_IP=${{ steps.bootstrap_cluster.outputs.cluster_ip }} - export API_BASE_URL=${{ steps.bootstrap_cluster.outputs.cluster_api_gateway_endpoint }} - export BASTION_SERVER=${{ steps.terraform_outputs.outputs.bastion_public_ip }} - export KEY_NAME=${{ steps.terraform_outputs.outputs.key_name }} - export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} - export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} - export AWS_REGION=${{ secrets.AWS_REGION }} - export SBCLI_CMD=${SBCLI_CMD} - export SUPABASE_ANON_KEY=${{ secrets.SUPABASE_ANON_KEY }} - TESTNAME="" - if [ -n "${{ github.event.inputs.testname }}" ]; then - TESTNAME="--testname ${{ github.event.inputs.testname }}" - fi - - python3 e2e.py $TESTNAME \ - --ndcs $NDCS --npcs $NPCS --bs $BS --chunk_bs $CHUNK_BS \ - --run_k8s True - env: - SBCLI_CMD: sbctl - NDCS: ${{ env.NDCS }} - NPCS: ${{ env.NPCS }} - BS: ${{ env.BS }} - CHUNK_BS: ${{ env.CHUNK_BS }} - - - name: Record Test End Time - if: always() - run: echo "TEST_END_TIME=$(date +%s)" >> $GITHUB_ENV - - - name: Calculate Total Time Taken - if: always() - run: | - TEST_TIME=$(($TEST_END_TIME - $TEST_START_TIME)) - TEST_TIME_HOURS=$(($TEST_TIME / 3600)) - TEST_TIME_MINS=$((($TEST_TIME % 3600) / 60)) - TEST_TIME_SECS=$(($TEST_TIME % 60)) - echo "Test runtime: ${TEST_TIME_HOURS}h ${TEST_TIME_MINS}m ${TEST_TIME_SECS}s" - echo "TEST_TIME_HOURS=$TEST_TIME_HOURS" >> $GITHUB_ENV - echo "TEST_TIME_MINS=$TEST_TIME_MINS" >> $GITHUB_ENV - echo "TEST_TIME_SECS=$TEST_TIME_SECS" >> $GITHUB_ENV - - - name: Parse test results - if: always() && (github.event_name == 'schedule' || env.send_slack_notification == 'true') - id: parse_results - run: | - cd $GITHUB_WORKSPACE/e2e/logs - echo "Looking for the latest non-empty log file..." - # Find the latest non-empty log file - COUNTER=0 - MAX_ATTEMPTS=10 - while [ $COUNTER -lt $MAX_ATTEMPTS ]; do - LATEST_LOG=$(ls -t *.log | head -n 1) - if [ -s "$LATEST_LOG" ]; then - echo "Found non-empty log file: $LATEST_LOG" - break - fi - echo "Attempt $((COUNTER + 1)): No non-empty log file found. Retrying..." - COUNTER=$((COUNTER + 1)) - sleep 1 # Add a small delay to avoid rapid looping - done - if [ ! -s "$LATEST_LOG" ]; then - echo "No non-empty log file found after $MAX_ATTEMPTS attempts" - exit 1 - fi - echo "Parsing the identified log file: $LATEST_LOG" - # Parse the identified log file - echo "Total tests" - TOTAL_TESTS=$(grep -i "Number of Total Cases" "$LATEST_LOG" | awk '{print $NF}') - echo "number Passed tests" - PASSED_TESTS=$(grep -i "Number of Passed Cases" "$LATEST_LOG" | awk '{print $NF}') - echo "number Failed tests" - FAILED_TESTS=$(grep -i "Number of Failed Cases" "$LATEST_LOG" | awk '{print $NF}') - echo "List Passed tests" - PASSED_CASES=$(grep "PASSED CASE" "$LATEST_LOG" | awk -F 'INFO - | FAILED CASE' '{print $2}') - echo "List Failed tests" - FAILED_CASES=$(grep "FAILED CASE" "$LATEST_LOG" | awk -F 'INFO - | FAILED CASE' '{print $2}') - # Format passed and failed cases as bullet points - echo "Adding PASSED cases with bullets: $PASSED_CASES" - echo "Adding FAILED cases with bullets: $PASSED_CASES" - PASSED_CASES_BULLETS=$(echo "$PASSED_CASES" | awk '{printf " • %s\n", $0}') - FAILED_CASES_BULLETS=$(echo "$FAILED_CASES" | awk '{printf " • %s\n", $0}') - echo "PASSED cases with bullets: $PASSED_CASES_BULLETS" - echo "FAILED cases with bullets: $FAILED_CASES_BULLETS" - echo "TOTAL_TESTS=${TOTAL_TESTS}" - echo "PASSED_TESTS=${PASSED_TESTS}" - echo "FAILED_TESTS=${FAILED_TESTS}" - echo "PASSED_CASES=${PASSED_CASES}" - echo "FAILED_CASES=${FAILED_CASES}" - echo "PASSED_TESTS=${PASSED_TESTS}" >> $GITHUB_ENV - echo "FAILED_TESTS=${FAILED_TESTS}" >> $GITHUB_ENV - echo "TOTAL_TESTS=${TOTAL_TESTS}" >> $GITHUB_ENV - echo "PASSED_CASES<> $GITHUB_ENV - echo "${PASSED_CASES}" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - echo "FAILED_CASES<> $GITHUB_ENV - echo "${FAILED_CASES}" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - echo "PASSED_CASES_BULLETS<> $GITHUB_ENV - echo "${PASSED_CASES_BULLETS}" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - echo "FAILED_CASES_BULLETS<> $GITHUB_ENV - echo "${FAILED_CASES_BULLETS}" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - - name: Write Job Summary - if: always() - run: | - { - echo "## K8s E2E Run Summary" - echo "" - if [[ "${{ job.status }}" == "success" ]]; then - echo "**Result:** SUCCESS" - else - echo "**Result:** FAILED" - fi - echo "" - echo "### Test Results" - echo "- **Total:** ${TOTAL_TESTS:-?} | **Passed:** ${PASSED_TESTS:-?} | **Failed:** ${FAILED_TESTS:-?}" - echo "" - echo "### Run Info" - echo "- **Branch:** \`${{ github.ref_name }}\`" - echo "- **Duration:** ${TEST_TIME_HOURS:-0}h ${TEST_TIME_MINS:-0}m ${TEST_TIME_SECS:-0}s" - echo "" - echo "> **Note:** Graylog/OpenSearch logs are currently being collected and will be available in the NFS log directory shortly." - } >> "$GITHUB_STEP_SUMMARY" - - - name: Send Slack Notification - if: always() && (github.event_name == 'schedule' || env.send_slack_notification == 'true') - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_REPOSITORY: ${{ github.repository }} - S3_BUCKET_NAME: "simplyblock-e2e-test-logs" - RUN_ID: ${{ github.run_id }} - PASSED_TESTS: ${{ env.PASSED_TESTS }} - FAILED_TESTS: ${{ env.FAILED_TESTS }} - TOTAL_TESTS: ${{ env.TOTAL_TESTS }} - PASSED_CASES: ${{ env.PASSED_CASES }} - FAILED_CASES: ${{ env.FAILED_CASES }} - PASSED_CASES_BULLETS: ${{ env.PASSED_CASES_BULLETS }} - FAILED_CASES_BULLETS: ${{ env.FAILED_CASES_BULLETS }} - BRANCH_NAME: ${{ github.ref_name }} - TEST_TIME_HOURS: ${{ env.TEST_TIME_HOURS }} - TEST_TIME_MINS: ${{ env.TEST_TIME_MINS }} - TEST_TIME_SECS: ${{ env.TEST_TIME_SECS }} - NDCS: ${{ env.NDCS }} - NPCS: ${{ env.NPCS }} - CHUNK_BS: ${{ env.CHUNK_BS }} - BS: ${{ env.BS }} - run: | - GITHUB_RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}" - AWS_LOGS_URL="https://s3.console.aws.amazon.com/s3/buckets/${S3_BUCKET_NAME}?prefix=${RUN_ID}/®ion=us-east-2" - if [[ ${{ job.status }} == 'success' ]]; then - OVERALL_STATUS=":white_check_mark: Overall Status: SUCCESS" - else - OVERALL_STATUS=":x: Overall Status: FAILURE " - fi - - TIME_TAKEN="${TEST_TIME_HOURS}h ${TEST_TIME_MINS}m ${TEST_TIME_SECS}s" - - MESSAGE="Python E2E *K8s* tests run triggered on branch *${BRANCH_NAME}*. \nTotal Time Taken to run the tests: ${TIME_TAKEN}. \n\n${OVERALL_STATUS}\nGitHub Run: ${GITHUB_RUN_URL}\nAWS Logs: ${AWS_LOGS_URL}\n\n*Configuration*: *NDCS: ${NDCS}, NPCS: ${NPCS}, Block Size: ${BS}, Chunk Block Size: ${CHUNK_BS}*\n\nTotal Tests: *${TOTAL_TESTS}*, Passed Tests: *${PASSED_TESTS}*, Failed Tests: *${FAILED_TESTS}*\n\n:hourglass_flowing_sand: _Logs are being collected and will be available shortly._\n\n-- Test Cases Passed :white_check_mark:\n${PASSED_CASES_BULLETS}\n\n-- Test Cases Failed :x:\n${FAILED_CASES_BULLETS}" - - curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"${MESSAGE}\"}" $SLACK_WEBHOOK_URL - - - - name: Upload automation and docker logs to s3 - run: | - cd $GITHUB_WORKSPACE/e2e/logs - ./upload_logs.sh - cd $GITHUB_WORKSPACE/simplyBlockDeploy - ./upload_docker_logs_to_s3.sh --k8s --namespace "spdk-csi" - if: always() - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} - S3_BUCKET_NAME: "simplyblock-e2e-test-logs" - RUN_ID: ${{ github.run_id }} - - name: Destroy Cluster - if: always() - run: | - cd $GITHUB_WORKSPACE/simplyBlockDeploy - terraform destroy --auto-approve - - - name: Cleanup build folder - run: | - ls -la ./ - rm -rf ./* || true - rm -rf ./.??* || true - ls -la ./ - rm -f "$HOME/.kube/config-${{ github.run_id }}" diff --git a/.github/workflows/k8s-native-cross-cluster-restore.yaml b/.github/workflows/k8s-native-cross-cluster-restore.yaml new file mode 100644 index 0000000000..40d614aefb --- /dev/null +++ b/.github/workflows/k8s-native-cross-cluster-restore.yaml @@ -0,0 +1,1135 @@ +name: K8s Native Cross-Cluster Restore Test +run-name: "K8s Cross-Cluster Restore | ${{ inputs.operator_repo_branch || 'main' }} | ${{ inputs.cluster_environment || 'openshift-baremetal' }} | ${{ inputs.cluster_params || 'ndcs=1,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4' }}" + +# ============================================================================ +# Cross-cluster restore test — deploys TWO simplyblock clusters in separate +# K8s namespaces (simplyblock + simplyblock-c2) and runs +# TestBackupCrossClusterRestore in K8s mode. +# +# Architecture: +# - Same K8s cluster (OpenShift / K3s) +# - Worker nodes split: first half → C1, second half → C2 +# - Shared MinIO in minio namespace +# - Shared operator Helm chart (one install, watches both namespaces) +# - Backup credentials created in both namespaces +# - StorageCluster + Pool + StorageNodeSet CRDs in each namespace +# ============================================================================ + +on: + workflow_dispatch: + inputs: + operator_repo_branch: + description: 'Branch for simplyblock-operator repo' + required: true + default: 'main' + simplyblock_image: + description: 'Simplyblock image tag' + required: true + default: 'main' + simplyblock_repository: + description: 'Simplyblock image repository' + required: false + default: 'public.ecr.aws/simply-block/simplyblock' + operator_repository: + description: 'Operator image repository' + required: false + default: 'simplyblock/simplyblock-operator' + operator_tag: + description: 'Operator image tag' + required: false + default: 'main' + csi_repository: + description: 'CSI driver image repository (leave empty for chart default)' + required: false + default: 'simplyblock/spdkcsi' + csi_tag: + description: 'CSI driver image tag (leave empty for chart default)' + required: false + default: 'latest' + spdk_image: + description: 'SPDK container image' + required: true + default: 'simplyblock/spdk:main-latest' + worker_nodes: + description: 'Comma-separated K8s worker node names (split evenly: first half→C1, second half→C2)' + required: true + default: 'worker-0.ocp.simplyblock.ai,worker-1.ocp.simplyblock.ai,worker-2.ocp.simplyblock.ai,worker-3.ocp.simplyblock.ai,worker-4.ocp.simplyblock.ai,worker-5.ocp.simplyblock.ai' + ifc_names: + description: 'Network interfaces (mgmt_ifc:data_nics)' + required: false + default: 'br-ex:enp2s0f0' + max_lvol: + description: 'Max logical volume count per storage node' + required: false + default: '30' + ssh_user: + description: 'SSH user for client nodes' + required: false + default: 'root' + key_path: + description: "SSH private key path on runner" + required: true + default: "/home/ec2-user/.ssh/simplyblock-us-east-2.pem" + client_ips: + description: 'Space-separated client IPs for FIO' + required: false + default: '' + nfs_mountpoint: + description: 'NFS mountpoint on client nodes' + required: false + default: '/mnt/nfs_share/' + cluster_params: + description: 'Cluster config: ndcs=1,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4' + required: false + default: 'ndcs=1,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4' + cluster_environment: + description: 'Target cluster environment' + required: true + default: 'openshift-baremetal' + type: choice + options: + - local + - aws-openshift + - openshift-local + - openshift-baremetal + - gcp + skip_nfs: + description: 'Skip NFS mounting' + required: false + default: 'false' + type: choice + options: + - 'false' + - 'true' + tls_enabled: + description: 'Enable TLS' + required: false + default: true + type: boolean + send_slack_notification: + description: 'Send Slack notification on completion' + required: false + default: true + type: boolean + sbcli_branch: + description: 'sbcli repo branch to clone for tests' + required: false + default: 'main' + +jobs: + cross-cluster-restore: + runs-on: ${{ github.event.inputs.cluster_environment == 'aws-openshift' && 'vm-runner-43' || 'self-hosted' }} + timeout-minutes: 4320 + concurrency: + group: k8s-cross-cluster-${{ github.event.inputs.cluster_environment || 'local' }} + cancel-in-progress: false + env: + KEY_PATH: ${{ github.event.inputs.key_path || '/home/ec2-user/.ssh/simplyblock-us-east-2.pem' }} + CLIENT_IPS: ${{ github.event.inputs.client_ips || '' }} + NFS_MOUNTPOINT: ${{ github.event.inputs.nfs_mountpoint || '/mnt/nfs_share/' }} + SSH_USER: ${{ github.event.inputs.ssh_user || 'root' }} + SSH_PASSWORD: ${{ secrets.SSH_PASSWORD }} + NS_C1: simplyblock + NS_C2: simplyblock-c2 + + steps: + - name: Fix workspace permissions + run: sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE" 2>/dev/null || true + + - name: Checkout code + uses: actions/checkout@v4 + + - uses: actions/checkout@master + name: Checkout simplyblock-operator + with: + repository: simplyblock/simplyblock-operator + ref: ${{ github.event.inputs.operator_repo_branch || 'main' }} + path: 'simplyblock-operator' + + # ── Resolve SSH key ────────────────────────────────────────────────── + + - name: Resolve KEY_PATH and validate key exists + shell: bash + run: | + set -euxo pipefail + kp="${KEY_PATH}" + kp="${kp%\"}"; kp="${kp#\"}" + kp="${kp%\'}"; kp="${kp#\'}" + if [[ "$kp" == .ssh/* ]]; then kp="${HOME}/${kp}"; fi + if [[ "$kp" == ~/* ]]; then kp="${HOME}/${kp#~/}"; fi + echo "KEY_PATH=$kp" >> "$GITHUB_ENV" + test -f "$kp" || (echo "ERROR: SSH key not found at $kp" && exit 1) + chmod 600 "$kp" || true + + - name: Export KEY_NAME from KEY_PATH + shell: bash + run: | + key_name="$(basename "${KEY_PATH}")" + echo "KEY_NAME=${key_name}" >> "$GITHUB_ENV" + + # ── Validate and split worker nodes ───────────────────────────────── + + - name: Validate worker node count and split between clusters + id: split + shell: bash + run: | + set -euo pipefail + + PARAMS="${{ github.event.inputs.cluster_params || 'ndcs=1,npcs=2' }}" + _ndcs=1; _npcs=2 + for pair in $(echo "$PARAMS" | tr ',' ' '); do + key=$(echo "$pair" | cut -d= -f1) + val=$(echo "$pair" | cut -d= -f2) + case "$key" in + ndcs) _ndcs="$val" ;; + npcs) _npcs="$val" ;; + esac + done + + IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" + TOTAL=${#NODES[@]} + MIN_PER_CLUSTER=$((_ndcs + _npcs)) + MIN_TOTAL=$((MIN_PER_CLUSTER * 2)) + + echo "=== Cross-Cluster Node Validation ===" + echo "Total worker nodes: ${TOTAL}" + echo "NDCS=${_ndcs}, NPCS=${_npcs}, min per cluster: ${MIN_PER_CLUSTER}" + + if [[ ${TOTAL} -lt ${MIN_TOTAL} ]]; then + echo "ERROR: Need at least ${MIN_TOTAL} nodes for 2 clusters, got ${TOTAL}" + exit 1 + fi + + HALF=$((TOTAL / 2)) + C1_CSV="" + C2_CSV="" + for i in "${!NODES[@]}"; do + if [[ $i -lt $HALF ]]; then + C1_CSV="${C1_CSV:+${C1_CSV},}${NODES[$i]}" + else + C2_CSV="${C2_CSV:+${C2_CSV},}${NODES[$i]}" + fi + done + + echo "" + echo "Cluster-1 workers: ${C1_CSV}" + echo "Cluster-2 workers: ${C2_CSV}" + + echo "C1_WORKERS=${C1_CSV}" >> "$GITHUB_ENV" + echo "C2_WORKERS=${C2_CSV}" >> "$GITHUB_ENV" + echo "ALL_WORKERS=${{ github.event.inputs.worker_nodes }}" >> "$GITHUB_ENV" + + # ── Set parameters ────────────────────────────────────────────────── + + - name: Set cluster parameters + run: | + PARAMS="${{ github.event.inputs.cluster_params || 'ndcs=1,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4' }}" + _ndcs="" _npcs="" _bs="" _chunk_bs="" _partitions="" _jm_count="" + for pair in $(echo "$PARAMS" | tr ',' ' '); do + key=$(echo "$pair" | cut -d= -f1) + val=$(echo "$pair" | cut -d= -f2) + case "$key" in + ndcs) _ndcs="$val" ;; + npcs) _npcs="$val" ;; + bs) _bs="$val" ;; + chunk_bs) _chunk_bs="$val" ;; + partitions) _partitions="$val" ;; + jm_count) _jm_count="$val" ;; + esac + done + echo "NDCS=${_ndcs:-1}" >> $GITHUB_ENV + echo "NPCS=${_npcs:-2}" >> $GITHUB_ENV + echo "BS=${_bs:-4096}" >> $GITHUB_ENV + echo "CHUNK_BS=${_chunk_bs:-4096}" >> $GITHUB_ENV + echo "PARTITIONS=${_partitions:-1}" >> $GITHUB_ENV + echo "JM_COUNT=${_jm_count:-4}" >> $GITHUB_ENV + + # ── Setup kubeconfig + tools ──────────────────────────────────────── + + - name: Setup kubeconfig + run: | + mkdir -p $HOME/.kube + KUBECONFIG_FILE="$HOME/.kube/config-${{ github.run_id }}" + echo "${KUBECONFIG_DATA}" > "$KUBECONFIG_FILE" + chmod 600 "$KUBECONFIG_FILE" + echo "KUBECONFIG=$KUBECONFIG_FILE" >> $GITHUB_ENV + env: + KUBECONFIG_DATA: ${{ + github.event.inputs.cluster_environment == 'aws-openshift' && secrets.KUBECONFIG_AWS_OPENSHIFT || + github.event.inputs.cluster_environment == 'openshift-local' && secrets.KUBECONFIG_OPENSHIFT_LOCAL || + github.event.inputs.cluster_environment == 'openshift-baremetal' && secrets.KUBECONFIG_OPENSHIFT_BM || + github.event.inputs.cluster_environment == 'gcp' && secrets.KUBECONFIG_GCP || + secrets.KUBECONFIG_LOCAL || secrets.KUBECONFIG_CONTENT + }} + + - name: Install Helm v3.15.4 + run: | + if ! helm version 2>/dev/null | grep -q 'v3.15'; then + curl -L https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz -o helm-v3.15.4-linux-amd64.tar.gz + tar -zxvf helm-v3.15.4-linux-amd64.tar.gz + sudo mv linux-amd64/helm /usr/local/bin/helm + rm -rf linux-amd64 helm-v3.15.4-linux-amd64.tar.gz + fi + helm version + + - name: Install kubectl v1.31.0 + run: | + if ! kubectl version --client 2>/dev/null | grep -q 'v1.31'; then + curl -LO "https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl" + chmod +x kubectl + sudo mv kubectl /usr/local/bin/ + fi + kubectl version --client + + - name: Verify kubectl connectivity + run: kubectl get nodes + + # ── Pre-clean clients ─────────────────────────────────────────────── + + - name: Pre-clean FIO/NVMe on clients + shell: bash + run: | + set -euxo pipefail + run_remote() { + sshpass -p "${SSH_PASSWORD}" ssh \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "${SSH_USER}@${1}" "bash -s" <<< "$2" + } + normalized="$(echo "${CLIENT_IPS}" | tr ',' ' ' | xargs -n1 | sort -u | xargs || true)" + if [ -z "${normalized}" ]; then exit 0; fi + for ip in ${normalized}; do + echo "---- ${ip}: kill fio + disconnect NVMe + umount NFS ----" + run_remote "${ip}" "set -euxo pipefail; + pkill -9 fio || true; + for mp in /mnt/test_location/*; do + [ -e \"\$mp\" ] || continue; + timeout 10 umount -f \"\$mp\" 2>/dev/null || umount -l \"\$mp\" 2>/dev/null || true; + done; + rm -rf /mnt/test_location/* 2>/dev/null || true; + for nqn in \$(nvme list-subsys -o json 2>/dev/null | jq -r '.Subsystems[]?.NQN // empty' | grep lvol || true); do + nvme disconnect -n \"\$nqn\" || true; + done; + mp='${NFS_MOUNTPOINT}'; + timeout 10 umount -f \"\$mp\" 2>/dev/null || umount -l \"\$mp\" 2>/dev/null || true" || true + done + + # ── Cleanup old deployments (BOTH namespaces) ─────────────────────── + + - name: Cleanup old CSI deployment (both namespaces) + run: | + set +e + for NAMESPACE in ${NS_C1} ${NS_C2}; do + echo "=== Cleaning namespace: ${NAMESPACE} ===" + if [ -x "$GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh" ]; then + bash $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh -f $NAMESPACE || true + fi + bash $GITHUB_WORKSPACE/e2e/scripts/cleanup_k8s.sh $NAMESPACE + done + + echo "=== Delete Released PVs ===" + for pv in $(kubectl get pv --no-headers 2>/dev/null | grep 'simplyblock/' | awk '{print $1}'); do + kubectl delete pv "$pv" --ignore-not-found 2>/dev/null || true + done + + echo "=== Delete CRDs ===" + for crd in $(kubectl get crd -o name 2>/dev/null | grep simplyblock); do + crd_name="${crd#customresourcedefinition.apiextensions.k8s.io/}" + for cr in $(kubectl get "$crd_name" -A --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null); do + IFS=' ' read -r ns name <<< "$cr" + kubectl patch "$crd_name" "$name" -n "$ns" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + done + kubectl patch crd "$crd_name" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + done + kubectl delete -f $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator/crds/ --ignore-not-found --timeout=60s 2>/dev/null || true + for crd in $(kubectl get crd -o name 2>/dev/null | grep simplyblock); do + crd_name="${crd#customresourcedefinition.apiextensions.k8s.io/}" + kubectl patch crd "$crd_name" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + kubectl delete crd "$crd_name" --ignore-not-found --timeout=30s 2>/dev/null || true + done + + echo "=== Reset hugepages on ALL worker nodes ===" + CLUSTER_ENV="${{ github.event.inputs.cluster_environment || 'local' }}" + IFS=',' read -ra NODES <<< "${ALL_WORKERS}" + for NODE in "${NODES[@]}"; do + echo "Resetting hugepages on $NODE..." + if [[ "$CLUSTER_ENV" == *"openshift"* ]]; then + oc debug node/"$NODE" -- chroot /host bash -c \ + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + oc debug node/"$NODE" -- chroot /host systemctl restart kubelet 2>/dev/null || true + else + kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ + "systemctl restart kubelet" 2>/dev/null || true + fi + done + echo "=== Cleanup complete ===" + + - name: Cleanup old cert-manager + run: | + set +e + helm uninstall cert-manager -n cert-manager 2>/dev/null || true + kubectl delete namespace cert-manager --wait=false 2>/dev/null || true + kubectl wait --for=delete namespace/cert-manager --timeout=120s 2>/dev/null || true + kubectl delete crd certificaterequests.cert-manager.io certificates.cert-manager.io \ + challenges.acme.cert-manager.io clusterissuers.cert-manager.io \ + issuers.cert-manager.io orders.acme.cert-manager.io --ignore-not-found 2>/dev/null || true + + # ── Remove stale labels from ALL workers ──────────────────────────── + + - name: Remove stale storagenodeset labels + run: | + IFS=',' read -ra NODES <<< "${ALL_WORKERS}" + for NODE in "${NODES[@]}"; do + kubectl label node "$NODE" io.simplyblock.storagenodeset- 2>/dev/null || true + done + + # ── Label worker nodes ────────────────────────────────────────────── + + - name: Label all worker nodes + run: | + CLUSTER_ENV="${{ github.event.inputs.cluster_environment || 'local' }}" + IFS=',' read -ra NODES <<< "${ALL_WORKERS}" + for NODE in "${NODES[@]}"; do + if [ "$CLUSTER_ENV" = "local" ] || [ "$CLUSTER_ENV" = "openshift-local" ]; then + kubectl label node "$NODE" topology.kubernetes.io/zone=default --overwrite + fi + kubectl label node "$NODE" simplyblock.io/role=mgmt-plane --overwrite + done + + # ── Create BOTH namespaces ────────────────────────────────────────── + + - name: Create namespaces and set pod-security labels + run: | + for NS in ${NS_C1} ${NS_C2} minio; do + kubectl create namespace ${NS} --dry-run=client -o yaml | kubectl apply -f - + kubectl label namespace ${NS} \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/warn=privileged \ + pod-security.kubernetes.io/audit=privileged \ + --overwrite + done + + - name: Create Docker registry secrets in both namespaces + run: | + for NS in ${NS_C1} ${NS_C2}; do + kubectl create secret docker-registry regcred \ + --docker-server=https://index.docker.io/v1/ \ + --docker-username=${DOCKER_USER} \ + --docker-password=${DOCKER_PASSWORD} \ + --docker-email=a@b.com \ + -n ${NS} || true + done + env: + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + # ── Deploy MinIO (shared) ─────────────────────────────────────────── + + - name: Deploy MinIO for backup tests (shared by both clusters) + run: | + set -euxo pipefail + echo "=== Setting up MinIO ===" + + cat <<'MINIO_EOF' | kubectl apply -f - + apiVersion: apps/v1 + kind: Deployment + metadata: + name: minio + namespace: minio + spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio + command: ["/bin/sh", "-c", "minio server /data --console-address :9001"] + env: + - name: MINIO_ROOT_USER + value: minioadmin + - name: MINIO_ROOT_PASSWORD + value: minioadmin123 + ports: + - containerPort: 9000 + - containerPort: 9001 + MINIO_EOF + + kubectl -n minio expose deploy/minio --port 9000 \ + --dry-run=client -o yaml | kubectl apply -f - + + echo "Waiting for MinIO pod..." + for i in $(seq 1 30); do + READY=$(kubectl -n minio get pods --no-headers 2>/dev/null | grep -c "Running" || true) + if [ "$READY" -ge 1 ]; then echo "MinIO is running"; break; fi + echo "MinIO not ready ($i/30)..."; sleep 10 + done + + # Create backup-credentials in BOTH namespaces + for NS in ${NS_C1} ${NS_C2}; do + cat <> "$GITHUB_ENV" + sleep 3 + mc alias set myminio http://localhost:9000 minioadmin minioadmin123 2>/dev/null || true + MINIO_TRACE_LOG="/tmp/minio-trace-${{ github.run_id }}.log" + echo "MINIO_TRACE_LOG=${MINIO_TRACE_LOG}" >> "$GITHUB_ENV" + nohup mc admin trace -v --all myminio > "${MINIO_TRACE_LOG}" 2>&1 & + echo "MC_TRACE_PID=$!" >> "$GITHUB_ENV" + + # ── Install cert-manager (TLS) ───────────────────────────────────── + + - name: Install cert-manager + if: ${{ github.event.inputs.tls_enabled == 'true' }} + run: | + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm upgrade --install cert-manager jetstack/cert-manager \ + --namespace cert-manager --create-namespace \ + --version v1.13.0 --set installCRDs=true + kubectl wait --for=condition=Ready pods --all -n cert-manager --timeout=120s + + # ── Install Helm Chart (operator) in C1 namespace ─────────────────── + + - name: Install Helm Chart for simplyblock-operator (C1 namespace) + run: | + set -euxo pipefail + cd $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator/ + + TLS_FLAGS="" + if [ "${{ github.event.inputs.tls_enabled }}" = "true" ]; then + TLS_FLAGS="--set tls.enabled=true --set tls.mutual_enabled=true" + fi + + CSI_FLAGS="" + if [ -n "${{ github.event.inputs.csi_repository }}" ]; then + CSI_FLAGS="$CSI_FLAGS --set image.csi.repository=${{ github.event.inputs.csi_repository }}" + fi + if [ -n "${{ github.event.inputs.csi_tag }}" ]; then + CSI_FLAGS="$CSI_FLAGS --set image.csi.tag=${{ github.event.inputs.csi_tag }}" + fi + + helm upgrade --install spdk-csi ./ \ + --namespace ${NS_C1} \ + --create-namespace \ + --debug \ + --timeout 10m \ + --set controlplane.enabled=true \ + --set operator.enabled=true \ + --set controlplane.csiHostpathDriver.enabled=true \ + --set controlplane.storageclass.name=local-hostpath \ + --set image.simplyblock.repository="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" \ + --set image.simplyblock.tag="${{ github.event.inputs.simplyblock_image }}" \ + --set image.operator.repository="${{ github.event.inputs.operator_repository || 'simplyblock/simplyblock-operator' }}" \ + --set image.operator.tag="${{ github.event.inputs.operator_tag || 'main' }}" \ + --set csiConfig.simplybk.ip="http://simplyblock-webappapi.${NS_C1}:5000" \ + --set prometheus.server.persistentVolume.storageClass=local-hostpath \ + --set controlplane.observability.enabled=true \ + --set opensearch.persistence.storageClass=local-hostpath \ + --set controlplane.observability.minio.storageClass=local-hostpath \ + $TLS_FLAGS $CSI_FLAGS + + # ── Install Helm Chart (operator) in C2 namespace ─────────────────── + + - name: Install Helm Chart for simplyblock-operator (C2 namespace) + run: | + set -euxo pipefail + cd $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator/ + + TLS_FLAGS="" + if [ "${{ github.event.inputs.tls_enabled }}" = "true" ]; then + TLS_FLAGS="--set tls.enabled=true --set tls.mutual_enabled=true" + fi + + CSI_FLAGS="" + if [ -n "${{ github.event.inputs.csi_repository }}" ]; then + CSI_FLAGS="$CSI_FLAGS --set image.csi.repository=${{ github.event.inputs.csi_repository }}" + fi + if [ -n "${{ github.event.inputs.csi_tag }}" ]; then + CSI_FLAGS="$CSI_FLAGS --set image.csi.tag=${{ github.event.inputs.csi_tag }}" + fi + + helm upgrade --install spdk-csi-c2 ./ \ + --namespace ${NS_C2} \ + --create-namespace \ + --debug \ + --timeout 10m \ + --set controlplane.enabled=true \ + --set operator.enabled=true \ + --set controlplane.csiHostpathDriver.enabled=true \ + --set controlplane.storageclass.name=local-hostpath-c2 \ + --set image.simplyblock.repository="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" \ + --set image.simplyblock.tag="${{ github.event.inputs.simplyblock_image }}" \ + --set image.operator.repository="${{ github.event.inputs.operator_repository || 'simplyblock/simplyblock-operator' }}" \ + --set image.operator.tag="${{ github.event.inputs.operator_tag || 'main' }}" \ + --set csiConfig.simplybk.ip="http://simplyblock-webappapi.${NS_C2}:5000" \ + --set prometheus.server.persistentVolume.storageClass=local-hostpath \ + --set controlplane.observability.enabled=true \ + --set opensearch.persistence.storageClass=local-hostpath \ + --set controlplane.observability.minio.storageClass=local-hostpath \ + $TLS_FLAGS $CSI_FLAGS + + # ── Patch service accounts + imagePullSecrets in BOTH namespaces ──── + + - name: Patch service accounts with imagePullSecrets (both namespaces) + run: | + for NS in ${NS_C1} ${NS_C2}; do + echo "=== Patching SAs in ${NS} ===" + for sa in $(kubectl get serviceaccounts -n ${NS} --no-headers 2>/dev/null | awk '{print $1}'); do + kubectl patch serviceaccount "$sa" -n ${NS} \ + --patch '{"imagePullSecrets": [{"name": "regcred"}]}' || true + done + for deploy in $(kubectl get deployments -n ${NS} --no-headers -o custom-columns=:metadata.name 2>/dev/null); do + kubectl patch deployment "$deploy" -n ${NS} \ + --type=merge -p '{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"regcred"}]}}}}' || true + done + done + + # ── Wait for operator pods in BOTH namespaces ─────────────────────── + + - name: Wait for operator pods to be ready + run: | + for NS in ${NS_C1} ${NS_C2}; do + echo "=== Waiting for operator in ${NS} ===" + for i in $(seq 1 60); do + POD=$(kubectl -n ${NS} get pods \ + -l app=simplyblock-admin-control \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true + if [ -n "$POD" ]; then + PHASE=$(kubectl -n ${NS} get pod "$POD" \ + -o jsonpath='{.status.phase}' 2>/dev/null) || true + if [ "$PHASE" = "Running" ]; then + echo "Operator pod $POD is Running in ${NS}" + break + fi + fi + echo "Waiting ($i/60)..." + sleep 10 + done + done + + # ── Apply StorageCluster + Pool + StorageNodeSet CRDs ─────────────── + + - name: Apply CRDs for Cluster-1 (C1) + run: | + IFC_NAMES="${{ github.event.inputs.ifc_names || 'br-ex:enp2s0f0' }}" + MGMT_IFC="${IFC_NAMES%%:*}" + DATA_NICS="${IFC_NAMES#*:}" + MAX_LVOL="${{ github.event.inputs.max_lvol || '30' }}" + SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" + SB_TAG="${{ github.event.inputs.simplyblock_image }}" + SPDK_IMAGE="${{ github.event.inputs.spdk_image }}" + OPENSHIFT_CLUSTER="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" + CORE_PERCENTAGE="${{ github.event.inputs.cluster_environment == 'openshift-baremetal' && '50' || github.event.inputs.cluster_environment == 'local' && '35' || '65' }}" + SKIP_KUBELET_CONFIG="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'false' || 'true' }}" + FORCE_FORMAT_4K="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" + DRIVE_SIZE_RANGE="${{ github.event.inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + + # Build C1 worker nodes YAML + WORKER_YAML="" + IFS=',' read -ra C1_NODES <<< "${C1_WORKERS}" + for NODE in "${C1_NODES[@]}"; do + WORKER_YAML="${WORKER_YAML} - ${NODE}"$'\n' + done + + BACKUP_SPEC=" backup: + credentialsSecretRef: + name: backup-credentials + localEndpoint: http://minio.minio.svc.cluster.local:9000 + localTesting: true + secondaryTarget: 0 + snapshotBackups: true + withCompression: false" + + echo "=== Applying C1 CRDs in ${NS_C1} ===" + cat </dev/null | awk '{print $1}'); do + kubectl patch serviceaccount "$sa" -n ${NS} \ + --patch '{"imagePullSecrets": [{"name": "regcred"}]}' || true + done + for deploy in $(kubectl get deployments -n ${NS} --no-headers -o custom-columns=:metadata.name 2>/dev/null); do + kubectl patch deployment "$deploy" -n ${NS} \ + --type=merge -p '{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"regcred"}]}}}}' || true + done + done + + # ── Wait for storage daemonsets + patch ───────────────────────────── + + - name: Wait for storage daemonsets and restart (both namespaces) + run: | + for NS in ${NS_C1} ${NS_C2}; do + echo "=== Waiting for storage daemonset in ${NS} ===" + DS_NAME="simplyblock-storage-node-ds-simplyblock-cluster" + for i in $(seq 1 60); do + if kubectl get daemonset ${DS_NAME} -n ${NS} &>/dev/null; then + echo "Daemonset found in ${NS}, waiting for rollout..." + kubectl rollout status daemonset ${DS_NAME} -n ${NS} --timeout=300s || true + kubectl rollout restart -n ${NS} ds ${DS_NAME} + break + fi + echo "DS not found in ${NS} ($i/60)..." + sleep 10 + done + + # Patch SAs for storage node SA + for sa in $(kubectl get serviceaccounts -n ${NS} --no-headers 2>/dev/null | awk '{print $1}'); do + kubectl patch serviceaccount "$sa" -n ${NS} \ + --patch '{"imagePullSecrets": [{"name": "regcred"}]}' || true + done + done + + # ── Wait for BOTH clusters to become active ───────────────────────── + + - name: Wait for Cluster-1 to become active + timeout-minutes: 80 + run: | + NAMESPACE=${NS_C1} + MAX_POLL=300 + echo "=== Waiting for C1 cluster active in ${NAMESPACE} ===" + + ADMIN_POD="" + for i in $(seq 1 60); do + ADMIN_POD=$(kubectl -n $NAMESPACE get pods \ + -l app=simplyblock-admin-control \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true + if [ -n "$ADMIN_POD" ]; then + PHASE=$(kubectl -n $NAMESPACE get pod "$ADMIN_POD" -o jsonpath='{.status.phase}' 2>/dev/null) || true + if [ "$PHASE" = "Running" ]; then break; fi + fi + sleep 10 + done + + IFS=',' read -ra C1_NODES <<< "${C1_WORKERS}" + EXPECTED_SNODES=${#C1_NODES[@]} + for i in $(seq 1 $MAX_POLL); do + READY=$(kubectl -n $NAMESPACE get pods -l role=simplyblock-storage-node --no-headers 2>/dev/null | grep -c "Running" || true) + if [ "$READY" -ge "$EXPECTED_SNODES" ]; then break; fi + echo "snode-spdk: $READY/$EXPECTED_SNODES ($i/$MAX_POLL)" + sleep 10 + done + + for i in $(seq 1 $MAX_POLL); do + ADMIN_POD=$(kubectl -n $NAMESPACE get pods \ + -l app=simplyblock-admin-control \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true + OUTPUT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbctl cluster list 2>&1) || true + if echo "$OUTPUT" | grep -qi "active"; then + echo "C1 cluster is active!" + CLUSTER_ID=$(echo "$OUTPUT" | awk 'NR==4{print $2}') + CLUSTER_SECRET=$(echo "$OUTPUT" | awk 'NR==4{print $NF}') + if [ -z "$CLUSTER_ID" ] || [ "$CLUSTER_ID" = "+" ]; then + JSON_OUT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbctl cluster list --json 2>&1) || true + CLUSTER_ID=$(echo "$JSON_OUT" | jq -r '.[0].id // .[0].uuid // empty') + CLUSTER_SECRET=$(echo "$JSON_OUT" | jq -r '.[0].secret // empty') + fi + echo "C1_CLUSTER_ID=${CLUSTER_ID}" >> $GITHUB_ENV + echo "C1_CLUSTER_SECRET=${CLUSTER_SECRET}" >> $GITHUB_ENV + echo "CLUSTER_ID=${CLUSTER_ID}" >> $GITHUB_ENV + echo "CLUSTER_SECRET=${CLUSTER_SECRET}" >> $GITHUB_ENV + exit 0 + fi + echo "C1 not active ($i/$MAX_POLL)..." + sleep 10 + done + echo "ERROR: C1 cluster did not become active" + exit 1 + + - name: Wait for Cluster-2 to become active + timeout-minutes: 80 + run: | + NAMESPACE=${NS_C2} + MAX_POLL=300 + echo "=== Waiting for C2 cluster active in ${NAMESPACE} ===" + + ADMIN_POD="" + for i in $(seq 1 60); do + ADMIN_POD=$(kubectl -n $NAMESPACE get pods \ + -l app=simplyblock-admin-control \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true + if [ -n "$ADMIN_POD" ]; then + PHASE=$(kubectl -n $NAMESPACE get pod "$ADMIN_POD" -o jsonpath='{.status.phase}' 2>/dev/null) || true + if [ "$PHASE" = "Running" ]; then break; fi + fi + sleep 10 + done + + IFS=',' read -ra C2_NODES <<< "${C2_WORKERS}" + EXPECTED_SNODES=${#C2_NODES[@]} + for i in $(seq 1 $MAX_POLL); do + READY=$(kubectl -n $NAMESPACE get pods -l role=simplyblock-storage-node --no-headers 2>/dev/null | grep -c "Running" || true) + if [ "$READY" -ge "$EXPECTED_SNODES" ]; then break; fi + echo "snode-spdk: $READY/$EXPECTED_SNODES ($i/$MAX_POLL)" + sleep 10 + done + + for i in $(seq 1 $MAX_POLL); do + ADMIN_POD=$(kubectl -n $NAMESPACE get pods \ + -l app=simplyblock-admin-control \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true + OUTPUT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbctl cluster list 2>&1) || true + if echo "$OUTPUT" | grep -qi "active"; then + echo "C2 cluster is active!" + C2_ID=$(echo "$OUTPUT" | awk 'NR==4{print $2}') + C2_SECRET=$(echo "$OUTPUT" | awk 'NR==4{print $NF}') + if [ -z "$C2_ID" ] || [ "$C2_ID" = "+" ]; then + JSON_OUT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbctl cluster list --json 2>&1) || true + C2_ID=$(echo "$JSON_OUT" | jq -r '.[0].id // .[0].uuid // empty') + C2_SECRET=$(echo "$JSON_OUT" | jq -r '.[0].secret // empty') + fi + echo "C2_CLUSTER_ID=${C2_ID}" >> $GITHUB_ENV + echo "C2_CLUSTER_SECRET=${C2_SECRET}" >> $GITHUB_ENV + echo "CLUSTER2_ID=${C2_ID}" >> $GITHUB_ENV + echo "CLUSTER2_SECRET=${C2_SECRET}" >> $GITHUB_ENV + exit 0 + fi + echo "C2 not active ($i/$MAX_POLL)..." + sleep 10 + done + echo "ERROR: C2 cluster did not become active" + exit 1 + + # ── Patch fluent-bit + CSI daemonsets ─────────────────────────────── + + - name: Patch fluent-bit and CSI daemonsets (both namespaces) + run: | + for NS in ${NS_C1} ${NS_C2}; do + if kubectl get daemonset simplyblock-fluent-bit -n ${NS} &>/dev/null; then + kubectl patch daemonset simplyblock-fluent-bit -n ${NS} \ + --type=merge -p '{"spec":{"template":{"spec":{"affinity":null}}}}' || true + fi + if kubectl get daemonset simplyblock-csi-node -n ${NS} &>/dev/null; then + kubectl patch daemonset simplyblock-csi-node -n ${NS} \ + --type=merge -p '{"spec":{"template":{"spec":{"tolerations":[{"key":"node-role","operator":"Equal","value":"client","effect":"NoSchedule"}]}}}}' || true + fi + done + + # ── Run tests ────────────────────────────────────────────────────── + + - name: Set RUN_DIR_FILE + run: echo "RUN_DIR_FILE=/tmp/sb_k8s_run_dir_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}.txt" >> "$GITHUB_ENV" + + - name: Record Test Start Time + run: echo "TEST_START_TIME=$(date +%s)" >> $GITHUB_ENV + + - name: Setup and Run Cross-Cluster Restore Test + run: | + set -o pipefail + cd $GITHUB_WORKSPACE/e2e + python3 -m venv myenv + source myenv/bin/activate + python3 -m pip install -r requirements.txt + + export CLUSTER_ID="${{ env.C1_CLUSTER_ID }}" + export CLUSTER_SECRET="${{ env.C1_CLUSTER_SECRET }}" + export CLUSTER2_ID="${{ env.C2_CLUSTER_ID }}" + export CLUSTER2_SECRET="${{ env.C2_CLUSTER_SECRET }}" + export CLUSTER2_NAMESPACE="${NS_C2}" + export K8S_LOCAL_KUBECTL=1 + export SBCLI_CMD=sbctl + + python3 -u e2e.py \ + --testname TestBackupCrossClusterRestore \ + --ndcs $NDCS --npcs $NPCS --bs $BS --chunk_bs $CHUNK_BS \ + --run_k8s True --run_ha true 2>&1 | tee output.log + env: + NDCS: ${{ env.NDCS }} + NPCS: ${{ env.NPCS }} + BS: ${{ env.BS }} + CHUNK_BS: ${{ env.CHUNK_BS }} + SUPABASE_ANON_KEY: ${{ secrets.SUPABASE_ANON_KEY }} + CLIENT_IP: ${{ github.event.inputs.client_ips }} + SKIP_NFS: ${{ github.event.inputs.skip_nfs || 'false' }} + RUN_DIR_FILE: ${{ env.RUN_DIR_FILE }} + + # ── Post-test: logs, summary, notifications ───────────────────────── + + - name: Export RUN_BASE_DIR + if: always() + run: | + test -f "${RUN_DIR_FILE}" || exit 0 + RUN_BASE_DIR="$(cat "${RUN_DIR_FILE}" | tr -d '\r\n')" + [[ -n "${RUN_BASE_DIR}" ]] && echo "RUN_BASE_DIR=${RUN_BASE_DIR}" >> "$GITHUB_ENV" || true + + - name: Copy raw output.log to NFS + if: always() + shell: bash + run: | + set +e + out_log="$GITHUB_WORKSPACE/e2e/output.log" + [ -f "${out_log}" ] || exit 0 + if [ -n "${RUN_BASE_DIR:-}" ] && [ -d "${RUN_BASE_DIR}" ]; then + cp "${out_log}" "${RUN_BASE_DIR}/github_raw_output.log" || true + fi + + - name: Record Test End Time + if: always() + run: echo "TEST_END_TIME=$(date +%s)" >> $GITHUB_ENV + + - name: Calculate Total Time Taken + if: always() + run: | + TEST_TIME=$(($TEST_END_TIME - $TEST_START_TIME)) + echo "TEST_TIME_HOURS=$(($TEST_TIME / 3600))" >> $GITHUB_ENV + echo "TEST_TIME_MINS=$((($TEST_TIME % 3600) / 60))" >> $GITHUB_ENV + echo "TEST_TIME_SECS=$(($TEST_TIME % 60))" >> $GITHUB_ENV + + - name: Upload test output + if: always() + uses: actions/upload-artifact@v4 + with: + name: cross-cluster-restore-output + path: | + e2e/output.log + if-no-files-found: warn + + - name: Write Job Summary + if: always() + shell: bash + run: | + set +e + out_log="$GITHUB_WORKSPACE/e2e/output.log" + dur_fmt="${TEST_TIME_HOURS:-0}h ${TEST_TIME_MINS:-0}m ${TEST_TIME_SECS:-0}s" + conclusion="SUCCESS" + [[ "${{ job.status }}" != "success" ]] && conclusion="FAILED" + { + echo "## K8s Cross-Cluster Restore Test Summary" + echo "" + echo "**Result:** ${conclusion}" + echo "" + echo "### Run Info" + echo "- **Cluster-1 ID:** \`${C1_CLUSTER_ID}\` (namespace: \`${NS_C1}\`)" + echo "- **Cluster-2 ID:** \`${C2_CLUSTER_ID}\` (namespace: \`${NS_C2}\`)" + echo "- **C1 workers:** \`${C1_WORKERS}\`" + echo "- **C2 workers:** \`${C2_WORKERS}\`" + echo "- **NDCS/NPCS:** \`${NDCS}/${NPCS}\`" + echo "- **Duration:** ${dur_fmt}" + echo "" + if [[ -f "${out_log}" ]]; then + echo "### Last 30 lines" + echo '```' + tail -n 30 "${out_log}" | sed 's/\x1b\[[0-9;]*m//g' || true + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Stop MinIO trace + if: always() + run: | + set +e + [ -n "${MC_TRACE_PID:-}" ] && kill "${MC_TRACE_PID}" 2>/dev/null || true + [ -n "${MC_PORT_FWD_PID:-}" ] && kill "${MC_PORT_FWD_PID}" 2>/dev/null || true + if [ -f "${MINIO_TRACE_LOG:-}" ]; then + cp "${MINIO_TRACE_LOG}" $GITHUB_WORKSPACE/e2e/minio-trace.log 2>/dev/null || true + fi + + - name: Upload MinIO trace log + if: always() + uses: actions/upload-artifact@v4 + with: + name: minio-trace-log + path: e2e/minio-trace.log + if-no-files-found: ignore + + - name: Cleanup MinIO namespace + if: always() + run: kubectl delete namespace minio --wait=false 2>/dev/null || true + + - name: Send Slack Notification + if: always() && (github.event.inputs.send_slack_notification || 'true') == 'true' + shell: bash + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + JOB_STATUS: ${{ job.status }} + SLACK_RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + run: | + if [ -z "${SLACK_WEBHOOK_URL:-}" ]; then exit 0; fi + STATUS_EMOJI="$([ "$JOB_STATUS" = "success" ] && echo ":white_check_mark:" || echo ":x:")" + curl -s -X POST "${SLACK_WEBHOOK_URL}" \ + -H 'Content-Type: application/json' \ + -d "{\"text\":\"${STATUS_EMOJI} *K8s Cross-Cluster Restore*: ${JOB_STATUS}\\nC1: \`${C1_CLUSTER_ID:-?}\` | C2: \`${C2_CLUSTER_ID:-?}\`\\nNDCS/NPCS: ${NDCS}/${NPCS}\\n<${SLACK_RUN_URL}|View Run>\"}" || true diff --git a/.github/workflows/k8s-native-e2e-add-node.yaml b/.github/workflows/k8s-native-e2e-add-node.yaml index bba1546a14..828f289976 100644 --- a/.github/workflows/k8s-native-e2e-add-node.yaml +++ b/.github/workflows/k8s-native-e2e-add-node.yaml @@ -75,6 +75,14 @@ on: nfs_mountpoint: type: string default: '/mnt/nfs_share/' + cluster_security: + type: string + default: 'none' + description: "Cluster setup mode. 'backup' deploys MinIO and enables backup on StorageCluster." + preserve_resources_on_failure: + type: boolean + default: true + description: 'Preserve PVCs, FIO pods, snapshots on test failure for debugging' workflow_dispatch: inputs: testname: @@ -118,7 +126,7 @@ on: required: true default: 'worker-0.ocp.simplyblock.ai,worker-1.ocp.simplyblock.ai,worker-2.ocp.simplyblock.ai,worker-3.ocp.simplyblock.ai,worker-4.ocp.simplyblock.ai,worker-5.ocp.simplyblock.ai' new_worker_nodes: - description: 'Comma-separated new K8s worker node names to add during test (at least 2, multiples of 2)' + description: 'Comma-separated new K8s worker node names to add during test (at least 1)' required: true ifc_names: description: 'Network interfaces (mgmt_ifc:data_nics)' @@ -185,6 +193,19 @@ on: description: 'NFS mountpoint on client nodes' required: false default: '/mnt/nfs_share/' + cluster_security: + description: "Cluster setup mode. 'backup' deploys MinIO and enables backup on StorageCluster. Auto-enabled for backup/backup-topology test names." + required: false + type: choice + default: "none" + options: + - none + - backup + preserve_resources_on_failure: + description: 'Preserve PVCs, FIO pods, snapshots on test failure for debugging' + required: false + default: true + type: boolean jobs: e2e-add-node: @@ -376,6 +397,11 @@ jobs: set +e NAMESPACE=simplyblock + # Run the operator's own cleanup script first (thorough helm + CR cleanup) + if [ -x "$GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh" ]; then + bash $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh -f $NAMESPACE || true + fi + # Run the shared cleanup script (handles etcd overload with # --request-timeout, bulk deletes, and parallel finalizer patching) bash $GITHUB_WORKSPACE/e2e/scripts/cleanup_k8s.sh $NAMESPACE @@ -411,12 +437,18 @@ jobs: echo "Resetting hugepages to 0 on $NODE..." if [[ "$CLUSTER_ENV" == *"openshift"* ]]; then oc debug node/"$NODE" -- chroot /host bash -c \ - "echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + echo "Cleaning stale simplyblock config on $NODE..." + oc debug node/"$NODE" -- chroot /host bash -c \ + "rm -rf /etc/simplyblock" 2>/dev/null || true echo "Restarting kubelet on $NODE..." oc debug node/"$NODE" -- chroot /host systemctl restart kubelet 2>/dev/null || true else kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ - "echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + echo "Cleaning stale simplyblock config on $NODE..." + kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ + "rm -rf /etc/simplyblock" 2>/dev/null || true echo "Restarting kubelet on $NODE..." kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ "systemctl restart kubelet" 2>/dev/null || true @@ -424,6 +456,33 @@ jobs: echo "Done with $NODE" done + # Also reset hugepages and clean config on new_worker_nodes (may have been used in a previous run) + IFS=',' read -ra NEW_NODES <<< "${{ inputs.new_worker_nodes }}" + for NODE in "${NEW_NODES[@]}"; do + NODE=$(echo "$NODE" | xargs) + [[ -z "$NODE" ]] && continue + echo "Resetting hugepages on new node $NODE..." + if [[ "$CLUSTER_ENV" == *"openshift"* ]]; then + oc debug node/"$NODE" -- chroot /host bash -c \ + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + echo "Cleaning stale simplyblock config on new node $NODE..." + oc debug node/"$NODE" -- chroot /host bash -c \ + "rm -rf /etc/simplyblock" 2>/dev/null || true + oc debug node/"$NODE" -- chroot /host systemctl restart kubelet 2>/dev/null || true + else + kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + echo "Cleaning stale simplyblock config on new node $NODE..." + kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ + "rm -rf /etc/simplyblock" 2>/dev/null || true + kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ + "systemctl restart kubelet" 2>/dev/null || true + fi + echo "Removing simplyblock labels from new node $NODE..." + kubectl label node "$NODE" simplyblock.io/role- 2>/dev/null || true + echo "Done with new node $NODE" + done + echo "=== Cleanup complete ===" - name: Cleanup old cert-manager deployment @@ -459,38 +518,44 @@ jobs: kubectl wait --for=delete namespace/vault --timeout=120s 2>/dev/null || true echo "=== KMS cleanup complete ===" - # ── Label nodes ───────────────────────────────────────────────────────── + # ── Remove stale labels from previous runs ────────────────────────────── - - name: Label initial worker nodes + - name: Remove stale storagenodeset labels from all worker nodes if: ${{ inputs.use_existing_cluster != 'true' }} run: | - CLUSTER_ENV="${{ inputs.cluster_environment || 'local' }}" + echo "Removing stale io.simplyblock.storagenodeset label from all worker nodes..." + # Remove from initial workers IFS=',' read -ra NODES <<< "${{ inputs.worker_nodes }}" + # Include the new worker nodes (may have labels from a previous run) + IFS=',' read -ra NEW_NODES <<< "${{ inputs.new_worker_nodes }}" + NODES+=("${NEW_NODES[@]}") for NODE in "${NODES[@]}"; do - if [ "$CLUSTER_ENV" = "local" ] || [ "$CLUSTER_ENV" = "openshift-local" ]; then - echo "Labeling node $NODE with zone=default (local cluster)" - kubectl label node "$NODE" topology.kubernetes.io/zone=default --overwrite - else - echo "Skipping zone label for $NODE (cloud cluster: $CLUSTER_ENV)" - fi - kubectl label node "$NODE" simplyblock.io/role=mgmt-plane --overwrite + echo "Removing storagenodeset label from $NODE" + kubectl label node "$NODE" io.simplyblock.storagenodeset- 2>/dev/null || true done + echo "Stale label cleanup complete." - - name: Label new worker nodes + # ── Label nodes ───────────────────────────────────────────────────────── + + - name: Label initial worker nodes if: ${{ inputs.use_existing_cluster != 'true' }} run: | CLUSTER_ENV="${{ inputs.cluster_environment || 'local' }}" - IFS=',' read -ra NODES <<< "${{ inputs.new_worker_nodes }}" + IFS=',' read -ra NODES <<< "${{ inputs.worker_nodes }}" for NODE in "${NODES[@]}"; do if [ "$CLUSTER_ENV" = "local" ] || [ "$CLUSTER_ENV" = "openshift-local" ]; then - echo "Labeling new node $NODE with zone=default (local cluster)" + echo "Labeling node $NODE with zone=default (local cluster)" kubectl label node "$NODE" topology.kubernetes.io/zone=default --overwrite else - echo "Skipping zone label for new node $NODE (cloud cluster: $CLUSTER_ENV)" + echo "Skipping zone label for $NODE (cloud cluster: $CLUSTER_ENV)" fi kubectl label node "$NODE" simplyblock.io/role=mgmt-plane --overwrite done + # NOTE: new_worker_nodes are NOT labeled here. They must remain + # unlabeled so the DaemonSet does not schedule storage-node pods on them + # before the test adds them to the cluster via StorageNodeSet CR update. + # ── Prepare namespace ─────────────────────────────────────────────────── - name: Create namespace and set pod-security labels @@ -527,6 +592,116 @@ jobs: pod-security.kubernetes.io/warn=privileged \ --overwrite + # ── Deploy MinIO for backup tests (conditional) ──────────────────────── + + - name: Deploy MinIO for backup tests + if: >- + ${{ + inputs.use_existing_cluster != 'true' && + (inputs.cluster_security == 'backup' || + inputs.testname == 'backup-topology' || + contains(inputs.testname, 'Backup')) + }} + run: | + set -euxo pipefail + NAMESPACE=simplyblock + + echo "=== Setting up MinIO for backup tests ===" + + # 1. Create minio namespace + deployment + service + kubectl create ns minio --dry-run=client -o yaml | kubectl apply -f - + + cat <<'MINIO_EOF' | kubectl apply -f - + apiVersion: apps/v1 + kind: Deployment + metadata: + name: minio + namespace: minio + spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio + command: ["/bin/sh", "-c", "minio server /data --console-address :9001"] + env: + - name: MINIO_ROOT_USER + value: minioadmin + - name: MINIO_ROOT_PASSWORD + value: minioadmin123 + ports: + - containerPort: 9000 + - containerPort: 9001 + MINIO_EOF + + kubectl -n minio expose deploy/minio --port 9000 \ + --dry-run=client -o yaml | kubectl apply -f - + + # 2. Wait for MinIO pod to be ready (up to 5 min) + echo "Waiting for MinIO pod to be ready..." + for i in $(seq 1 30); do + READY=$(kubectl -n minio get pods --no-headers 2>/dev/null | grep -c "Running" || true) + if [ "$READY" -ge 1 ]; then + echo "MinIO pod is running" + break + fi + echo "MinIO not ready yet ($i/30)..." + sleep 10 + done + + # 3. Create backup-credentials K8s Secret in cluster namespace + cat <- + ${{ + inputs.use_existing_cluster != 'true' && + (inputs.cluster_security == 'backup' || + inputs.testname == 'backup-topology' || + contains(inputs.testname, 'Backup')) + }} + run: | + set -euxo pipefail + + # 1. Install mc (MinIO Client) + curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc + chmod +x /usr/local/bin/mc + + # 2. Port-forward MinIO service to localhost + kubectl -n minio port-forward svc/minio 9000:9000 & + echo "MC_PORT_FWD_PID=$!" >> "$GITHUB_ENV" + sleep 3 + + # 3. Configure mc alias + mc alias set myminio http://localhost:9000 minioadmin minioadmin123 + + # 4. Start admin trace in background + MINIO_TRACE_LOG="/tmp/minio-trace-${GITHUB_RUN_ID}.log" + echo "MINIO_TRACE_LOG=${MINIO_TRACE_LOG}" >> "$GITHUB_ENV" + nohup mc admin trace -v --all myminio > "${MINIO_TRACE_LOG}" 2>&1 & + echo "MC_TRACE_PID=$!" >> "$GITHUB_ENV" + + echo "=== MinIO trace logging started (PID=$!, log=${MINIO_TRACE_LOG}) ===" + - name: Wait before helm install if: ${{ inputs.use_existing_cluster != 'true' }} run: sleep 30 @@ -534,7 +709,7 @@ jobs: # ── Deploy CSI stack ──────────────────────────────────────────────────── - name: Install cert-manager (TLS prerequisite) - if: ${{ inputs.use_existing_cluster != 'true' && inputs.tls_enabled == 'true' }} + if: ${{ inputs.use_existing_cluster != 'true' && inputs.tls_enabled }} run: | helm repo add jetstack https://charts.jetstack.io helm repo update @@ -549,6 +724,7 @@ jobs: - name: Install Helm Chart for simplyblock-operator if: ${{ inputs.use_existing_cluster != 'true' }} run: | + set -euxo pipefail cd $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator/ TLS_FLAGS="" @@ -676,7 +852,7 @@ jobs: done - name: Setup KMS (vault) for encryption - if: ${{ inputs.use_existing_cluster != 'true' && inputs.tls_enabled == 'true' }} + if: ${{ inputs.use_existing_cluster != 'true' && inputs.tls_enabled }} run: | # Ensure vault namespace from previous run is fully terminated before installing. echo "Checking if vault namespace is still terminating..." @@ -759,6 +935,12 @@ jobs: RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' fi + PCIE_MODEL_YAML="" + CLUSTER_ENV="${{ inputs.cluster_environment }}" + if [ "$CLUSTER_ENV" != "openshift-baremetal" ] && [ "$CLUSTER_ENV" != "openshift-local" ]; then + PCIE_MODEL_YAML=' pcieModel: "SAMSUNG MZQLB1T9HAJR-00007"' + fi + # Build workerNodes YAML list from initial worker nodes only WORKER_YAML="" IFS=',' read -ra NODES <<< "${{ inputs.worker_nodes }}" @@ -766,6 +948,21 @@ jobs: WORKER_YAML="${WORKER_YAML} - ${NODE}"$'\n' done + # Conditionally build backup spec block + BACKUP_SPEC="" + CLUSTER_SEC="${{ inputs.cluster_security || 'none' }}" + TESTNAME_INPUT="${{ inputs.testname || '' }}" + if [ "$CLUSTER_SEC" = "backup" ] || [ "$TESTNAME_INPUT" = "backup-topology" ] || echo "$TESTNAME_INPUT" | grep -qi "Backup"; then + BACKUP_SPEC=" backup: + credentialsSecretRef: + name: backup-credentials + localEndpoint: http://minio.minio.svc.cluster.local:9000 + localTesting: true + secondaryTarget: 0 + snapshotBackups: true + withCompression: false" + fi + # Build vault settings and encryption pool blocks (conditional on TLS) VAULT_SETTINGS="" ENCRYPTION_POOL_YAML="" @@ -805,6 +1002,7 @@ jobs: criticalThreshold: capacity: 96 provisionedCapacity: 98 + ${BACKUP_SPEC} ${VAULT_SETTINGS} --- apiVersion: storage.simplyblock.io/v1alpha1 @@ -826,7 +1024,7 @@ jobs: clusterImage: "${SB_REPO}:${SB_TAG}" spdkImage: "${SPDK_IMAGE}" spdkProxyImage: "${SB_REPO}:${SB_TAG}" - pcieModel: "SAMSUNG MZQLB1T9HAJR-00007" + ${PCIE_MODEL_YAML} driveSizeRange: "${DRIVE_SIZE_RANGE}" mgmtIfname: ${MGMT_IFC} dataIfname: @@ -1184,9 +1382,15 @@ jobs: export K8S_LOCAL_KUBECTL=1 export SBCLI_CMD=sbctl + PRESERVE_ARG="" + if [ "${{ inputs.preserve_resources_on_failure }}" = "true" ]; then + PRESERVE_ARG="--preserve_resources_on_failure True" + fi + python3 -u e2e.py \ --testname "${{ inputs.testname || 'K8sNativeAddNodeTest' }}" \ --new_worker_nodes "${{ inputs.new_worker_nodes }}" \ + ${PRESERVE_ARG} \ --ndcs $NDCS --npcs $NPCS --bs $BS --chunk_bs $CHUNK_BS \ --run_k8s True \ --namespace simplyblock \ @@ -1386,7 +1590,7 @@ jobs: retention-days: 7 - name: Send Slack Notification - if: always() && (inputs.send_slack_notification || 'true') == 'true' + if: always() && inputs.send_slack_notification != false shell: bash env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} @@ -1739,6 +1943,35 @@ jobs: env: CLUSTER_ID: ${{ env.CLUSTER_ID }} MON_SECRET: ${{ secrets.MON_SECRET }} + - name: Stop MinIO trace logging and save + if: always() + run: | + set +e + if [ -n "${MC_TRACE_PID:-}" ]; then + echo "Stopping MinIO trace (PID=${MC_TRACE_PID})..." + kill "${MC_TRACE_PID}" 2>/dev/null || true + wait "${MC_TRACE_PID}" 2>/dev/null || true + fi + if [ -n "${MC_PORT_FWD_PID:-}" ]; then + kill "${MC_PORT_FWD_PID}" 2>/dev/null || true + fi + if [ -f "${MINIO_TRACE_LOG:-}" ]; then + LINES=$(wc -l < "${MINIO_TRACE_LOG}") + SIZE=$(du -h "${MINIO_TRACE_LOG}" | cut -f1) + echo "=== MinIO trace log: ${LINES} lines, ${SIZE} ===" + cp "${MINIO_TRACE_LOG}" e2e/minio-trace.log 2>/dev/null || true + else + echo "No MinIO trace log found (backup tests may not have run)" + fi + + - name: Upload MinIO trace log + if: always() + uses: actions/upload-artifact@v4 + with: + name: minio-trace-${{ github.run_id }} + path: e2e/minio-trace.log + if-no-files-found: ignore + - name: Cleanup build folder if: always() run: | diff --git a/.github/workflows/k8s-native-e2e-node-migration.yaml b/.github/workflows/k8s-native-e2e-node-migration.yaml index dc2032b8d8..2cc9d2f7a2 100644 --- a/.github/workflows/k8s-native-e2e-node-migration.yaml +++ b/.github/workflows/k8s-native-e2e-node-migration.yaml @@ -81,6 +81,14 @@ on: nfs_mountpoint: type: string default: '/mnt/nfs_share/' + cluster_security: + type: string + default: 'none' + description: "Cluster setup mode. 'backup' deploys MinIO and enables backup on StorageCluster." + preserve_resources_on_failure: + type: boolean + default: true + description: 'Preserve PVCs, FIO pods, snapshots on test failure for debugging' workflow_dispatch: inputs: testname: @@ -200,6 +208,10 @@ on: description: 'NFS mountpoint on client nodes' required: false default: '/mnt/nfs_share/' + # cluster_security is available via workflow_call only (25 input limit). + # For workflow_dispatch, backup is auto-enabled when testname contains "Backup". + # preserve_resources_on_failure is also workflow_call only (25 input limit); + # defaults to true when not provided. jobs: e2e-node-migration: @@ -392,6 +404,11 @@ jobs: NAMESPACE=simplyblock KUBECTL_TIMEOUT="--request-timeout=120s" + # Run the operator's own cleanup script first (thorough helm + CR cleanup) + if [ -x "$GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh" ]; then + bash $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh -f $NAMESPACE || true + fi + # Run the shared cleanup script (handles etcd overload with # --request-timeout, bulk deletes, and parallel finalizer patching) bash $GITHUB_WORKSPACE/e2e/scripts/cleanup_k8s.sh $NAMESPACE @@ -432,12 +449,18 @@ jobs: echo "Resetting hugepages to 0 on $NODE..." if [[ "$CLUSTER_ENV" == *"openshift"* ]]; then oc debug node/"$NODE" -- chroot /host bash -c \ - "echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + echo "Cleaning stale simplyblock config on $NODE..." + oc debug node/"$NODE" -- chroot /host bash -c \ + "rm -rf /etc/simplyblock" 2>/dev/null || true echo "Restarting kubelet on $NODE..." oc debug node/"$NODE" -- chroot /host systemctl restart kubelet 2>/dev/null || true else kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ - "echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + echo "Cleaning stale simplyblock config on $NODE..." + kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ + "rm -rf /etc/simplyblock" 2>/dev/null || true echo "Restarting kubelet on $NODE..." kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ "systemctl restart kubelet" 2>/dev/null || true @@ -480,6 +503,25 @@ jobs: kubectl wait --for=delete namespace/vault --timeout=120s 2>/dev/null || true echo "=== KMS cleanup complete ===" + # ── Remove stale labels from previous runs ────────────────────────────── + + - name: Remove stale storagenodeset labels from all worker nodes + if: ${{ inputs.use_existing_cluster != 'true' }} + run: | + echo "Removing stale io.simplyblock.storagenodeset label from all worker nodes..." + # Remove from initial workers + IFS=',' read -ra NODES <<< "${{ inputs.worker_nodes }}" + # Include the migration target + MIGRATE_NODE="${{ inputs.migrate_to_worker }}" + if [ -n "$MIGRATE_NODE" ]; then + NODES+=("$MIGRATE_NODE") + fi + for NODE in "${NODES[@]}"; do + echo "Removing storagenodeset label from $NODE" + kubectl label node "$NODE" io.simplyblock.storagenodeset- 2>/dev/null || true + done + echo "Stale label cleanup complete." + # ── Label nodes ───────────────────────────────────────────────────────── - name: Label worker nodes @@ -546,6 +588,116 @@ jobs: pod-security.kubernetes.io/warn=privileged \ --overwrite + # ── Deploy MinIO for backup tests (conditional) ──────────────────────── + + - name: Deploy MinIO for backup tests + if: >- + ${{ + inputs.use_existing_cluster != 'true' && + (inputs.cluster_security == 'backup' || + inputs.testname == 'backup-topology' || + contains(inputs.testname, 'Backup')) + }} + run: | + set -euxo pipefail + NAMESPACE=simplyblock + + echo "=== Setting up MinIO for backup tests ===" + + # 1. Create minio namespace + deployment + service + kubectl create ns minio --dry-run=client -o yaml | kubectl apply -f - + + cat <<'MINIO_EOF' | kubectl apply -f - + apiVersion: apps/v1 + kind: Deployment + metadata: + name: minio + namespace: minio + spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio + command: ["/bin/sh", "-c", "minio server /data --console-address :9001"] + env: + - name: MINIO_ROOT_USER + value: minioadmin + - name: MINIO_ROOT_PASSWORD + value: minioadmin123 + ports: + - containerPort: 9000 + - containerPort: 9001 + MINIO_EOF + + kubectl -n minio expose deploy/minio --port 9000 \ + --dry-run=client -o yaml | kubectl apply -f - + + # 2. Wait for MinIO pod to be ready (up to 5 min) + echo "Waiting for MinIO pod to be ready..." + for i in $(seq 1 30); do + READY=$(kubectl -n minio get pods --no-headers 2>/dev/null | grep -c "Running" || true) + if [ "$READY" -ge 1 ]; then + echo "MinIO pod is running" + break + fi + echo "MinIO not ready yet ($i/30)..." + sleep 10 + done + + # 3. Create backup-credentials K8s Secret in cluster namespace + cat <- + ${{ + inputs.use_existing_cluster != 'true' && + (inputs.cluster_security == 'backup' || + inputs.testname == 'backup-topology' || + contains(inputs.testname, 'Backup')) + }} + run: | + set -euxo pipefail + + # 1. Install mc (MinIO Client) + curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc + chmod +x /usr/local/bin/mc + + # 2. Port-forward MinIO service to localhost + kubectl -n minio port-forward svc/minio 9000:9000 & + echo "MC_PORT_FWD_PID=$!" >> "$GITHUB_ENV" + sleep 3 + + # 3. Configure mc alias + mc alias set myminio http://localhost:9000 minioadmin minioadmin123 + + # 4. Start admin trace in background + MINIO_TRACE_LOG="/tmp/minio-trace-${GITHUB_RUN_ID}.log" + echo "MINIO_TRACE_LOG=${MINIO_TRACE_LOG}" >> "$GITHUB_ENV" + nohup mc admin trace -v --all myminio > "${MINIO_TRACE_LOG}" 2>&1 & + echo "MC_TRACE_PID=$!" >> "$GITHUB_ENV" + + echo "=== MinIO trace logging started (PID=$!, log=${MINIO_TRACE_LOG}) ===" + - name: Wait before helm install if: ${{ inputs.use_existing_cluster != 'true' }} run: sleep 30 @@ -553,7 +705,7 @@ jobs: # ── Deploy CSI stack ──────────────────────────────────────────────────── - name: Install cert-manager (TLS prerequisite) - if: ${{ inputs.use_existing_cluster != 'true' && inputs.tls_enabled == 'true' }} + if: ${{ inputs.use_existing_cluster != 'true' && inputs.tls_enabled }} run: | helm repo add jetstack https://charts.jetstack.io helm repo update @@ -568,6 +720,7 @@ jobs: - name: Install Helm Chart for simplyblock-operator if: ${{ inputs.use_existing_cluster != 'true' }} run: | + set -euxo pipefail cd $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator/ TLS_FLAGS="" @@ -695,7 +848,7 @@ jobs: done - name: Setup KMS (vault) for encryption - if: ${{ inputs.use_existing_cluster != 'true' && inputs.tls_enabled == 'true' }} + if: ${{ inputs.use_existing_cluster != 'true' && inputs.tls_enabled }} run: | # Ensure vault namespace from previous run is fully terminated before installing. echo "Checking if vault namespace is still terminating..." @@ -778,6 +931,12 @@ jobs: RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' fi + PCIE_MODEL_YAML="" + CLUSTER_ENV="${{ inputs.cluster_environment }}" + if [ "$CLUSTER_ENV" != "openshift-baremetal" ] && [ "$CLUSTER_ENV" != "openshift-local" ]; then + PCIE_MODEL_YAML=' pcieModel: "SAMSUNG MZQLB1T9HAJR-00007"' + fi + # Build workerNodes YAML list from all worker nodes WORKER_YAML="" IFS=',' read -ra NODES <<< "${{ inputs.worker_nodes }}" @@ -785,6 +944,21 @@ jobs: WORKER_YAML="${WORKER_YAML} - ${NODE}"$'\n' done + # Conditionally build backup spec block + BACKUP_SPEC="" + CLUSTER_SEC="${{ inputs.cluster_security || 'none' }}" + TESTNAME_INPUT="${{ inputs.testname || '' }}" + if [ "$CLUSTER_SEC" = "backup" ] || [ "$TESTNAME_INPUT" = "backup-topology" ] || echo "$TESTNAME_INPUT" | grep -qi "Backup"; then + BACKUP_SPEC=" backup: + credentialsSecretRef: + name: backup-credentials + localEndpoint: http://minio.minio.svc.cluster.local:9000 + localTesting: true + secondaryTarget: 0 + snapshotBackups: true + withCompression: false" + fi + # Build vault settings and encryption pool blocks (conditional on TLS) VAULT_SETTINGS="" ENCRYPTION_POOL_YAML="" @@ -824,6 +998,7 @@ jobs: criticalThreshold: capacity: 96 provisionedCapacity: 98 + ${BACKUP_SPEC} ${VAULT_SETTINGS} --- apiVersion: storage.simplyblock.io/v1alpha1 @@ -845,7 +1020,7 @@ jobs: clusterImage: "${SB_REPO}:${SB_TAG}" spdkImage: "${SPDK_IMAGE}" spdkProxyImage: "${SB_REPO}:${SB_TAG}" - pcieModel: "SAMSUNG MZQLB1T9HAJR-00007" + ${PCIE_MODEL_YAML} driveSizeRange: "${DRIVE_SIZE_RANGE}" mgmtIfname: ${MGMT_IFC} dataIfname: @@ -1213,10 +1388,15 @@ jobs: REATTACH_VOL_ARG="--reattach_volume True" fi + PRESERVE_ARG="--preserve_resources_on_failure True" + if [ "${{ inputs.preserve_resources_on_failure }}" = "false" ]; then + PRESERVE_ARG="" + fi + python3 -u e2e.py \ --testname "${{ inputs.testname || 'K8sNativeNodeMigrationTest' }}" \ --migrate_to_worker "${{ inputs.migrate_to_worker }}" \ - ${NEW_SSD_PCIE_ARG} ${REATTACH_VOL_ARG} \ + ${NEW_SSD_PCIE_ARG} ${REATTACH_VOL_ARG} ${PRESERVE_ARG} \ --ndcs $NDCS --npcs $NPCS --bs $BS --chunk_bs $CHUNK_BS \ --run_k8s True \ --namespace simplyblock \ @@ -1416,7 +1596,7 @@ jobs: retention-days: 7 - name: Send Slack Notification - if: always() && (inputs.send_slack_notification || 'true') == 'true' + if: always() && inputs.send_slack_notification != false shell: bash env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} @@ -1769,6 +1949,35 @@ jobs: env: CLUSTER_ID: ${{ env.CLUSTER_ID }} MON_SECRET: ${{ secrets.MON_SECRET }} + - name: Stop MinIO trace logging and save + if: always() + run: | + set +e + if [ -n "${MC_TRACE_PID:-}" ]; then + echo "Stopping MinIO trace (PID=${MC_TRACE_PID})..." + kill "${MC_TRACE_PID}" 2>/dev/null || true + wait "${MC_TRACE_PID}" 2>/dev/null || true + fi + if [ -n "${MC_PORT_FWD_PID:-}" ]; then + kill "${MC_PORT_FWD_PID}" 2>/dev/null || true + fi + if [ -f "${MINIO_TRACE_LOG:-}" ]; then + LINES=$(wc -l < "${MINIO_TRACE_LOG}") + SIZE=$(du -h "${MINIO_TRACE_LOG}" | cut -f1) + echo "=== MinIO trace log: ${LINES} lines, ${SIZE} ===" + cp "${MINIO_TRACE_LOG}" e2e/minio-trace.log 2>/dev/null || true + else + echo "No MinIO trace log found (backup tests may not have run)" + fi + + - name: Upload MinIO trace log + if: always() + uses: actions/upload-artifact@v4 + with: + name: minio-trace-${{ github.run_id }} + path: e2e/minio-trace.log + if-no-files-found: ignore + - name: Cleanup build folder if: always() run: | diff --git a/.github/workflows/k8s-native-e2e.yaml b/.github/workflows/k8s-native-e2e.yaml index 5177f9f919..6f7e48f929 100644 --- a/.github/workflows/k8s-native-e2e.yaml +++ b/.github/workflows/k8s-native-e2e.yaml @@ -343,6 +343,11 @@ jobs: set +e NAMESPACE=simplyblock + # Run the operator's own cleanup script first (thorough helm + CR cleanup) + if [ -x "$GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh" ]; then + bash $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh -f $NAMESPACE || true + fi + # Run the shared cleanup script (handles etcd overload with # --request-timeout, bulk deletes, and parallel finalizer patching) bash $GITHUB_WORKSPACE/e2e/scripts/cleanup_k8s.sh $NAMESPACE @@ -375,15 +380,15 @@ jobs: CLUSTER_ENV="${{ github.event.inputs.cluster_environment || 'local' }}" IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" for NODE in "${NODES[@]}"; do - echo "Resetting hugepages to 0 on $NODE..." + echo "Disconnecting stale NVMe-oF and resetting hugepages on $NODE..." if [[ "$CLUSTER_ENV" == *"openshift"* ]]; then oc debug node/"$NODE" -- chroot /host bash -c \ - "echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true echo "Restarting kubelet on $NODE..." oc debug node/"$NODE" -- chroot /host systemctl restart kubelet 2>/dev/null || true else kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ - "echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true echo "Restarting kubelet on $NODE..." kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ "systemctl restart kubelet" 2>/dev/null || true @@ -426,6 +431,19 @@ jobs: kubectl wait --for=delete namespace/vault --timeout=120s 2>/dev/null || true echo "=== KMS cleanup complete ===" + # ── Remove stale labels from previous runs ────────────────────────────── + + - name: Remove stale storagenodeset labels from all worker nodes + if: ${{ github.event.inputs.use_existing_cluster != 'true' }} + run: | + echo "Removing stale io.simplyblock.storagenodeset label from all worker nodes..." + IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" + for NODE in "${NODES[@]}"; do + echo "Removing storagenodeset label from $NODE" + kubectl label node "$NODE" io.simplyblock.storagenodeset- 2>/dev/null || true + done + echo "Stale label cleanup complete." + # ── Label nodes (after cluster is active) ─────────────────────────────── - name: Label worker nodes @@ -558,6 +576,37 @@ jobs: echo "=== MinIO and backup credentials setup complete ===" + - name: Start MinIO trace logging + if: >- + ${{ + github.event.inputs.use_existing_cluster != 'true' && + (github.event.inputs.cluster_security == 'backup' || + github.event.inputs.testname == 'backup' || + github.event.inputs.testname == 'backup-stress') + }} + run: | + set -euxo pipefail + + # 1. Install mc (MinIO Client) + curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc + chmod +x /usr/local/bin/mc + + # 2. Port-forward MinIO service to localhost + kubectl -n minio port-forward svc/minio 9000:9000 & + echo "MC_PORT_FWD_PID=$!" >> "$GITHUB_ENV" + sleep 3 + + # 3. Configure mc alias + mc alias set myminio http://localhost:9000 minioadmin minioadmin123 + + # 4. Start admin trace in background + MINIO_TRACE_LOG="/tmp/minio-trace-${GITHUB_RUN_ID}.log" + echo "MINIO_TRACE_LOG=${MINIO_TRACE_LOG}" >> "$GITHUB_ENV" + nohup mc admin trace -v --all myminio > "${MINIO_TRACE_LOG}" 2>&1 & + echo "MC_TRACE_PID=$!" >> "$GITHUB_ENV" + + echo "=== MinIO trace logging started (PID=$!, log=${MINIO_TRACE_LOG}) ===" + - name: Wait before helm install if: ${{ github.event.inputs.use_existing_cluster != 'true' }} run: sleep 30 @@ -580,6 +629,7 @@ jobs: - name: Install Helm Chart for simplyblock-operator if: ${{ github.event.inputs.use_existing_cluster != 'true' }} run: | + set -euxo pipefail cd $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator/ TLS_FLAGS="" @@ -792,6 +842,12 @@ jobs: RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' fi + PCIE_MODEL_YAML="" + CLUSTER_ENV="${{ github.event.inputs.cluster_environment }}" + if [ "$CLUSTER_ENV" != "openshift-baremetal" ] && [ "$CLUSTER_ENV" != "openshift-local" ]; then + PCIE_MODEL_YAML=' pcieModel: "SAMSUNG MZQLB1T9HAJR-00007"' + fi + # Build workerNodes YAML list from comma-separated input WORKER_YAML="" IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" @@ -875,7 +931,7 @@ jobs: clusterImage: "${SB_REPO}:${SB_TAG}" spdkImage: "${SPDK_IMAGE}" spdkProxyImage: "${SB_REPO}:${SB_TAG}" - pcieModel: "SAMSUNG MZQLB1T9HAJR-00007" + ${PCIE_MODEL_YAML} driveSizeRange: "${DRIVE_SIZE_RANGE}" mgmtIfname: ${MGMT_IFC} dataIfname: @@ -939,7 +995,7 @@ jobs: clusterImage: "${SB_REPO}:${SB_TAG}" spdkImage: "${SPDK_IMAGE}" spdkProxyImage: "${SB_REPO}:${SB_TAG}" - pcieModel: "SAMSUNG MZQLB1T9HAJR-00007" + ${PCIE_MODEL_YAML} driveSizeRange: "${DRIVE_SIZE_RANGE}" mgmtIfname: ${MGMT_IFC} dataIfname: @@ -1923,6 +1979,36 @@ jobs: e2e/output.log if-no-files-found: warn + - name: Stop MinIO trace logging and save + if: always() + run: | + set +e + if [ -n "${MC_TRACE_PID:-}" ]; then + echo "Stopping MinIO trace (PID=${MC_TRACE_PID})..." + kill "${MC_TRACE_PID}" 2>/dev/null || true + wait "${MC_TRACE_PID}" 2>/dev/null || true + fi + if [ -n "${MC_PORT_FWD_PID:-}" ]; then + kill "${MC_PORT_FWD_PID}" 2>/dev/null || true + fi + if [ -f "${MINIO_TRACE_LOG:-}" ]; then + LINES=$(wc -l < "${MINIO_TRACE_LOG}") + SIZE=$(du -h "${MINIO_TRACE_LOG}" | cut -f1) + echo "=== MinIO trace log: ${LINES} lines, ${SIZE} ===" + # Copy to e2e/ so it gets picked up by artifact upload + cp "${MINIO_TRACE_LOG}" e2e/minio-trace.log 2>/dev/null || true + else + echo "No MinIO trace log found (backup tests may not have run)" + fi + + - name: Upload MinIO trace log + if: always() + uses: actions/upload-artifact@v4 + with: + name: minio-trace-${{ github.run_id }} + path: e2e/minio-trace.log + if-no-files-found: ignore + - name: Cleanup MinIO namespace (if deployed) if: always() run: kubectl delete namespace minio --wait=false 2>/dev/null || true diff --git a/.github/workflows/k8s-native-stress.yaml b/.github/workflows/k8s-native-stress.yaml index fd1aab74dd..4069c5d663 100644 --- a/.github/workflows/k8s-native-stress.yaml +++ b/.github/workflows/k8s-native-stress.yaml @@ -322,6 +322,11 @@ jobs: set +e NAMESPACE=simplyblock + # Run the operator's own cleanup script first (thorough helm + CR cleanup) + if [ -x "$GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh" ]; then + bash $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh -f $NAMESPACE || true + fi + # Run the shared cleanup script (handles etcd overload with # --request-timeout, bulk deletes, and parallel finalizer patching) bash $GITHUB_WORKSPACE/e2e/scripts/cleanup_k8s.sh $NAMESPACE @@ -354,10 +359,10 @@ jobs: CLUSTER_ENV="${{ github.event.inputs.cluster_environment || 'local' }}" IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" for NODE in "${NODES[@]}"; do - echo "Resetting hugepages to 0 on $NODE..." + echo "Disconnecting stale NVMe-oF and resetting hugepages on $NODE..." if [[ "$CLUSTER_ENV" == *"openshift"* ]]; then oc debug node/"$NODE" -- chroot /host bash -c \ - "echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" 2>/dev/null || true echo "Restarting kubelet on $NODE..." oc debug node/"$NODE" -- chroot /host systemctl restart kubelet 2>/dev/null || true else @@ -405,6 +410,19 @@ jobs: kubectl wait --for=delete namespace/vault --timeout=120s 2>/dev/null || true echo "=== KMS cleanup complete ===" + # ── Remove stale labels from previous runs ────────────────────────────── + + - name: Remove stale storagenodeset labels from all worker nodes + if: ${{ github.event.inputs.use_existing_cluster != 'true' }} + run: | + echo "Removing stale io.simplyblock.storagenodeset label from all worker nodes..." + IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" + for NODE in "${NODES[@]}"; do + echo "Removing storagenodeset label from $NODE" + kubectl label node "$NODE" io.simplyblock.storagenodeset- 2>/dev/null || true + done + echo "Stale label cleanup complete." + # ── Label nodes (after cluster is active) ─────────────────────────────── - name: Label worker nodes @@ -480,6 +498,7 @@ jobs: - name: Install Helm Chart for simplyblock-operator if: ${{ github.event.inputs.use_existing_cluster != 'true' }} run: | + set -euxo pipefail cd $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator/ TLS_FLAGS="" @@ -688,6 +707,12 @@ jobs: RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' fi + PCIE_MODEL_YAML="" + CLUSTER_ENV="${{ github.event.inputs.cluster_environment }}" + if [ "$CLUSTER_ENV" != "openshift-baremetal" ] && [ "$CLUSTER_ENV" != "openshift-local" ]; then + PCIE_MODEL_YAML=' pcieModel: "SAMSUNG MZQLB1T9HAJR-00007"' + fi + # Build workerNodes YAML list from comma-separated input WORKER_YAML="" IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" @@ -755,7 +780,7 @@ jobs: clusterImage: "${SB_REPO}:${SB_TAG}" spdkImage: "${SPDK_IMAGE}" spdkProxyImage: "${SB_REPO}:${SB_TAG}" - pcieModel: "SAMSUNG MZQLB1T9HAJR-00007" + ${PCIE_MODEL_YAML} driveSizeRange: "${DRIVE_SIZE_RANGE}" mgmtIfname: ${MGMT_IFC} dataIfname: @@ -818,7 +843,7 @@ jobs: clusterImage: "${SB_REPO}:${SB_TAG}" spdkImage: "${SPDK_IMAGE}" spdkProxyImage: "${SB_REPO}:${SB_TAG}" - pcieModel: "SAMSUNG MZQLB1T9HAJR-00007" + ${PCIE_MODEL_YAML} driveSizeRange: "${DRIVE_SIZE_RANGE}" mgmtIfname: ${MGMT_IFC} dataIfname: diff --git a/.github/workflows/k8s-native-upgrade.yaml b/.github/workflows/k8s-native-upgrade.yaml index 90627ade9c..efce767da7 100644 --- a/.github/workflows/k8s-native-upgrade.yaml +++ b/.github/workflows/k8s-native-upgrade.yaml @@ -39,9 +39,9 @@ on: required: false default: '' - # ── Operator repo branches (base,target — e.g. "main,main" or "release-25,main") ── - operator_repo_branches: - description: 'Operator repo branches: base_branch,target_branch (e.g. main,main)' + # ── Helm repo branches (base,target — e.g. "main,main" or "release-25,main") ── + helm_repo_branches: + description: 'Helm chart repo branches: base_branch,target_branch (R25: base has CSI chart, R26+: base has operator chart)' required: true default: 'main,main' @@ -74,9 +74,9 @@ on: required: false default: 'br-ex:enp2s0f0' cluster_params: - description: 'Cluster config: ndcs=2,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4,max_lvol=30' + description: 'Cluster config: ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_lvol=30' required: false - default: 'ndcs=2,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4,max_lvol=30' + default: 'ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_lvol=30' cluster_environment: description: 'Target cluster environment' required: true @@ -159,8 +159,8 @@ jobs: run: | set -euo pipefail - # Parse operator_repo_branches (format: "base_branch,target_branch") - IFS=',' read -r BASE_HELM TARGET_HELM <<< "${{ github.event.inputs.operator_repo_branches || 'main,main' }}" + # Parse helm_repo_branches (format: "base_branch,target_branch") + IFS=',' read -r BASE_HELM TARGET_HELM <<< "${{ github.event.inputs.helm_repo_branches || 'main,main' }}" TARGET_HELM="${TARGET_HELM:-$BASE_HELM}" echo "BASE_HELM_BRANCH=${BASE_HELM}" >> "$GITHUB_ENV" echo "TARGET_HELM_BRANCH=${TARGET_HELM}" >> "$GITHUB_ENV" @@ -239,7 +239,7 @@ jobs: # ── Parse cluster params ── - name: Parse cluster parameters run: | - PARAMS="${{ github.event.inputs.cluster_params || 'ndcs=2,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4' }}" + PARAMS="${{ github.event.inputs.cluster_params || 'ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_lvol=30' }}" for kv in $(echo "$PARAMS" | tr ',' '\n'); do key=$(echo "$kv" | cut -d= -f1 | tr '[:lower:]' '[:upper:]') val=$(echo "$kv" | cut -d= -f2) @@ -260,6 +260,145 @@ jobs: # ── Common steps (both upgrade types) ── + - name: Setup kubeconfig + run: | + mkdir -p $HOME/.kube + KUBECONFIG_FILE="$HOME/.kube/config-${{ github.run_id }}" + echo "${KUBECONFIG_DATA}" > "$KUBECONFIG_FILE" + chmod 600 "$KUBECONFIG_FILE" + echo "KUBECONFIG=$KUBECONFIG_FILE" >> $GITHUB_ENV + env: + KUBECONFIG_DATA: ${{ + github.event.inputs.cluster_environment == 'aws-openshift' && secrets.KUBECONFIG_AWS_OPENSHIFT || + github.event.inputs.cluster_environment == 'openshift-local' && secrets.KUBECONFIG_OPENSHIFT_LOCAL || + github.event.inputs.cluster_environment == 'openshift-baremetal' && secrets.KUBECONFIG_OPENSHIFT_BM || + github.event.inputs.cluster_environment == 'gcp' && secrets.KUBECONFIG_GCP || + secrets.KUBECONFIG_LOCAL || secrets.KUBECONFIG_CONTENT + }} + + - name: Verify kubectl connectivity + run: kubectl get nodes + + - name: Force cleanup old deployment + if: ${{ github.event.inputs.use_existing_cluster != 'true' }} + timeout-minutes: 15 + run: | + set +e + NAMESPACE=simplyblock + CLUSTER_ENV="${{ github.event.inputs.cluster_environment || 'local' }}" + + echo "=== Phase 1: Uninstall all Helm releases (R25 + R26) ===" + # Uninstall known R25 charts explicitly (they won't be in cleanup scripts) + helm uninstall spdk-csi -n $NAMESPACE --no-hooks --timeout 60s 2>/dev/null || true + helm uninstall sbcli -n $NAMESPACE --no-hooks --timeout 60s 2>/dev/null || true + # Uninstall any remaining releases (R26 operator, etc.) + for rel in $(helm list -n $NAMESPACE -q 2>/dev/null); do + echo "Uninstalling Helm release: $rel" + helm uninstall "$rel" -n $NAMESPACE --no-hooks --timeout 60s 2>/dev/null || true + done + helm uninstall cert-manager -n cert-manager --no-hooks --timeout 60s 2>/dev/null || true + + echo "=== Phase 2: Delete simplyblock resources in kube-system ===" + # The spdk-csi chart installs simplyblock-snapshot-controller in kube-system + # with helm.sh/resource-policy: keep, so it survives helm uninstall and + # blocks future installs (both R25 spdk-csi and R26 operator). + for RTYPE in deployment service sa configmap; do + for NAME in $(kubectl -n kube-system get $RTYPE --no-headers -o custom-columns=:metadata.name 2>/dev/null | grep -i simplyblock 2>/dev/null); do + echo "Deleting kube-system $RTYPE/$NAME" + kubectl -n kube-system delete $RTYPE "$NAME" --ignore-not-found 2>/dev/null || true + done + done + + echo "=== Phase 3: Run shared cleanup scripts ===" + # Run the operator's own cleanup script (thorough helm + CR cleanup) + if [ -x "$GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh" ]; then + bash $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh -f $NAMESPACE || true + fi + # Run the shared cleanup script (handles etcd overload with + # --request-timeout, bulk deletes, parallel finalizer patching, + # kube-system resources, StorageClasses, webhooks, etc.) + bash $GITHUB_WORKSPACE/e2e/scripts/cleanup_k8s.sh $NAMESPACE + + echo "=== Delete Released PVs from simplyblock ===" + for pv in $(kubectl get pv --no-headers 2>/dev/null | grep 'simplyblock/' | awk '{print $1}'); do + echo "Deleting PV $pv" + kubectl delete pv "$pv" --ignore-not-found 2>/dev/null || true + done + + echo "=== Delete CRDs ===" + # Clear finalizers on all remaining CR instances so CRD deletion doesn't hang + for crd in $(kubectl get crd -o name 2>/dev/null | grep simplyblock); do + crd_name="${crd#customresourcedefinition.apiextensions.k8s.io/}" + for cr in $(kubectl get "$crd_name" -A --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null); do + IFS=' ' read -r ns name <<< "$cr" + kubectl patch "$crd_name" "$name" -n "$ns" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + done + kubectl patch crd "$crd_name" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + done + kubectl delete -f $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator/crds/ --ignore-not-found --timeout=60s 2>/dev/null || true + # Force-delete any CRDs that survived the timeout + for crd in $(kubectl get crd -o name 2>/dev/null | grep simplyblock); do + crd_name="${crd#customresourcedefinition.apiextensions.k8s.io/}" + kubectl patch crd "$crd_name" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + kubectl delete crd "$crd_name" --ignore-not-found --timeout=30s 2>/dev/null || true + done + + echo "=== Force delete + finalize namespaces ===" + kubectl delete namespace $NAMESPACE --force --grace-period=0 --wait=false 2>/dev/null || true + kubectl delete namespace cert-manager --force --grace-period=0 --wait=false 2>/dev/null || true + for ns in $NAMESPACE cert-manager; do + kubectl get namespace $ns -o json 2>/dev/null \ + | jq '.spec.finalizers = []' \ + | kubectl replace --raw "/api/v1/namespaces/$ns/finalize" -f - 2>/dev/null || true + done + + # Delete cert-manager CRDs + kubectl delete crd certificaterequests.cert-manager.io certificates.cert-manager.io \ + challenges.acme.cert-manager.io clusterissuers.cert-manager.io \ + issuers.cert-manager.io orders.acme.cert-manager.io \ + --ignore-not-found --force --grace-period=0 2>/dev/null || true + + echo "=== Disconnect stale NVMe-oF, reset hugepages + restart kubelet ===" + IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" + for NODE in "${NODES[@]}"; do + ( + if [[ "$CLUSTER_ENV" == *"openshift"* ]]; then + timeout 90 oc debug node/"$NODE" -- chroot /host bash -c \ + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages && systemctl restart kubelet" 2>/dev/null || true + else + timeout 90 kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages && systemctl restart kubelet" 2>/dev/null || true + fi + echo "Done: $NODE" + ) & + done + wait + + echo "=== Verify namespace gone ===" + for i in $(seq 1 4); do + if ! kubectl get namespace $NAMESPACE &>/dev/null; then + echo "Namespace $NAMESPACE deleted" + break + fi + echo "Namespace still terminating ($i/4), force-finalizing..." + kubectl get namespace $NAMESPACE -o json 2>/dev/null \ + | jq '.spec.finalizers = []' \ + | kubectl replace --raw "/api/v1/namespaces/$NAMESPACE/finalize" -f - 2>/dev/null || true + sleep 5 + done + echo "=== Cleanup complete ===" + + - name: Remove stale storagenodeset labels from all worker nodes + if: ${{ github.event.inputs.use_existing_cluster != 'true' }} + run: | + echo "Removing stale io.simplyblock.storagenodeset label from all worker nodes..." + IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" + for NODE in "${NODES[@]}"; do + echo "Removing storagenodeset label from $NODE" + kubectl label node "$NODE" io.simplyblock.storagenodeset- 2>/dev/null || true + done + echo "Stale label cleanup complete." + - name: Cleanup stale CSI hostpath data on worker nodes if: ${{ github.event.inputs.use_existing_cluster != 'true' }} run: | @@ -278,7 +417,7 @@ jobs: echo "CSI hostpath data cleanup complete." - name: Install cert-manager (TLS prerequisite) - if: ${{ github.event.inputs.use_existing_cluster != 'true' && github.event.inputs.tls_enabled == 'true' }} + if: ${{ github.event.inputs.use_existing_cluster != 'true' && github.event.inputs.tls_enabled == 'true' && github.event.inputs.upgrade_type != 'r25-to-r2x' }} run: | helm repo add jetstack https://charts.jetstack.io helm repo update @@ -295,6 +434,7 @@ jobs: - name: Install Helm Chart for simplyblock-operator (BASE version) if: ${{ github.event.inputs.use_existing_cluster != 'true' && github.event.inputs.upgrade_type != 'r25-to-r2x' }} run: | + set -euxo pipefail cd $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator/ TLS_FLAGS="" @@ -396,6 +536,11 @@ jobs: RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' fi + PCIE_MODEL_YAML="" + if [ "$CLUSTER_ENV" != "openshift-baremetal" ] && [ "$CLUSTER_ENV" != "openshift-local" ]; then + PCIE_MODEL_YAML=' pcieModel: "SAMSUNG MZQLB1T9HAJR-00007"' + fi + # Build vault settings (conditional on TLS) VAULT_SETTINGS="" if [ "${{ github.event.inputs.tls_enabled }}" = "true" ]; then @@ -444,7 +589,7 @@ jobs: clusterImage: "${SB_REPO}:${SB_TAG}" spdkImage: "${SPDK_IMAGE}" spdkProxyImage: "${SB_REPO}:${SB_TAG}" - pcieModel: "SAMSUNG MZQLB1T9HAJR-00007" + ${PCIE_MODEL_YAML} driveSizeRange: "${DRIVE_SIZE_RANGE}" mgmtIfname: ${MGMT_IFC} dataIfname: @@ -561,10 +706,12 @@ jobs: set -euxo pipefail echo "=== Installing R25 sbcli control plane chart ===" cd $GITHUB_WORKSPACE/sbcli-r25/simplyblock_core/scripts/charts/ + helm dependency build . helm upgrade --install sbcli \ --namespace simplyblock \ --create-namespace \ --timeout 10m \ + --set ingress-nginx.controller.admissionWebhooks.enabled=false \ ./ echo "sbcli control plane chart installed" @@ -605,7 +752,7 @@ jobs: ADMIN_POD="" for i in $(seq 1 30); do ADMIN_POD=$(kubectl -n $NAMESPACE get pods --no-headers 2>/dev/null \ - | grep -i "webappapi\|admin\|sbcli" \ + | grep -i "admin-control\|webappapi" \ | grep "Running" | head -1 | awk '{print $1}') || true if [ -n "$ADMIN_POD" ]; then echo "Found admin pod: $ADMIN_POD" @@ -628,6 +775,26 @@ jobs: exit 1 fi + echo "=== Waiting for FDB to be ready ===" + for i in $(seq 1 30); do + FDB_STATUS=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbcli-dev cluster list 2>&1) || true + if echo "$FDB_STATUS" | grep -qiE 'Connection.*invalid|Error.*reading.*FDB|not found'; then + echo "FDB not ready yet ($i/30): $(echo "$FDB_STATUS" | head -1)" + sleep 10 + else + echo "FDB appears ready" + break + fi + if [ "$i" -eq 30 ]; then + echo "ERROR: FDB not ready after 5 minutes" + echo "$FDB_STATUS" + kubectl -n $NAMESPACE get pods + kubectl -n $NAMESPACE logs "$ADMIN_POD" --tail=50 2>/dev/null || true + exit 1 + fi + done + echo "=== Creating R25 cluster ===" CREATE_OUTPUT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ sbcli-dev -d --dev cluster create \ @@ -648,24 +815,51 @@ jobs: sbcli-dev cluster list 2>&1) || true echo "Cluster list output: $LIST_OUTPUT" CLUSTER_ID=$(echo "$LIST_OUTPUT" | awk 'NR==4{print $2}') - CLUSTER_SECRET=$(echo "$LIST_OUTPUT" | awk 'NR==4{print $NF}') - else + fi + + if [ -z "$CLUSTER_ID" ] || [ "$CLUSTER_ID" = "+" ]; then + echo "ERROR: Failed to parse cluster ID" + kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbcli-dev cluster list 2>&1 || true + exit 1 + fi + + # Get cluster secret using dedicated command (disable tracing to avoid leaking secret) + set +x + CLUSTER_SECRET=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbcli-dev cluster get-secret "$CLUSTER_ID" 2>/dev/null | tr -d '[:space:]') || true + if [ -z "$CLUSTER_SECRET" ]; then + echo "WARNING: Failed to get cluster secret via get-secret, trying JSON fallback" CLUSTER_SECRET=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ sbcli-dev cluster list --json 2>/dev/null \ | jq -r '.[0].secret // empty') || true fi - if [ -z "$CLUSTER_ID" ] || [ "$CLUSTER_ID" = "+" ]; then - echo "ERROR: Failed to parse cluster ID" + # Validate the secret — should be a short alphanumeric token, not an error dump + if [ -z "$CLUSTER_SECRET" ] || [ "${#CLUSTER_SECRET}" -gt 100 ]; then + echo "ERROR: CLUSTER_SECRET is empty or invalid (length=${#CLUSTER_SECRET})" + echo "This usually means FDB is not reachable from the admin pod." kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbcli-dev cluster list 2>&1 || true exit 1 fi + echo "::add-mask::${CLUSTER_SECRET}" + set -x echo "R25 cluster created: CLUSTER_ID=$CLUSTER_ID" echo "CLUSTER_ID=${CLUSTER_ID}" >> $GITHUB_ENV echo "CLUSTER_SECRET=${CLUSTER_SECRET}" >> $GITHUB_ENV echo "R25_ADMIN_POD=${ADMIN_POD}" >> $GITHUB_ENV + - name: Label worker nodes for R25 storage plane + if: ${{ github.event.inputs.use_existing_cluster != 'true' && github.event.inputs.upgrade_type == 'r25-to-r2x' }} + run: | + set -euxo pipefail + echo "=== Labeling worker nodes for R25 storage plane ===" + IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" + for NODE in "${NODES[@]}"; do + kubectl label node "$NODE" io.simplyblock.node-type=simplyblock-storage-plane --overwrite + echo "Labeled $NODE with io.simplyblock.node-type=simplyblock-storage-plane" + done + - name: Install spdk-csi chart for R25 storage nodes if: ${{ github.event.inputs.use_existing_cluster != 'true' && github.event.inputs.upgrade_type == 'r25-to-r2x' }} run: | @@ -673,9 +867,21 @@ jobs: NAMESPACE=simplyblock echo "=== Installing R25 spdk-csi chart ===" - cd $GITHUB_WORKSPACE/simplyblock-operator/csi-driver/charts/spdk-csi/latest/spdk-csi/ + # v0.2.4 has charts/ at root; main moved it to csi-driver/charts/ + if [ -d "$GITHUB_WORKSPACE/simplyblock-operator/csi-driver/charts/spdk-csi/latest/spdk-csi" ]; then + cd $GITHUB_WORKSPACE/simplyblock-operator/csi-driver/charts/spdk-csi/latest/spdk-csi/ + elif [ -d "$GITHUB_WORKSPACE/simplyblock-operator/charts/spdk-csi/latest/spdk-csi" ]; then + cd $GITHUB_WORKSPACE/simplyblock-operator/charts/spdk-csi/latest/spdk-csi/ + else + echo "ERROR: Cannot find spdk-csi chart directory" + ls -la $GITHUB_WORKSPACE/simplyblock-operator/ + ls -la $GITHUB_WORKSPACE/simplyblock-operator/charts/ 2>/dev/null || true + ls -la $GITHUB_WORKSPACE/simplyblock-operator/csi-driver/ 2>/dev/null || true + exit 1 + fi + helm dependency build . 2>/dev/null || true - helm install -n $NAMESPACE spdk-csi ./ \ + helm install -n $NAMESPACE --create-namespace spdk-csi ./ \ --set csiConfig.simplybk.uuid="${CLUSTER_ID}" \ --set csiConfig.simplybk.ip="http://simplyblock-webappapi.simplyblock:5000" \ --set csiSecret.simplybk.secret="${CLUSTER_SECRET}" \ @@ -687,10 +893,10 @@ jobs: --set storageclass.volumeBindingMode=Immediate \ --set cachingnode.create=false \ --set logicalVolume.encryption=false \ - --set logicalVolume.snapshot="True" \ --set storagenode.ifname="${MGMT_IFC}" \ --set storagenode.create=true \ - --set storagenode.numPartitions="${PARTITIONS}" \ + --set storagenode.numPartitions=1 \ + --set storagenode.coresPercentage=50 \ --set image.storageNode.tag="${STORAGENODE_TAG}" echo "spdk-csi chart installed" @@ -739,7 +945,7 @@ jobs: # Re-resolve admin pod (it may have restarted) for i in $(seq 1 30); do NEW_POD=$(kubectl -n $NAMESPACE get pods --no-headers 2>/dev/null \ - | grep -i "webappapi\|admin\|sbcli" \ + | grep -i "admin-control\|webappapi" \ | grep "Running" | head -1 | awk '{print $1}') || true if [ -n "$NEW_POD" ]; then ADMIN_POD="$NEW_POD" @@ -748,6 +954,41 @@ jobs: sleep 10 done + echo "=== Waiting for storage nodes to register ===" + CLUSTER_ID="${CLUSTER_ID:-}" + if [ -z "$CLUSTER_ID" ]; then + CLUSTER_ID=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbcli-dev cluster list --json 2>/dev/null \ + | jq -r '.[0].id // .[0].uuid // empty') || true + fi + for i in $(seq 1 60); do + SN_COUNT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbcli-dev sn list --json 2>/dev/null | jq 'length' 2>/dev/null || echo "0") + echo "Storage nodes registered: $SN_COUNT/$EXPECTED_SNODES ($i/60)" + if [ "$SN_COUNT" -ge "$EXPECTED_SNODES" ]; then + echo "All storage nodes registered" + break + fi + sleep 10 + done + + echo "=== Waiting for all storage nodes to be online ===" + for i in $(seq 1 60); do + ONLINE_COUNT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbcli-dev sn list --json 2>/dev/null \ + | jq '[.[] | select(.status == "online")] | length' 2>/dev/null || echo "0") + echo "Storage nodes online: $ONLINE_COUNT/$EXPECTED_SNODES ($i/60)" + if [ "$ONLINE_COUNT" -ge "$EXPECTED_SNODES" ]; then + echo "All storage nodes are online" + break + fi + sleep 10 + done + + echo "=== Activating R25 cluster (no auto-activate in R25) ===" + kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbcli-dev cluster activate "$CLUSTER_ID" 2>&1 || true + echo "=== Polling cluster status (admin pod: $ADMIN_POD) ===" for i in $(seq 1 $MAX_POLL); do OUTPUT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ @@ -756,15 +997,28 @@ jobs: if echo "$OUTPUT" | grep -qi "active"; then echo "R25 cluster is active!" - # Re-capture cluster ID/secret in case they changed + # Re-capture cluster ID CLUSTER_ID=$(echo "$OUTPUT" | awk 'NR==4{print $2}') - CLUSTER_SECRET=$(echo "$OUTPUT" | awk 'NR==4{print $NF}') if [ -z "$CLUSTER_ID" ] || [ "$CLUSTER_ID" = "+" ]; then JSON_OUT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ sbcli-dev cluster list --json 2>&1) || true CLUSTER_ID=$(echo "$JSON_OUT" | jq -r '.[0].id // .[0].uuid // empty') - CLUSTER_SECRET=$(echo "$JSON_OUT" | jq -r '.[0].secret // empty') fi + # Get cluster secret using dedicated command (disable tracing to avoid leaking secret) + set +x + CLUSTER_SECRET=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbcli-dev cluster get-secret "$CLUSTER_ID" 2>/dev/null | tr -d '[:space:]') || true + if [ -z "$CLUSTER_SECRET" ]; then + CLUSTER_SECRET=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbcli-dev cluster list --json 2>/dev/null \ + | jq -r '.[0].secret // empty') || true + fi + if [ -z "$CLUSTER_SECRET" ] || [ "${#CLUSTER_SECRET}" -gt 100 ]; then + echo "ERROR: CLUSTER_SECRET is empty or invalid (length=${#CLUSTER_SECRET})" + exit 1 + fi + echo "::add-mask::${CLUSTER_SECRET}" + set -x echo "CLUSTER_ID=${CLUSTER_ID}" >> $GITHUB_ENV echo "CLUSTER_SECRET=${CLUSTER_SECRET}" >> $GITHUB_ENV exit 0 @@ -777,6 +1031,7 @@ jobs: exit 1 env: R25_ADMIN_POD: ${{ env.R25_ADMIN_POD }} + CLUSTER_ID: ${{ env.CLUSTER_ID }} # ── Wait for cluster readiness (operator-based bootstrap only) ── @@ -877,13 +1132,26 @@ jobs: if echo "$OUTPUT" | grep -qi "active"; then echo "Cluster is active!" CLUSTER_ID=$(echo "$OUTPUT" | awk 'NR==4{print $2}') - CLUSTER_SECRET=$(echo "$OUTPUT" | awk 'NR==4{print $NF}') if [ -z "$CLUSTER_ID" ] || [ "$CLUSTER_ID" = "+" ]; then JSON_OUT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ sbctl cluster list --json 2>&1) || true CLUSTER_ID=$(echo "$JSON_OUT" | jq -r '.[0].id // .[0].uuid // empty') - CLUSTER_SECRET=$(echo "$JSON_OUT" | jq -r '.[0].secret // empty') fi + # Get cluster secret using dedicated command + set +x + CLUSTER_SECRET=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbctl cluster get-secret "$CLUSTER_ID" 2>/dev/null | tr -d '[:space:]') || true + if [ -z "$CLUSTER_SECRET" ]; then + CLUSTER_SECRET=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbctl cluster list --json 2>/dev/null \ + | jq -r '.[0].secret // empty') || true + fi + if [ -z "$CLUSTER_SECRET" ] || [ "${#CLUSTER_SECRET}" -gt 100 ]; then + echo "ERROR: CLUSTER_SECRET is empty or invalid (length=${#CLUSTER_SECRET})" + exit 1 + fi + echo "::add-mask::${CLUSTER_SECRET}" + set -x echo "CLUSTER_ID=${CLUSTER_ID}" >> $GITHUB_ENV echo "CLUSTER_SECRET=${CLUSTER_SECRET}" >> $GITHUB_ENV exit 0 @@ -930,6 +1198,7 @@ jobs: export CSI_TAG="${CSI_TAG}" export WORKER_NODES="${WORKER_NODES}" export HELM_RELEASE_SBCLI="${HELM_RELEASE_SBCLI}" + export R25_CHART_PATH="${R25_CHART_PATH}" TESTNAME_ARG="" if [ -n "${TESTNAME}" ]; then @@ -960,6 +1229,7 @@ jobs: WORKER_NODES: ${{ github.event.inputs.worker_nodes }} TESTNAME: ${{ github.event.inputs.testname || 'K8sNativeMajorUpgrade' }} HELM_RELEASE_SBCLI: ${{ github.event.inputs.upgrade_type == 'r25-to-r2x' && 'sbcli' || '' }} + R25_CHART_PATH: ${{ github.event.inputs.upgrade_type == 'r25-to-r2x' && format('{0}/sbcli-r25/simplyblock_core/scripts/charts/', github.workspace) || '' }} NDCS: ${{ env.NDCS }} NPCS: ${{ env.NPCS }} SUPABASE_ANON_KEY: ${{ secrets.SUPABASE_ANON_KEY }} @@ -1305,3 +1575,10 @@ jobs: env: CLUSTER_ID: ${{ env.CLUSTER_ID }} MON_SECRET: ${{ secrets.MON_SECRET }} + + - name: Cleanup build folder + if: always() + run: | + rm -rf ./* || true + rm -rf ./.??* || true + rm -f "$HOME/.kube/config-${{ github.run_id }}" diff --git a/.github/workflows/monitoring-suite-docker.yaml b/.github/workflows/monitoring-suite-docker.yaml index 538863374b..836d759ec1 100755 --- a/.github/workflows/monitoring-suite-docker.yaml +++ b/.github/workflows/monitoring-suite-docker.yaml @@ -612,6 +612,35 @@ jobs: echo "TEST_START_EPOCH=$(date +%s)" >> "$GITHUB_ENV" echo "TEST_START_HUMAN=$(date -u +'%Y-%m-%d %H:%M:%S UTC')" >> "$GITHUB_ENV" + - name: Start MinIO trace logging + if: ${{ env.CLUSTER_SECURITY == 'backup' }} + shell: bash + run: | + set -euxo pipefail + + # 1. Install mc (MinIO Client) + if ! command -v mc &>/dev/null; then + if curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc 2>/dev/null; then + chmod +x /usr/local/bin/mc + else + curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc + chmod +x /tmp/mc + export PATH="/tmp:$PATH" + echo "/tmp" >> "$GITHUB_PATH" + fi + fi + + # 2. Configure mc alias to external MinIO + mc alias set myminio http://192.168.10.164:9000 "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" + + # 3. Start admin trace in background + MINIO_TRACE_LOG="/tmp/minio-trace-${GITHUB_RUN_ID}.log" + echo "MINIO_TRACE_LOG=${MINIO_TRACE_LOG}" >> "$GITHUB_ENV" + nohup mc admin trace -v --all myminio > "${MINIO_TRACE_LOG}" 2>&1 & + echo "MC_TRACE_PID=$!" >> "$GITHUB_ENV" + + echo "=== MinIO trace logging started (PID=$!, log=${MINIO_TRACE_LOG}) ===" + - name: "Run monitoring test: ${{ matrix.test }}" shell: bash working-directory: sbcli/e2e @@ -965,6 +994,25 @@ jobs: path: monitoring_results/${{ matrix.test }}/ if-no-files-found: warn + - name: Stop MinIO trace logging and save + if: always() + shell: bash + run: | + set +e + if [ -n "${MC_TRACE_PID:-}" ]; then + echo "Stopping MinIO trace (PID=${MC_TRACE_PID})..." + kill "${MC_TRACE_PID}" 2>/dev/null || true + wait "${MC_TRACE_PID}" 2>/dev/null || true + fi + if [ -f "${MINIO_TRACE_LOG:-}" ]; then + LINES=$(wc -l < "${MINIO_TRACE_LOG}") + SIZE=$(du -h "${MINIO_TRACE_LOG}" | cut -f1) + echo "=== MinIO trace log: ${LINES} lines, ${SIZE} ===" + cp "${MINIO_TRACE_LOG}" sbcli/e2e/minio-trace.log 2>/dev/null || true + else + echo "No MinIO trace log found (backup tests may not have run)" + fi + - name: Upload bootstrap + test logs if: always() uses: actions/upload-artifact@v4 @@ -973,6 +1021,7 @@ jobs: path: | simplyBlockDeploy/bare-metal/bootstrap.log sbcli/e2e/output.log + sbcli/e2e/minio-trace.log sbcli/e2e/logs/** if-no-files-found: warn diff --git a/.github/workflows/monitoring-suite-k8s-native.yaml b/.github/workflows/monitoring-suite-k8s-native.yaml index 8cc4a6f696..60076d9bd7 100644 --- a/.github/workflows/monitoring-suite-k8s-native.yaml +++ b/.github/workflows/monitoring-suite-k8s-native.yaml @@ -336,6 +336,11 @@ jobs: set +e NAMESPACE=simplyblock + # Run the operator's own cleanup script first (thorough helm + CR cleanup) + if [ -x "$GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh" ]; then + bash $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh -f $NAMESPACE || true + fi + # Run the shared cleanup script (handles etcd overload with # --request-timeout, bulk deletes, and parallel finalizer patching) bash $GITHUB_WORKSPACE/e2e/scripts/cleanup_k8s.sh $NAMESPACE @@ -628,6 +633,12 @@ jobs: RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' fi + PCIE_MODEL_YAML="" + CLUSTER_ENV="${{ github.event.inputs.cluster_environment }}" + if [ "$CLUSTER_ENV" != "openshift-baremetal" ] && [ "$CLUSTER_ENV" != "openshift-local" ]; then + PCIE_MODEL_YAML=' pcieModel: "SAMSUNG MZQLB1T9HAJR-00007"' + fi + WORKER_YAML="" IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" for NODE in "${NODES[@]}"; do @@ -692,7 +703,7 @@ jobs: clusterImage: "${SB_REPO}:${SB_TAG}" spdkImage: "${SPDK_IMAGE}" spdkProxyImage: "${SB_REPO}:${SB_TAG}" - pcieModel: "SAMSUNG MZQLB1T9HAJR-00007" + ${PCIE_MODEL_YAML} driveSizeRange: "${DRIVE_SIZE_RANGE}" mgmtIfname: ${MGMT_IFC} dataIfname: diff --git a/.github/workflows/stress-run-bootstrap.yml b/.github/workflows/stress-run-bootstrap.yml index 8e2e42adc3..75c72d359e 100755 --- a/.github/workflows/stress-run-bootstrap.yml +++ b/.github/workflows/stress-run-bootstrap.yml @@ -34,7 +34,7 @@ on: SBCLI_BRANCH: description: "sbcli repo branch to clone for tests" required: true - default: "R25.10-Hotfix" + default: "main" CUSTOM_IMAGES: description: "Image overrides: set spdk and/or docker values, leave as \"\" to skip." @@ -736,6 +736,35 @@ jobs: echo "TEST_START_EPOCH=$(date +%s)" >> "$GITHUB_ENV" echo "TEST_START_HUMAN=$(date -u +'%Y-%m-%d %H:%M:%S UTC')" >> "$GITHUB_ENV" + - name: Start MinIO trace logging + if: ${{ env.CLUSTER_SECURITY == 'backup' }} + shell: bash + run: | + set -euxo pipefail + + # 1. Install mc (MinIO Client) + if ! command -v mc &>/dev/null; then + if curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc 2>/dev/null; then + chmod +x /usr/local/bin/mc + else + curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc + chmod +x /tmp/mc + export PATH="/tmp:$PATH" + echo "/tmp" >> "$GITHUB_PATH" + fi + fi + + # 2. Configure mc alias to external MinIO + mc alias set myminio http://192.168.10.164:9000 "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" + + # 3. Start admin trace in background + MINIO_TRACE_LOG="/tmp/minio-trace-${GITHUB_RUN_ID}.log" + echo "MINIO_TRACE_LOG=${MINIO_TRACE_LOG}" >> "$GITHUB_ENV" + nohup mc admin trace -v --all myminio > "${MINIO_TRACE_LOG}" 2>&1 & + echo "MC_TRACE_PID=$!" >> "$GITHUB_ENV" + + echo "=== MinIO trace logging started (PID=$!, log=${MINIO_TRACE_LOG}) ===" + - name: Enable shared placement shell: bash run: | @@ -1545,6 +1574,25 @@ jobs: else echo "All post-test collection steps succeeded." fi + - name: Stop MinIO trace logging and save + if: always() + shell: bash + run: | + set +e + if [ -n "${MC_TRACE_PID:-}" ]; then + echo "Stopping MinIO trace (PID=${MC_TRACE_PID})..." + kill "${MC_TRACE_PID}" 2>/dev/null || true + wait "${MC_TRACE_PID}" 2>/dev/null || true + fi + if [ -f "${MINIO_TRACE_LOG:-}" ]; then + LINES=$(wc -l < "${MINIO_TRACE_LOG}") + SIZE=$(du -h "${MINIO_TRACE_LOG}" | cut -f1) + echo "=== MinIO trace log: ${LINES} lines, ${SIZE} ===" + cp "${MINIO_TRACE_LOG}" sbcli/e2e/minio-trace.log 2>/dev/null || true + else + echo "No MinIO trace log found (backup tests may not have run)" + fi + - name: Upload logs (always) if: always() uses: actions/upload-artifact@v4 @@ -1553,6 +1601,7 @@ jobs: path: | simplyBlockDeploy/bare-metal/bootstrap.log sbcli/e2e/output.log + sbcli/e2e/minio-trace.log sbcli/e2e/logs/** if-no-files-found: warn diff --git a/.github/workflows/topology-suite-docker.yml b/.github/workflows/topology-suite-docker.yml index e8b82770be..64edce2921 100755 --- a/.github/workflows/topology-suite-docker.yml +++ b/.github/workflows/topology-suite-docker.yml @@ -322,14 +322,21 @@ jobs: if not os.path.isfile(md_path): continue content = open(md_path).read() + # Extract test name — try "Summary — " first, + # fall back to "Test class: ``" (e2e-bootstrap format) name_m = re.search(r'Summary\s*[-—]\s*(\S+)', content) + if not name_m: + name_m = re.search(r'\*\*Test class:\*\*\s*`([^`]+)`', content) tname = name_m.group(1) if name_m else d - res_m = re.search(r'\*\*Result:\*\*\s*(\S+)', content) - result = res_m.group(1) if res_m else "?" - r_icon = ":white_check_mark:" if "pass" in result.lower() or "success" in result.lower() else ":x:" - total_m = re.search(r'\*\*Total:\*\*\s*(\d+)', content) - passed_m = re.search(r'\*\*Passed:\*\*\s*(\d+)', content) - failed_m = re.search(r'\*\*Failed:\*\*\s*(\d+)', content) + # Extract result — handle both "SUCCESS"/"FAILED" and emoji prefixed "✅ SUCCESS" + res_m = re.search(r'\*\*Result:\*\*\s*(.+?)(?:\s*[&<\n]|$)', content) + result = res_m.group(1).strip() if res_m else "?" + r_icon = ":white_check_mark:" if "success" in result.lower() else ":x:" + # Extract counts — handle both inline "**Total:** N" and + # table format "| ✅ Passed | N |" from e2e-bootstrap + total_m = re.search(r'\*\*Total[:\*]*\*?\*?\s*\|?\s*\*?\*?(\d+)', content) + passed_m = re.search(r'(?:\*\*Passed:\*\*|[✅] Passed)\s*\|?\s*(\d+)', content) + failed_m = re.search(r'(?:\*\*Failed:\*\*|[❌] Failed)\s*\|?\s*(\d+)', content) total = total_m.group(1) if total_m else "?" passed = passed_m.group(1) if passed_m else "?" failed = failed_m.group(1) if failed_m else "?" diff --git a/.github/workflows/topology-suite-k8s-add-node.yml b/.github/workflows/topology-suite-k8s-add-node.yml index 95b4f0acaf..7a76bd7f1c 100755 --- a/.github/workflows/topology-suite-k8s-add-node.yml +++ b/.github/workflows/topology-suite-k8s-add-node.yml @@ -1,5 +1,5 @@ name: K8s Topology Suite - Add Node (Per-Test Bootstrap) -run-name: "K8s Topology Add-Node | ${{ inputs.cluster_environment || 'local' }} | ${{ inputs.TEST_CLASSES }} | ${{ inputs.cluster_params }}" +run-name: "K8s Topology Add-Node | ${{ inputs.cluster_environment || 'openshift-baremetal' }} | ${{ inputs.TEST_CLASSES }} | ${{ inputs.cluster_params }}" on: workflow_dispatch: @@ -36,11 +36,13 @@ on: required: false default: 'main' csi_repository: - description: 'CSI driver image repository (leave empty for chart default)' + description: 'CSI driver image repository' required: false + default: 'simplyblock/spdkcsi' csi_tag: - description: 'CSI driver image tag (leave empty for chart default)' + description: 'CSI driver image tag' required: false + default: 'latest' spdk_image: description: 'SPDK container image' required: true @@ -53,7 +55,7 @@ on: description: 'Comma-separated initial K8s worker node names' required: true new_worker_nodes: - description: 'Comma-separated new K8s worker node names to add during test (at least 2, multiples of 2)' + description: 'Comma-separated new K8s worker node names to add during test (at least 1)' required: true # ========================= @@ -62,7 +64,7 @@ on: ifc_names: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false - default: 'ens18:enp1s0' + default: 'br-ex:enp2s0f0' max_lvol: description: 'Max logical volume count per storage node' required: false @@ -74,7 +76,7 @@ on: cluster_environment: description: 'Target cluster environment' required: true - default: 'local' + default: 'openshift-baremetal' type: choice options: - local @@ -132,9 +134,22 @@ on: description: 'NFS mountpoint on client nodes' required: false default: '/mnt/nfs_share/' + cluster_security: + description: "Cluster setup mode. 'backup' deploys MinIO and enables backup on StorageCluster. Auto-enabled for backup test names." + required: false + type: choice + default: "none" + options: + - none + - backup + preserve_resources_on_failure: + description: 'Preserve PVCs, FIO pods, snapshots on test failure for debugging' + required: false + default: true + type: boolean concurrency: - group: k8s-topology-add-node-${{ inputs.cluster_environment || 'local' }} + group: k8s-topology-add-node-${{ inputs.cluster_environment || 'openshift-baremetal' }} cancel-in-progress: false # ============================================================================ @@ -191,23 +206,25 @@ jobs: simplyblock_repository: ${{ inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }} operator_repository: ${{ inputs.operator_repository || 'simplyblock/simplyblock-operator' }} operator_tag: ${{ inputs.operator_tag || 'main' }} - csi_repository: ${{ inputs.csi_repository || '' }} - csi_tag: ${{ inputs.csi_tag || '' }} + csi_repository: ${{ inputs.csi_repository || 'simplyblock/spdkcsi' }} + csi_tag: ${{ inputs.csi_tag || 'latest' }} spdk_image: ${{ inputs.spdk_image }} worker_nodes: ${{ inputs.worker_nodes }} new_worker_nodes: ${{ inputs.new_worker_nodes }} - ifc_names: ${{ inputs.ifc_names || 'ens18:enp1s0' }} + ifc_names: ${{ inputs.ifc_names || 'br-ex:enp2s0f0' }} max_lvol: ${{ inputs.max_lvol || '30' }} ssh_user: ${{ inputs.ssh_user || 'root' }} key_path: ${{ inputs.key_path }} send_slack_notification: false cluster_params: ${{ inputs.cluster_params || 'ndcs=2,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4' }} - cluster_environment: ${{ inputs.cluster_environment || 'local' }} + cluster_environment: ${{ inputs.cluster_environment || 'openshift-baremetal' }} skip_nfs: ${{ inputs.skip_nfs || 'false' }} use_existing_cluster: ${{ inputs.use_existing_cluster || 'false' }} tls_enabled: ${{ inputs.tls_enabled }} client_ips: ${{ inputs.client_ips || '' }} nfs_mountpoint: ${{ inputs.nfs_mountpoint || '/mnt/nfs_share/' }} + cluster_security: ${{ inputs.cluster_security || 'none' }} + preserve_resources_on_failure: ${{ inputs.preserve_resources_on_failure }} secrets: inherit # ───────────────────────────────────────────────────────────────────────── @@ -290,7 +307,7 @@ jobs: GITHUB_REF_NAME: ${{ github.ref_name }} TEST_COUNT: ${{ needs.setup-matrix.outputs.test_count }} TEST_CLASSES: ${{ inputs.TEST_CLASSES }} - CLUSTER_ENV: ${{ inputs.cluster_environment || 'local' }} + CLUSTER_ENV: ${{ inputs.cluster_environment || 'openshift-baremetal' }} run: | python3 - <<'PYEOF' import json, os, re, sys, urllib.request @@ -321,17 +338,21 @@ jobs: if not os.path.isfile(md_path): continue content = open(md_path).read() - # Extract test name from "## ... Summary — " + # Extract test name — try "Summary — " first, + # fall back to "Test class: ``" (e2e-bootstrap format) name_m = re.search(r'Summary\s*[-—]\s*(\S+)', content) + if not name_m: + name_m = re.search(r'\*\*Test class:\*\*\s*`([^`]+)`', content) tname = name_m.group(1) if name_m else d - # Extract result - res_m = re.search(r'\*\*Result:\*\*\s*(\S+)', content) - result = res_m.group(1) if res_m else "?" - r_icon = ":white_check_mark:" if "pass" in result.lower() or "success" in result.lower() else ":x:" - # Extract counts - total_m = re.search(r'\*\*Total:\*\*\s*(\d+)', content) - passed_m = re.search(r'\*\*Passed:\*\*\s*(\d+)', content) - failed_m = re.search(r'\*\*Failed:\*\*\s*(\d+)', content) + # Extract result — handle both "SUCCESS"/"FAILED" and emoji prefixed "✅ SUCCESS" + res_m = re.search(r'\*\*Result:\*\*\s*(.+?)(?:\s*[&<\n]|$)', content) + result = res_m.group(1).strip() if res_m else "?" + r_icon = ":white_check_mark:" if "success" in result.lower() else ":x:" + # Extract counts — handle both inline "**Total:** N" and + # table format "| ✅ Passed | N |" from e2e-bootstrap + total_m = re.search(r'\*\*Total[:\*]*\*?\*?\s*\|?\s*\*?\*?(\d+)', content) + passed_m = re.search(r'(?:\*\*Passed:\*\*|[✅] Passed)\s*\|?\s*(\d+)', content) + failed_m = re.search(r'(?:\*\*Failed:\*\*|[❌] Failed)\s*\|?\s*(\d+)', content) total = total_m.group(1) if total_m else "?" passed = passed_m.group(1) if passed_m else "?" failed = failed_m.group(1) if failed_m else "?" diff --git a/.github/workflows/topology-suite-k8s-migration.yml b/.github/workflows/topology-suite-k8s-migration.yml index 62aa85b710..8c7c499e06 100755 --- a/.github/workflows/topology-suite-k8s-migration.yml +++ b/.github/workflows/topology-suite-k8s-migration.yml @@ -1,5 +1,5 @@ name: K8s Topology Suite - Node Migration (Per-Test Bootstrap) -run-name: "K8s Topology Migration | ${{ inputs.cluster_environment || 'local' }} | ${{ inputs.TEST_CLASSES }} | ${{ inputs.cluster_params }}" +run-name: "K8s Topology Migration | ${{ inputs.cluster_environment || 'openshift-baremetal' }} | ${{ inputs.TEST_CLASSES }} | ${{ inputs.cluster_params }}" on: workflow_dispatch: @@ -36,11 +36,13 @@ on: required: false default: 'main' csi_repository: - description: 'CSI driver image repository (leave empty for chart default)' + description: 'CSI driver image repository' required: false + default: 'simplyblock/spdkcsi' csi_tag: - description: 'CSI driver image tag (leave empty for chart default)' + description: 'CSI driver image tag' required: false + default: 'latest' spdk_image: description: 'SPDK container image' required: true @@ -71,7 +73,7 @@ on: ifc_names: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false - default: 'ens18:enp1s0' + default: 'br-ex:enp2s0f0' max_lvol: description: 'Max logical volume count per storage node' required: false @@ -83,7 +85,7 @@ on: cluster_environment: description: 'Target cluster environment' required: true - default: 'local' + default: 'openshift-baremetal' type: choice options: - local @@ -141,9 +143,13 @@ on: description: 'NFS mountpoint on client nodes' required: false default: '/mnt/nfs_share/' + # cluster_security removed from workflow_dispatch (25 input limit). + # Backup is auto-enabled when testname contains "Backup". + # Use workflow_call (via topology suite parent) to pass cluster_security explicitly. + # preserve_resources_on_failure also omitted (25 input limit); defaults to true in child. concurrency: - group: k8s-topology-migration-${{ inputs.cluster_environment || 'local' }} + group: k8s-topology-migration-${{ inputs.cluster_environment || 'openshift-baremetal' }} cancel-in-progress: false # ============================================================================ @@ -200,25 +206,27 @@ jobs: simplyblock_repository: ${{ inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }} operator_repository: ${{ inputs.operator_repository || 'simplyblock/simplyblock-operator' }} operator_tag: ${{ inputs.operator_tag || 'main' }} - csi_repository: ${{ inputs.csi_repository || '' }} - csi_tag: ${{ inputs.csi_tag || '' }} + csi_repository: ${{ inputs.csi_repository || 'simplyblock/spdkcsi' }} + csi_tag: ${{ inputs.csi_tag || 'latest' }} spdk_image: ${{ inputs.spdk_image }} worker_nodes: ${{ inputs.worker_nodes }} migrate_to_worker: ${{ inputs.migrate_to_worker }} new_ssd_pcie: ${{ inputs.new_ssd_pcie || '' }} reattach_volume: ${{ inputs.reattach_volume }} - ifc_names: ${{ inputs.ifc_names || 'ens18:enp1s0' }} + ifc_names: ${{ inputs.ifc_names || 'br-ex:enp2s0f0' }} max_lvol: ${{ inputs.max_lvol || '30' }} ssh_user: ${{ inputs.ssh_user || 'root' }} key_path: ${{ inputs.key_path }} send_slack_notification: false cluster_params: ${{ inputs.cluster_params || 'ndcs=2,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4' }} - cluster_environment: ${{ inputs.cluster_environment || 'local' }} + cluster_environment: ${{ inputs.cluster_environment || 'openshift-baremetal' }} skip_nfs: ${{ inputs.skip_nfs || 'false' }} use_existing_cluster: ${{ inputs.use_existing_cluster || 'false' }} tls_enabled: ${{ inputs.tls_enabled }} client_ips: ${{ inputs.client_ips || '' }} nfs_mountpoint: ${{ inputs.nfs_mountpoint || '/mnt/nfs_share/' }} + preserve_resources_on_failure: true + # cluster_security defaults to 'none' in child; backup auto-enabled when testname contains "Backup" secrets: inherit # ───────────────────────────────────────────────────────────────────────── @@ -301,7 +309,7 @@ jobs: GITHUB_REF_NAME: ${{ github.ref_name }} TEST_COUNT: ${{ needs.setup-matrix.outputs.test_count }} TEST_CLASSES: ${{ inputs.TEST_CLASSES }} - CLUSTER_ENV: ${{ inputs.cluster_environment || 'local' }} + CLUSTER_ENV: ${{ inputs.cluster_environment || 'openshift-baremetal' }} run: | python3 - <<'PYEOF' import json, os, re, sys, urllib.request @@ -332,14 +340,21 @@ jobs: if not os.path.isfile(md_path): continue content = open(md_path).read() + # Extract test name — try "Summary — " first, + # fall back to "Test class: ``" (e2e-bootstrap format) name_m = re.search(r'Summary\s*[-—]\s*(\S+)', content) + if not name_m: + name_m = re.search(r'\*\*Test class:\*\*\s*`([^`]+)`', content) tname = name_m.group(1) if name_m else d - res_m = re.search(r'\*\*Result:\*\*\s*(\S+)', content) - result = res_m.group(1) if res_m else "?" - r_icon = ":white_check_mark:" if "pass" in result.lower() or "success" in result.lower() else ":x:" - total_m = re.search(r'\*\*Total:\*\*\s*(\d+)', content) - passed_m = re.search(r'\*\*Passed:\*\*\s*(\d+)', content) - failed_m = re.search(r'\*\*Failed:\*\*\s*(\d+)', content) + # Extract result — handle both "SUCCESS"/"FAILED" and emoji prefixed "✅ SUCCESS" + res_m = re.search(r'\*\*Result:\*\*\s*(.+?)(?:\s*[&<\n]|$)', content) + result = res_m.group(1).strip() if res_m else "?" + r_icon = ":white_check_mark:" if "success" in result.lower() else ":x:" + # Extract counts — handle both inline "**Total:** N" and + # table format "| ✅ Passed | N |" from e2e-bootstrap + total_m = re.search(r'\*\*Total[:\*]*\*?\*?\s*\|?\s*\*?\*?(\d+)', content) + passed_m = re.search(r'(?:\*\*Passed:\*\*|[✅] Passed)\s*\|?\s*(\d+)', content) + failed_m = re.search(r'(?:\*\*Failed:\*\*|[❌] Failed)\s*\|?\s*(\d+)', content) total = total_m.group(1) if total_m else "?" passed = passed_m.group(1) if passed_m else "?" failed = failed_m.group(1) if failed_m else "?" diff --git a/.github/workflows/upgrade-bootstrap-single-v2.yml b/.github/workflows/upgrade-bootstrap-single-v2.yml index 55003e1575..43860d1a9a 100644 --- a/.github/workflows/upgrade-bootstrap-single-v2.yml +++ b/.github/workflows/upgrade-bootstrap-single-v2.yml @@ -73,7 +73,7 @@ on: default: true TEST_CLASS: type: string - default: "major_upgrade_single" + default: "TestMajorUpgradeSingleNode" RUN_LABEL: type: string default: "" @@ -126,7 +126,7 @@ on: description: "Upgrade test class name; defaults to single-node variant (no FIO during upgrade)" required: false type: string - default: "major_upgrade_single" + default: "TestMajorUpgradeSingleNode" RUN_LABEL: description: "Optional label appended to artifact names to avoid collisions" @@ -179,11 +179,11 @@ jobs: BOOTSTRAP_HA_TYPE: ${{ inputs.BOOTSTRAP_HA_TYPE || 'single' }} BOOTSTRAP_DATA_NIC: ${{ inputs.BOOTSTRAP_DATA_NIC || 'eth1' }} - TEST_CLASS: ${{ inputs.TEST_CLASS || 'major_upgrade_single' }} + TEST_CLASS: ${{ inputs.TEST_CLASS || 'TestMajorUpgradeSingleNode' }} # Target images (overridden by CUSTOM_IMAGES target_spdk/target_docker) TARGET_SPDK_IMAGE: 'simplyblock/spdk:main-latest' - TARGET_DOCKER_IMAGE: 'public.ecr.aws/simply-block/simplyblock:main-latest' + TARGET_DOCKER_IMAGE: 'public.ecr.aws/simply-block/simplyblock:main' # Secrets SSH_PASSWORD: ${{ secrets.SSH_PASSWORD }} @@ -1070,6 +1070,140 @@ jobs: print(f"[{ip}] Saved -> {dest_dir}/{os.path.basename(last)}", flush=True) PY + - name: Collect Graylog/OpenSearch logs + if: '!cancelled()' + timeout-minutes: 480 + shell: bash + run: | + set +e + [ -z "${TEST_START_EPOCH:-}" ] || [ -z "${TEST_END_EPOCH:-}" ] && exit 0 + ELAPSED=$((TEST_END_EPOCH - TEST_START_EPOCH)) + [ "${ELAPSED}" -le 0 ] && exit 0 + + WINDOW_START=$((TEST_START_EPOCH - 3600)) + WINDOW_END=$((TEST_END_EPOCH + 3600)) + + MGMT_IP="$(echo "${MNODES}" | awk '{print $1}')" + SSH_OPTS=(-i "${KEY_PATH}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10) + + OUTPUT_DIR="${RUN_BASE_DIR}/graylog_collected" + mkdir -p "${OUTPUT_DIR}" 2>/dev/null || true + + epoch_to_iso() { + python3 -c "from datetime import datetime,timezone; print(datetime.fromtimestamp($1,tz=timezone.utc).strftime('%Y-%m-%dT%H:%M:%S'))" + } + + _CHUNK_STARTS=() + _C=${WINDOW_START} + while [ ${_C} -lt ${WINDOW_END} ]; do + _CHUNK_STARTS+=(${_C}) + _C=$((_C + 3600)) + done + NUM_CHUNKS=${#_CHUNK_STARTS[@]} + + run_collect() { + local iso=$1 mins=$2 outdir=$3 extra_flag=${4:-} + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" \ + "python3 -m simplyblock_core.scripts.collect_logs \ + '${iso}' '${mins}' \ + --mode docker \ + --output-dir '${outdir}' \ + ${extra_flag} \ + ${CLUSTER_ID:+--cluster-id '${CLUSTER_ID}'}" \ + 2>&1 + local rc=$? + [ $rc -ne 0 ] && return $rc + local has_content + has_content=$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" \ + "find '${outdir}' -name '*.tar.gz' -size +0 2>/dev/null | head -1") + if [ -z "${has_content}" ]; then + echo " WARN: collect_logs.py succeeded but no .tar.gz output found" + return 1 + fi + } + + collect_adaptive() { + local start_epoch=$1 end_epoch=$2 outdir=$3 extra_flag=${4:-} + local duration=$((end_epoch - start_epoch)) + local mins=$(( (duration + 59) / 60 )) + local iso=$(epoch_to_iso ${start_epoch}) + if run_collect "${iso}" "${mins}" "${outdir}" "${extra_flag}"; then + return 0 + fi + echo " WARN: ${mins}m window failed, retrying with 5-min sub-windows..." + local sub_start=${start_epoch} + while [ ${sub_start} -lt ${end_epoch} ]; do + local sub_end=$((sub_start + 300)) + [ ${sub_end} -gt ${end_epoch} ] && sub_end=${end_epoch} + local sub_mins=$(( ((sub_end - sub_start) + 59) / 60 )) + local sub_iso=$(epoch_to_iso ${sub_start}) + if ! run_collect "${sub_iso}" "${sub_mins}" "${outdir}" "${extra_flag}"; then + echo " WARN: 5-min window at ${sub_iso} failed, retrying with 1-min windows..." + local micro_start=${sub_start} + while [ ${micro_start} -lt ${sub_end} ]; do + local micro_end=$((micro_start + 60)) + [ ${micro_end} -gt ${sub_end} ] && micro_end=${sub_end} + local micro_mins=$(( ((micro_end - micro_start) + 59) / 60 )) + local micro_iso=$(epoch_to_iso ${micro_start}) + run_collect "${micro_iso}" "${micro_mins}" "${outdir}" "${extra_flag}" || \ + echo " WARN: 1-min window at ${micro_iso} also failed" + micro_start=${micro_end} + done + fi + sub_start=${sub_end} + done + } + + PREFER_OPENSEARCH="" + CHUNK=0 + for (( _IDX=NUM_CHUNKS-1; _IDX>=0; _IDX-- )); do + CHUNK=$((CHUNK + 1)) + CHUNK_START=${_CHUNK_STARTS[$_IDX]} + CHUNK_END=$((CHUNK_START + 3600)) + [ ${CHUNK_END} -gt ${WINDOW_END} ] && CHUNK_END=${WINDOW_END} + CHUNK_MINUTES=$(( ((CHUNK_END - CHUNK_START) + 59) / 60 )) + CHUNK_ISO=$(epoch_to_iso ${CHUNK_START}) + echo "--- Chunk ${CHUNK}/${NUM_CHUNKS}: ${CHUNK_ISO} for ${CHUNK_MINUTES}m (newest-first) ---" + REMOTE_OUTPUT_DIR="/tmp/graylog_collect_chunk${CHUNK}" + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "mkdir -p '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true + if [ -z "${PREFER_OPENSEARCH}" ]; then + echo " Probing OpenSearch availability..." + if run_collect "${CHUNK_ISO}" "${CHUNK_MINUTES}" "${REMOTE_OUTPUT_DIR}" "--use-opensearch"; then + PREFER_OPENSEARCH=true + echo " OpenSearch works — using it for all chunks" + else + PREFER_OPENSEARCH=false + echo " OpenSearch unavailable — using Graylog for all chunks" + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "rm -rf '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "mkdir -p '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" || \ + echo "WARN: Graylog also failed for chunk ${CHUNK}" + fi + elif [ "${PREFER_OPENSEARCH}" = "true" ]; then + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" "--use-opensearch" || { + echo "WARN: OpenSearch failed, falling back to Graylog..." + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" || true + } + else + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" || { + echo "WARN: Graylog failed, falling back to OpenSearch..." + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" "--use-opensearch" || true + } + fi + TARBALLS=$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" \ + "find '${REMOTE_OUTPUT_DIR}' -name '*.tar.gz' -type f 2>/dev/null") || true + if [ -n "${TARBALLS}" ]; then + for TB in ${TARBALLS}; do + scp "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}:${TB}" "${OUTPUT_DIR}/$(basename ${TB})" 2>&1 || true + done + for TB_FILE in "${OUTPUT_DIR}"/*.tar.gz; do + [ -f "${TB_FILE}" ] && tar -xzf "${TB_FILE}" -C "${OUTPUT_DIR}/" 2>/dev/null || true + done + fi + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "rm -rf '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true + done + echo "=== Log collection complete (${CHUNK} chunks, newest-first): ${OUTPUT_DIR} ===" + # ========================= # SUMMARY (always) # ========================= @@ -1213,7 +1347,7 @@ jobs: echo "" echo "" echo "" - echo "> **Note:** Graylog/OpenSearch logs are currently being collected and will be available in the NFS log directory shortly." + echo "> **Note:** Graylog/OpenSearch logs collected to \`${RUN_BASE_DIR}/graylog_collected/\`" } >> "$GITHUB_STEP_SUMMARY" - name: Send Slack Notification diff --git a/.github/workflows/upgrade-bootstrap-single.yml b/.github/workflows/upgrade-bootstrap-single.yml index 89ded2bb48..b3549caaec 100644 --- a/.github/workflows/upgrade-bootstrap-single.yml +++ b/.github/workflows/upgrade-bootstrap-single.yml @@ -82,7 +82,7 @@ on: description: "Cluster setup mode. 'backup' writes s3-backup.json and passes --use-backup to cluster create." TEST_CLASS: type: string - default: "major_upgrade_single" + default: "TestMajorUpgradeSingleNode" RUN_LABEL: type: string default: "" @@ -193,7 +193,7 @@ on: description: "Upgrade test class name; defaults to single-node variant (no FIO during upgrade)" required: false type: string - default: "major_upgrade_single" + default: "TestMajorUpgradeSingleNode" concurrency: group: simplyblock-lab-upgrade-single @@ -244,11 +244,11 @@ jobs: EXTRA_SN_ARGS: ${{ inputs.EXTRA_SN_ARGS || '' }} CLUSTER_SECURITY: ${{ inputs.CLUSTER_SECURITY || 'backup' }} - TEST_CLASS: ${{ inputs.TEST_CLASS || 'major_upgrade_single' }} + TEST_CLASS: ${{ inputs.TEST_CLASS || 'TestMajorUpgradeSingleNode' }} # Target images (overridden by CUSTOM_IMAGES target_spdk/target_docker) TARGET_SPDK_IMAGE: 'simplyblock/spdk:main-latest' - TARGET_DOCKER_IMAGE: 'public.ecr.aws/simply-block/simplyblock:main-latest' + TARGET_DOCKER_IMAGE: 'public.ecr.aws/simply-block/simplyblock:main' # Secrets SSH_PASSWORD: ${{ secrets.SSH_PASSWORD }} @@ -740,6 +740,35 @@ jobs: echo "TEST_START_EPOCH=$(date +%s)" >> "$GITHUB_ENV" echo "TEST_START_HUMAN=$(date -u +'%Y-%m-%d %H:%M:%S UTC')" >> "$GITHUB_ENV" + - name: Start MinIO trace logging + if: ${{ env.CLUSTER_SECURITY == 'backup' }} + shell: bash + run: | + set -euxo pipefail + + # 1. Install mc (MinIO Client) + if ! command -v mc &>/dev/null; then + if curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc 2>/dev/null; then + chmod +x /usr/local/bin/mc + else + curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc + chmod +x /tmp/mc + export PATH="/tmp:$PATH" + echo "/tmp" >> "$GITHUB_PATH" + fi + fi + + # 2. Configure mc alias to external MinIO + mc alias set myminio http://192.168.10.164:9000 "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" + + # 3. Start admin trace in background + MINIO_TRACE_LOG="/tmp/minio-trace-${GITHUB_RUN_ID}.log" + echo "MINIO_TRACE_LOG=${MINIO_TRACE_LOG}" >> "$GITHUB_ENV" + nohup mc admin trace -v --all myminio > "${MINIO_TRACE_LOG}" 2>&1 & + echo "MC_TRACE_PID=$!" >> "$GITHUB_ENV" + + echo "=== MinIO trace logging started (PID=$!, log=${MINIO_TRACE_LOG}) ===" + - name: Run upgrade e2e tests (major_upgrade_single) shell: bash working-directory: sbcli/e2e @@ -1068,6 +1097,140 @@ jobs: print(f"[{ip}] Saved -> {dest_dir}/{os.path.basename(last)}", flush=True) PY + - name: Collect Graylog/OpenSearch logs + if: '!cancelled()' + timeout-minutes: 480 + shell: bash + run: | + set +e + [ -z "${TEST_START_EPOCH:-}" ] || [ -z "${TEST_END_EPOCH:-}" ] && exit 0 + ELAPSED=$((TEST_END_EPOCH - TEST_START_EPOCH)) + [ "${ELAPSED}" -le 0 ] && exit 0 + + WINDOW_START=$((TEST_START_EPOCH - 3600)) + WINDOW_END=$((TEST_END_EPOCH + 3600)) + + MGMT_IP="$(echo "${MNODES}" | awk '{print $1}')" + SSH_OPTS=(-i "${KEY_PATH}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10) + + OUTPUT_DIR="${RUN_BASE_DIR}/graylog_collected" + mkdir -p "${OUTPUT_DIR}" 2>/dev/null || true + + epoch_to_iso() { + python3 -c "from datetime import datetime,timezone; print(datetime.fromtimestamp($1,tz=timezone.utc).strftime('%Y-%m-%dT%H:%M:%S'))" + } + + _CHUNK_STARTS=() + _C=${WINDOW_START} + while [ ${_C} -lt ${WINDOW_END} ]; do + _CHUNK_STARTS+=(${_C}) + _C=$((_C + 3600)) + done + NUM_CHUNKS=${#_CHUNK_STARTS[@]} + + run_collect() { + local iso=$1 mins=$2 outdir=$3 extra_flag=${4:-} + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" \ + "python3 -m simplyblock_core.scripts.collect_logs \ + '${iso}' '${mins}' \ + --mode docker \ + --output-dir '${outdir}' \ + ${extra_flag} \ + ${CLUSTER_ID:+--cluster-id '${CLUSTER_ID}'}" \ + 2>&1 + local rc=$? + [ $rc -ne 0 ] && return $rc + local has_content + has_content=$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" \ + "find '${outdir}' -name '*.tar.gz' -size +0 2>/dev/null | head -1") + if [ -z "${has_content}" ]; then + echo " WARN: collect_logs.py succeeded but no .tar.gz output found" + return 1 + fi + } + + collect_adaptive() { + local start_epoch=$1 end_epoch=$2 outdir=$3 extra_flag=${4:-} + local duration=$((end_epoch - start_epoch)) + local mins=$(( (duration + 59) / 60 )) + local iso=$(epoch_to_iso ${start_epoch}) + if run_collect "${iso}" "${mins}" "${outdir}" "${extra_flag}"; then + return 0 + fi + echo " WARN: ${mins}m window failed, retrying with 5-min sub-windows..." + local sub_start=${start_epoch} + while [ ${sub_start} -lt ${end_epoch} ]; do + local sub_end=$((sub_start + 300)) + [ ${sub_end} -gt ${end_epoch} ] && sub_end=${end_epoch} + local sub_mins=$(( ((sub_end - sub_start) + 59) / 60 )) + local sub_iso=$(epoch_to_iso ${sub_start}) + if ! run_collect "${sub_iso}" "${sub_mins}" "${outdir}" "${extra_flag}"; then + echo " WARN: 5-min window at ${sub_iso} failed, retrying with 1-min windows..." + local micro_start=${sub_start} + while [ ${micro_start} -lt ${sub_end} ]; do + local micro_end=$((micro_start + 60)) + [ ${micro_end} -gt ${sub_end} ] && micro_end=${sub_end} + local micro_mins=$(( ((micro_end - micro_start) + 59) / 60 )) + local micro_iso=$(epoch_to_iso ${micro_start}) + run_collect "${micro_iso}" "${micro_mins}" "${outdir}" "${extra_flag}" || \ + echo " WARN: 1-min window at ${micro_iso} also failed" + micro_start=${micro_end} + done + fi + sub_start=${sub_end} + done + } + + PREFER_OPENSEARCH="" + CHUNK=0 + for (( _IDX=NUM_CHUNKS-1; _IDX>=0; _IDX-- )); do + CHUNK=$((CHUNK + 1)) + CHUNK_START=${_CHUNK_STARTS[$_IDX]} + CHUNK_END=$((CHUNK_START + 3600)) + [ ${CHUNK_END} -gt ${WINDOW_END} ] && CHUNK_END=${WINDOW_END} + CHUNK_MINUTES=$(( ((CHUNK_END - CHUNK_START) + 59) / 60 )) + CHUNK_ISO=$(epoch_to_iso ${CHUNK_START}) + echo "--- Chunk ${CHUNK}/${NUM_CHUNKS}: ${CHUNK_ISO} for ${CHUNK_MINUTES}m (newest-first) ---" + REMOTE_OUTPUT_DIR="/tmp/graylog_collect_chunk${CHUNK}" + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "mkdir -p '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true + if [ -z "${PREFER_OPENSEARCH}" ]; then + echo " Probing OpenSearch availability..." + if run_collect "${CHUNK_ISO}" "${CHUNK_MINUTES}" "${REMOTE_OUTPUT_DIR}" "--use-opensearch"; then + PREFER_OPENSEARCH=true + echo " OpenSearch works — using it for all chunks" + else + PREFER_OPENSEARCH=false + echo " OpenSearch unavailable — using Graylog for all chunks" + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "rm -rf '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "mkdir -p '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" || \ + echo "WARN: Graylog also failed for chunk ${CHUNK}" + fi + elif [ "${PREFER_OPENSEARCH}" = "true" ]; then + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" "--use-opensearch" || { + echo "WARN: OpenSearch failed, falling back to Graylog..." + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" || true + } + else + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" || { + echo "WARN: Graylog failed, falling back to OpenSearch..." + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" "--use-opensearch" || true + } + fi + TARBALLS=$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" \ + "find '${REMOTE_OUTPUT_DIR}' -name '*.tar.gz' -type f 2>/dev/null") || true + if [ -n "${TARBALLS}" ]; then + for TB in ${TARBALLS}; do + scp "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}:${TB}" "${OUTPUT_DIR}/$(basename ${TB})" 2>&1 || true + done + for TB_FILE in "${OUTPUT_DIR}"/*.tar.gz; do + [ -f "${TB_FILE}" ] && tar -xzf "${TB_FILE}" -C "${OUTPUT_DIR}/" 2>/dev/null || true + done + fi + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "rm -rf '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true + done + echo "=== Log collection complete (${CHUNK} chunks, newest-first): ${OUTPUT_DIR} ===" + # ========================= # SUMMARY (always) # ========================= @@ -1211,7 +1374,7 @@ jobs: echo "" echo "" echo "" - echo "> **Note:** Graylog/OpenSearch logs are currently being collected and will be available in the NFS log directory shortly." + echo "> **Note:** Graylog/OpenSearch logs collected to \`${RUN_BASE_DIR}/graylog_collected/\`" } >> "$GITHUB_STEP_SUMMARY" - name: Send Slack Notification @@ -1333,6 +1496,25 @@ jobs: print(f"WARN: Slack notification failed: {exc}", file=sys.stderr) PYEOF + - name: Stop MinIO trace logging and save + if: always() + shell: bash + run: | + set +e + if [ -n "${MC_TRACE_PID:-}" ]; then + echo "Stopping MinIO trace (PID=${MC_TRACE_PID})..." + kill "${MC_TRACE_PID}" 2>/dev/null || true + wait "${MC_TRACE_PID}" 2>/dev/null || true + fi + if [ -f "${MINIO_TRACE_LOG:-}" ]; then + LINES=$(wc -l < "${MINIO_TRACE_LOG}") + SIZE=$(du -h "${MINIO_TRACE_LOG}" | cut -f1) + echo "=== MinIO trace log: ${LINES} lines, ${SIZE} ===" + cp "${MINIO_TRACE_LOG}" sbcli/e2e/minio-trace.log 2>/dev/null || true + else + echo "No MinIO trace log found (backup tests may not have run)" + fi + - name: Upload logs (always) if: always() uses: actions/upload-artifact@v4 @@ -1341,6 +1523,7 @@ jobs: path: | simplyBlockDeploy/bare-metal/bootstrap.log sbcli/e2e/output.log + sbcli/e2e/minio-trace.log sbcli/e2e/logs/** if-no-files-found: warn diff --git a/.github/workflows/upgrade-bootstrap.yml b/.github/workflows/upgrade-bootstrap.yml index 4f87623fcf..154845d2c0 100644 --- a/.github/workflows/upgrade-bootstrap.yml +++ b/.github/workflows/upgrade-bootstrap.yml @@ -85,7 +85,7 @@ on: description: "Cluster setup mode. 'backup' writes s3-backup.json and passes --use-backup to cluster create." TEST_CLASS: type: string - default: "major_upgrade" + default: "TestMajorUpgrade" RUN_LABEL: type: string default: "" @@ -201,7 +201,7 @@ on: description: "Upgrade test class name (--testname); leave empty to run all upgrade tests" required: false type: string - default: "major_upgrade" + default: "TestMajorUpgrade" concurrency: group: simplyblock-lab-upgrade @@ -252,11 +252,11 @@ jobs: EXTRA_SN_ARGS: ${{ inputs.EXTRA_SN_ARGS || '' }} CLUSTER_SECURITY: ${{ inputs.CLUSTER_SECURITY || 'backup' }} - TEST_CLASS: ${{ inputs.TEST_CLASS || 'major_upgrade' }} + TEST_CLASS: ${{ inputs.TEST_CLASS || 'TestMajorUpgrade' }} # Target images (overridden by CUSTOM_IMAGES target_spdk/target_docker) TARGET_SPDK_IMAGE: 'simplyblock/spdk:main-latest' - TARGET_DOCKER_IMAGE: 'public.ecr.aws/simply-block/simplyblock:main-latest' + TARGET_DOCKER_IMAGE: 'public.ecr.aws/simply-block/simplyblock:main' # Secrets SSH_PASSWORD: ${{ secrets.SSH_PASSWORD }} @@ -749,6 +749,35 @@ jobs: echo "TEST_START_EPOCH=$(date +%s)" >> "$GITHUB_ENV" echo "TEST_START_HUMAN=$(date -u +'%Y-%m-%d %H:%M:%S UTC')" >> "$GITHUB_ENV" + - name: Start MinIO trace logging + if: ${{ env.CLUSTER_SECURITY == 'backup' }} + shell: bash + run: | + set -euxo pipefail + + # 1. Install mc (MinIO Client) + if ! command -v mc &>/dev/null; then + if curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc 2>/dev/null; then + chmod +x /usr/local/bin/mc + else + curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc + chmod +x /tmp/mc + export PATH="/tmp:$PATH" + echo "/tmp" >> "$GITHUB_PATH" + fi + fi + + # 2. Configure mc alias to external MinIO + mc alias set myminio http://192.168.10.164:9000 "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" + + # 3. Start admin trace in background + MINIO_TRACE_LOG="/tmp/minio-trace-${GITHUB_RUN_ID}.log" + echo "MINIO_TRACE_LOG=${MINIO_TRACE_LOG}" >> "$GITHUB_ENV" + nohup mc admin trace -v --all myminio > "${MINIO_TRACE_LOG}" 2>&1 & + echo "MC_TRACE_PID=$!" >> "$GITHUB_ENV" + + echo "=== MinIO trace logging started (PID=$!, log=${MINIO_TRACE_LOG}) ===" + - name: Run upgrade e2e tests (major_upgrade) shell: bash working-directory: sbcli/e2e @@ -1077,6 +1106,141 @@ jobs: print(f"[{ip}] Saved -> {dest_dir}/{os.path.basename(last)}", flush=True) PY + - name: Collect Graylog/OpenSearch logs + if: '!cancelled()' + timeout-minutes: 480 + shell: bash + run: | + set +e + [ -z "${TEST_START_EPOCH:-}" ] || [ -z "${TEST_END_EPOCH:-}" ] && exit 0 + ELAPSED=$((TEST_END_EPOCH - TEST_START_EPOCH)) + [ "${ELAPSED}" -le 0 ] && exit 0 + + WINDOW_START=$((TEST_START_EPOCH - 3600)) + WINDOW_END=$((TEST_END_EPOCH + 3600)) + + MGMT_IP="$(echo "${MNODES}" | awk '{print $1}')" + SSH_OPTS=(-i "${KEY_PATH}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10) + + OUTPUT_DIR="${RUN_BASE_DIR}/graylog_collected" + mkdir -p "${OUTPUT_DIR}" 2>/dev/null || true + + epoch_to_iso() { + python3 -c "from datetime import datetime,timezone; print(datetime.fromtimestamp($1,tz=timezone.utc).strftime('%Y-%m-%dT%H:%M:%S'))" + } + + # Build chunk boundaries, then iterate in REVERSE order (newest first) + _CHUNK_STARTS=() + _C=${WINDOW_START} + while [ ${_C} -lt ${WINDOW_END} ]; do + _CHUNK_STARTS+=(${_C}) + _C=$((_C + 3600)) + done + NUM_CHUNKS=${#_CHUNK_STARTS[@]} + + run_collect() { + local iso=$1 mins=$2 outdir=$3 extra_flag=${4:-} + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" \ + "python3 -m simplyblock_core.scripts.collect_logs \ + '${iso}' '${mins}' \ + --mode docker \ + --output-dir '${outdir}' \ + ${extra_flag} \ + ${CLUSTER_ID:+--cluster-id '${CLUSTER_ID}'}" \ + 2>&1 + local rc=$? + [ $rc -ne 0 ] && return $rc + local has_content + has_content=$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" \ + "find '${outdir}' -name '*.tar.gz' -size +0 2>/dev/null | head -1") + if [ -z "${has_content}" ]; then + echo " WARN: collect_logs.py succeeded but no .tar.gz output found" + return 1 + fi + } + + collect_adaptive() { + local start_epoch=$1 end_epoch=$2 outdir=$3 extra_flag=${4:-} + local duration=$((end_epoch - start_epoch)) + local mins=$(( (duration + 59) / 60 )) + local iso=$(epoch_to_iso ${start_epoch}) + if run_collect "${iso}" "${mins}" "${outdir}" "${extra_flag}"; then + return 0 + fi + echo " WARN: ${mins}m window failed, retrying with 5-min sub-windows..." + local sub_start=${start_epoch} + while [ ${sub_start} -lt ${end_epoch} ]; do + local sub_end=$((sub_start + 300)) + [ ${sub_end} -gt ${end_epoch} ] && sub_end=${end_epoch} + local sub_mins=$(( ((sub_end - sub_start) + 59) / 60 )) + local sub_iso=$(epoch_to_iso ${sub_start}) + if ! run_collect "${sub_iso}" "${sub_mins}" "${outdir}" "${extra_flag}"; then + echo " WARN: 5-min window at ${sub_iso} failed, retrying with 1-min windows..." + local micro_start=${sub_start} + while [ ${micro_start} -lt ${sub_end} ]; do + local micro_end=$((micro_start + 60)) + [ ${micro_end} -gt ${sub_end} ] && micro_end=${sub_end} + local micro_mins=$(( ((micro_end - micro_start) + 59) / 60 )) + local micro_iso=$(epoch_to_iso ${micro_start}) + run_collect "${micro_iso}" "${micro_mins}" "${outdir}" "${extra_flag}" || \ + echo " WARN: 1-min window at ${micro_iso} also failed" + micro_start=${micro_end} + done + fi + sub_start=${sub_end} + done + } + + PREFER_OPENSEARCH="" + CHUNK=0 + for (( _IDX=NUM_CHUNKS-1; _IDX>=0; _IDX-- )); do + CHUNK=$((CHUNK + 1)) + CHUNK_START=${_CHUNK_STARTS[$_IDX]} + CHUNK_END=$((CHUNK_START + 3600)) + [ ${CHUNK_END} -gt ${WINDOW_END} ] && CHUNK_END=${WINDOW_END} + CHUNK_MINUTES=$(( ((CHUNK_END - CHUNK_START) + 59) / 60 )) + CHUNK_ISO=$(epoch_to_iso ${CHUNK_START}) + echo "--- Chunk ${CHUNK}/${NUM_CHUNKS}: ${CHUNK_ISO} for ${CHUNK_MINUTES}m (newest-first) ---" + REMOTE_OUTPUT_DIR="/tmp/graylog_collect_chunk${CHUNK}" + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "mkdir -p '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true + if [ -z "${PREFER_OPENSEARCH}" ]; then + echo " Probing OpenSearch availability..." + if run_collect "${CHUNK_ISO}" "${CHUNK_MINUTES}" "${REMOTE_OUTPUT_DIR}" "--use-opensearch"; then + PREFER_OPENSEARCH=true + echo " OpenSearch works — using it for all chunks" + else + PREFER_OPENSEARCH=false + echo " OpenSearch unavailable — using Graylog for all chunks" + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "rm -rf '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "mkdir -p '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" || \ + echo "WARN: Graylog also failed for chunk ${CHUNK}" + fi + elif [ "${PREFER_OPENSEARCH}" = "true" ]; then + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" "--use-opensearch" || { + echo "WARN: OpenSearch failed, falling back to Graylog..." + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" || true + } + else + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" || { + echo "WARN: Graylog failed, falling back to OpenSearch..." + collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" "--use-opensearch" || true + } + fi + TARBALLS=$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" \ + "find '${REMOTE_OUTPUT_DIR}' -name '*.tar.gz' -type f 2>/dev/null") || true + if [ -n "${TARBALLS}" ]; then + for TB in ${TARBALLS}; do + scp "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}:${TB}" "${OUTPUT_DIR}/$(basename ${TB})" 2>&1 || true + done + for TB_FILE in "${OUTPUT_DIR}"/*.tar.gz; do + [ -f "${TB_FILE}" ] && tar -xzf "${TB_FILE}" -C "${OUTPUT_DIR}/" 2>/dev/null || true + done + fi + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${MGMT_IP}" "rm -rf '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true + done + echo "=== Log collection complete (${CHUNK} chunks, newest-first): ${OUTPUT_DIR} ===" + # ========================= # SUMMARY (always) # ========================= @@ -1224,7 +1388,7 @@ jobs: echo "" echo "" echo "" - echo "> **Note:** Graylog/OpenSearch logs are currently being collected and will be available in the NFS log directory shortly." + echo "> **Note:** Graylog/OpenSearch logs collected to \`${RUN_BASE_DIR}/graylog_collected/\`" } >> "$GITHUB_STEP_SUMMARY" - name: Send Slack Notification @@ -1350,6 +1514,25 @@ jobs: print(f"WARN: Slack notification failed: {exc}", file=sys.stderr) PYEOF + - name: Stop MinIO trace logging and save + if: always() + shell: bash + run: | + set +e + if [ -n "${MC_TRACE_PID:-}" ]; then + echo "Stopping MinIO trace (PID=${MC_TRACE_PID})..." + kill "${MC_TRACE_PID}" 2>/dev/null || true + wait "${MC_TRACE_PID}" 2>/dev/null || true + fi + if [ -f "${MINIO_TRACE_LOG:-}" ]; then + LINES=$(wc -l < "${MINIO_TRACE_LOG}") + SIZE=$(du -h "${MINIO_TRACE_LOG}" | cut -f1) + echo "=== MinIO trace log: ${LINES} lines, ${SIZE} ===" + cp "${MINIO_TRACE_LOG}" sbcli/e2e/minio-trace.log 2>/dev/null || true + else + echo "No MinIO trace log found (backup tests may not have run)" + fi + - name: Upload logs (always) if: always() uses: actions/upload-artifact@v4 @@ -1358,6 +1541,7 @@ jobs: path: | simplyBlockDeploy/bare-metal/bootstrap.log sbcli/e2e/output.log + sbcli/e2e/minio-trace.log sbcli/e2e/logs/** if-no-files-found: warn diff --git a/R25_MANUAL_SETUP_STEPS.md b/R25_MANUAL_SETUP_STEPS.md new file mode 100644 index 0000000000..ee3d5e3c46 --- /dev/null +++ b/R25_MANUAL_SETUP_STEPS.md @@ -0,0 +1,317 @@ +# Manual Steps: Clean Up + Deploy R25 Setup (for dev investigation) + +These use the default parameters from the `k8s-native-upgrade.yaml` workflow: +- `ndcs=1, npcs=1, partitions=0, jm_count=3, max_lvol=30` +- Environment: `openshift-baremetal` +- Workers: `worker-0` through `worker-5` + +Purpose: Reproduce the R25 setup manually so dev can observe exactly when/how FDB disappears during the upgrade uninstall step. + +--- + +## Phase 1: Cleanup everything + +```bash +NAMESPACE=simplyblock + +# 1. Uninstall all Helm releases (R25 + R26) +helm uninstall spdk-csi -n $NAMESPACE --no-hooks --timeout 60s 2>/dev/null || true +helm uninstall sbcli -n $NAMESPACE --no-hooks --timeout 60s 2>/dev/null || true +for rel in $(helm list -n $NAMESPACE -q 2>/dev/null); do + echo "Uninstalling: $rel" + helm uninstall "$rel" -n $NAMESPACE --no-hooks --timeout 60s 2>/dev/null || true +done +helm uninstall cert-manager -n cert-manager --no-hooks --timeout 60s 2>/dev/null || true + +# 2. Delete simplyblock resources in kube-system +for RTYPE in deployment service sa configmap; do + for NAME in $(kubectl -n kube-system get $RTYPE --no-headers -o custom-columns=:metadata.name 2>/dev/null | grep -i simplyblock 2>/dev/null); do + echo "Deleting kube-system $RTYPE/$NAME" + kubectl -n kube-system delete $RTYPE "$NAME" --ignore-not-found 2>/dev/null || true + done +done + +# 3. Run shared cleanup scripts (if available on disk) +# bash /path/to/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh -f $NAMESPACE || true +# bash /path/to/e2e/scripts/cleanup_k8s.sh $NAMESPACE + +# 4. Delete released PVs +for pv in $(kubectl get pv --no-headers 2>/dev/null | grep 'simplyblock/' | awk '{print $1}'); do + kubectl delete pv "$pv" --ignore-not-found || true +done + +# 5. Delete simplyblock CRDs (clear finalizers first) +for crd in $(kubectl get crd -o name 2>/dev/null | grep simplyblock); do + crd_name="${crd#customresourcedefinition.apiextensions.k8s.io/}" + for cr in $(kubectl get "$crd_name" -A --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null); do + IFS=' ' read -r ns name <<< "$cr" + kubectl patch "$crd_name" "$name" -n "$ns" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + done + kubectl patch crd "$crd_name" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true +done +kubectl delete crd $(kubectl get crd -o name 2>/dev/null | grep simplyblock | sed 's|customresourcedefinition.apiextensions.k8s.io/||') --ignore-not-found --timeout=60s 2>/dev/null || true + +# 6. Delete + finalize namespaces +kubectl delete namespace $NAMESPACE --force --grace-period=0 --wait=false 2>/dev/null || true +kubectl delete namespace cert-manager --force --grace-period=0 --wait=false 2>/dev/null || true +for ns in $NAMESPACE cert-manager; do + kubectl get namespace $ns -o json 2>/dev/null \ + | jq '.spec.finalizers = []' \ + | kubectl replace --raw "/api/v1/namespaces/$ns/finalize" -f - 2>/dev/null || true +done + +# 7. Delete cert-manager CRDs +kubectl delete crd certificaterequests.cert-manager.io certificates.cert-manager.io \ + challenges.acme.cert-manager.io clusterissuers.cert-manager.io \ + issuers.cert-manager.io orders.acme.cert-manager.io \ + --ignore-not-found --force --grace-period=0 2>/dev/null || true + +# 8. Disconnect NVMe, reset hugepages, restart kubelet on each worker +for NODE in worker-0.ocp.simplyblock.ai worker-1.ocp.simplyblock.ai worker-2.ocp.simplyblock.ai worker-3.ocp.simplyblock.ai worker-4.ocp.simplyblock.ai worker-5.ocp.simplyblock.ai; do + oc debug node/"$NODE" -- chroot /host bash -c \ + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages && systemctl restart kubelet" 2>/dev/null || true + echo "Done: $NODE" +done + +# 9. Verify namespace is gone +kubectl get namespace $NAMESPACE 2>/dev/null && echo "WARNING: namespace still exists" || echo "Namespace gone" + +# 10. Remove stale storagenodeset labels +for NODE in worker-0.ocp.simplyblock.ai worker-1.ocp.simplyblock.ai worker-2.ocp.simplyblock.ai worker-3.ocp.simplyblock.ai worker-4.ocp.simplyblock.ai worker-5.ocp.simplyblock.ai; do + kubectl label node "$NODE" io.simplyblock.storagenodeset- 2>/dev/null || true +done +``` + +--- + +## Phase 2: Install R25 sbcli control plane + +```bash +# Clone sbcli R25 branch +git clone --branch remove_snode_init_container https://github.com/simplyblock-io/sbcli.git sbcli-r25 + +# Install sbcli chart +cd sbcli-r25/simplyblock_core/scripts/charts/ +helm dependency build . +helm upgrade --install sbcli \ + --namespace simplyblock \ + --create-namespace \ + --timeout 10m \ + --set ingress-nginx.controller.admissionWebhooks.enabled=false \ + ./ + +# Wait for control plane pods +kubectl wait --for=condition=Ready pods --all -n simplyblock --timeout=300s \ + --field-selector=status.phase!=Succeeded || true +kubectl get pods -n simplyblock +``` + +--- + +## Phase 3: Create R25 cluster + +```bash +NAMESPACE=simplyblock + +# Find admin pod +ADMIN_POD=$(kubectl -n $NAMESPACE get pods --no-headers | grep -i "admin-control\|webappapi" | grep "Running" | head -1 | awk '{print $1}') +echo "Admin pod: $ADMIN_POD" + +# Wait for FDB to be ready +for i in $(seq 1 30); do + FDB_STATUS=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbcli-dev cluster list 2>&1) || true + if echo "$FDB_STATUS" | grep -qiE 'Connection.*invalid|Error.*reading.*FDB|not found'; then + echo "FDB not ready yet ($i/30): $(echo "$FDB_STATUS" | head -1)" + sleep 10 + else + echo "FDB appears ready" + break + fi +done + +# Get mgmt IP from first worker +MGMT_IP=$(kubectl get node worker-0.ocp.simplyblock.ai -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') +echo "MGMT_IP=$MGMT_IP" + +# Create cluster +kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbcli-dev -d --dev cluster create \ + --mgmt-ip "$MGMT_IP" \ + --mode kubernetes \ + --disable-monitoring + +# Get cluster ID and secret +CLUSTER_ID=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbcli-dev cluster list --json 2>/dev/null | jq -r '.[0].id // .[0].uuid // empty') +CLUSTER_SECRET=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \ + sbcli-dev cluster get-secret "$CLUSTER_ID" 2>/dev/null | tr -d '[:space:]') + +echo "CLUSTER_ID=$CLUSTER_ID" +echo "CLUSTER_SECRET=$CLUSTER_SECRET" + +# Validate secret +if [ -z "$CLUSTER_SECRET" ] || [ "${#CLUSTER_SECRET}" -gt 100 ]; then + echo "ERROR: CLUSTER_SECRET invalid (length=${#CLUSTER_SECRET})" +fi +``` + +--- + +## Phase 4: Label workers + Install R25 spdk-csi chart + +```bash +NAMESPACE=simplyblock + +# Label workers +for NODE in worker-0.ocp.simplyblock.ai worker-1.ocp.simplyblock.ai worker-2.ocp.simplyblock.ai worker-3.ocp.simplyblock.ai worker-4.ocp.simplyblock.ai worker-5.ocp.simplyblock.ai; do + kubectl label node "$NODE" io.simplyblock.node-type=simplyblock-storage-plane --overwrite +done + +# Clone simplyblock-operator (R25 CSI chart branch) +git clone --branch v0.2.4 https://github.com/simplyblock/simplyblock-operator.git simplyblock-operator-r25 + +# Find spdk-csi chart directory +if [ -d "simplyblock-operator-r25/csi-driver/charts/spdk-csi/latest/spdk-csi" ]; then + cd simplyblock-operator-r25/csi-driver/charts/spdk-csi/latest/spdk-csi/ +elif [ -d "simplyblock-operator-r25/charts/spdk-csi/latest/spdk-csi" ]; then + cd simplyblock-operator-r25/charts/spdk-csi/latest/spdk-csi/ +fi +helm dependency build . 2>/dev/null || true + +# Install spdk-csi (use the CLUSTER_ID and CLUSTER_SECRET from Phase 3) +# Replace with the base image tag used in the run +helm install -n simplyblock --create-namespace spdk-csi ./ \ + --set csiConfig.simplybk.uuid="${CLUSTER_ID}" \ + --set csiConfig.simplybk.ip="http://simplyblock-webappapi.simplyblock:5000" \ + --set csiSecret.simplybk.secret="${CLUSTER_SECRET}" \ + --set logicalVolume.pool_name="testing1" \ + --set image.simplyblock.tag="" \ + --set image.csi.tag="v0.2.4" \ + --set logicalVolume.numDataChunks=1 \ + --set logicalVolume.numParityChunks=1 \ + --set storageclass.volumeBindingMode=Immediate \ + --set cachingnode.create=false \ + --set logicalVolume.encryption=false \ + --set storagenode.ifname=br-ex \ + --set storagenode.create=true \ + --set storagenode.numPartitions=1 \ + --set storagenode.coresPercentage=50 \ + --set image.storageNode.tag="v0.1.8" +``` + +--- + +## Phase 5: Wait for R25 cluster active + +```bash +NAMESPACE=simplyblock +ADMIN_POD=$(kubectl -n $NAMESPACE get pods --no-headers | grep -i "admin-control\|webappapi" | grep "Running" | head -1 | awk '{print $1}') + +# Wait for storage nodes to register +for i in $(seq 1 60); do + SN_COUNT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbcli-dev sn list --json 2>/dev/null | jq 'length' 2>/dev/null || echo "0") + echo "Storage nodes: $SN_COUNT/6 ($i/60)" + [ "$SN_COUNT" -ge 6 ] && break + sleep 10 +done + +# Wait for all online +for i in $(seq 1 60); do + ONLINE=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbcli-dev sn list --json 2>/dev/null | jq '[.[] | select(.status == "online")] | length' 2>/dev/null || echo "0") + echo "Online: $ONLINE/6 ($i/60)" + [ "$ONLINE" -ge 6 ] && break + sleep 10 +done + +# Activate cluster (R25 needs manual activation) +kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbcli-dev cluster activate "$CLUSTER_ID" + +# Verify +kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbcli-dev cluster list +kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbcli-dev sn list +``` + +--- + +## At this point — R25 is fully set up + +Now observe and record the FDB state, then run the upgrade uninstall to see where FDB disappears. + +### Key finding from dev + +`kubectl annotate` on live resources does NOT protect against `helm uninstall`. Helm reads +annotations from its stored release manifest (in `sh.helm.release.v1.*` secrets), not from +the live object in etcd. The correct fix is to add `helm.sh/resource-policy: keep` directly +in the Helm chart templates and run `helm upgrade` before uninstall. + +Also: `helm uninstall` does NOT remove CRDs, so CRD cascade deletion is not the issue. + +### Observation checkpoint: Before uninstall + +Check whether the R25 chart templates already include `helm.sh/resource-policy: keep` for FDB resources: + +```bash +echo "=== FDB CRDs ===" +kubectl get crd | grep foundationdb + +echo "=== FoundationDBCluster CR ===" +kubectl get foundationdbcluster -n simplyblock + +echo "=== FDB pods ===" +kubectl get pods -n simplyblock | grep fdb + +echo "=== FDB controller-manager deployment ===" +kubectl get deployment -n simplyblock | grep fdb + +echo "=== What helm charts manage ===" +echo "--- sbcli chart manifest (FDB references) ---" +helm get manifest sbcli -n simplyblock 2>/dev/null | grep -i "kind:\|name:.*fdb" | head -30 +echo "--- spdk-csi chart manifest (FDB references) ---" +helm get manifest spdk-csi -n simplyblock 2>/dev/null | grep -i "kind:\|name:.*fdb" | head -30 + +echo "=== Check if resource-policy: keep is in Helm's stored manifest (THIS IS THE KEY CHECK) ===" +echo "--- sbcli chart: resource-policy annotations ---" +helm get manifest sbcli -n simplyblock 2>/dev/null | grep -B10 "resource-policy" || echo "NO resource-policy annotations found in sbcli manifest" +echo "--- spdk-csi chart: resource-policy annotations ---" +helm get manifest spdk-csi -n simplyblock 2>/dev/null | grep -B10 "resource-policy" || echo "NO resource-policy annotations found in spdk-csi manifest" +``` + +### Test: Run helm uninstall sbcli and observe + +```bash +# Uninstall sbcli chart (control plane) +helm uninstall sbcli -n simplyblock --no-hooks --timeout 60s + +# Immediately check what's left +echo "=== After 'helm uninstall sbcli' ===" + +echo "=== FDB CRDs ===" +kubectl get crd | grep foundationdb + +echo "=== FoundationDBCluster CR ===" +kubectl get foundationdbcluster -n simplyblock + +echo "=== FDB pods ===" +kubectl get pods -n simplyblock | grep fdb + +echo "=== FDB controller-manager deployment ===" +kubectl get deployment -n simplyblock | grep fdb + +echo "=== All remaining pods ===" +kubectl get pods -n simplyblock +``` + +If FDB is still present after `helm uninstall sbcli`, then also test: + +```bash +# Uninstall spdk-csi chart +helm uninstall spdk-csi -n simplyblock --no-hooks --timeout 60s + +# Check again +echo "=== After 'helm uninstall spdk-csi' ===" +kubectl get crd | grep foundationdb +kubectl get foundationdbcluster -n simplyblock +kubectl get pods -n simplyblock | grep fdb +kubectl get deployment -n simplyblock | grep fdb +kubectl get pods -n simplyblock +``` diff --git a/UPGRADE.md b/UPGRADE.md new file mode 100644 index 0000000000..f1e8b17278 --- /dev/null +++ b/UPGRADE.md @@ -0,0 +1,1114 @@ +# K8s Upgrade: R25.x (Helm) to R26+ (Operator) — End-to-End Operations Guide + +This document covers the complete upgrade lifecycle from an R25.x legacy Helm-based deployment +to the R26+ operator-based architecture. It is intended for the dev team to confirm that the +setup, upgrade steps, and validation procedures are correct. + +--- + +## Overview + +| Phase | Description | Downtime? | +|-------|-------------|-----------| +| Phase 1 | Deploy R25.x cluster using legacy Helm charts | No (fresh setup) | +| Phase 2 | Pre-upgrade data setup — pool, PVCs, FIO, snapshots, clones, MD5 | No | +| Phase 3 | Maintenance window — 11-step migration from Helm to Operator | **Yes** | +| Phase 4 | Post-upgrade validation — verify old data, new provisioning, outages | No | + +--- + +## Phase 1: Deploy R25.x Cluster (Legacy Helm Charts) + +R25.x uses two Helm charts (`sbcli` + `spdk-csi`) with no operator. The cluster is created +manually via the admin pod. + +> **Branch**: Use PR #1044 branch (`remove_snode_init_container`) or the relevant R25.x tag. + +### 1.1 Install the `sbcli` Helm Chart + +This deploys the management plane (API, admin pod, FoundationDB). + +```bash +helm upgrade --install sbcli ./charts/sbcli \ + --namespace simplyblock --create-namespace \ + --set image.simplyblock.repository= \ + --set image.simplyblock.tag= +``` + +Wait for all pods to be ready: + +```bash +kubectl wait --for=condition=Ready pods --all -n simplyblock --timeout=300s +``` + +### 1.2 Create the Cluster via Admin Pod + +Exec into the admin pod and run `sbcli-dev cluster create`: + +```bash +# Find the admin pod +ADMIN_POD=$(kubectl get pods -n simplyblock -l app=simplyblock-admin -o jsonpath='{.items[0].metadata.name}') + +# Exec into it +kubectl exec -it -n simplyblock $ADMIN_POD -- bash + +# Inside the pod: +sbcli-dev cluster create \ + --fabric-type tcp \ + --ndcs 1 \ + --npcs 0 \ + --single-node false +``` + +**Expected output**: Cluster UUID and secret are printed. Save these — they are needed for the upgrade secret in Phase 3. + +``` +Cluster ID: +Cluster Secret: +``` + +### 1.3 Label Worker Nodes for Storage Plane + +The R25 spdk-csi chart uses the `io.simplyblock.node-type` label to discover which +worker nodes should run storage node pods. Label all workers before installing the chart: + +```bash +for NODE in ; do + kubectl label node "$NODE" io.simplyblock.node-type=simplyblock-storage-plane --overwrite +done +``` + +### 1.4 Install the `spdk-csi` Helm Chart (Includes Storage Node Creation) + +This deploys the CSI driver and creates storage nodes via `storagenode.create=true`. +Use the cluster UUID, secret, and pool name from step 1.2. + +```bash +helm install -n simplyblock --create-namespace spdk-csi ./ \ + --set csiConfig.simplybk.uuid= \ + --set csiConfig.simplybk.ip=http://simplyblock-webappapi.simplyblock:5000 \ + --set csiSecret.simplybk.secret= \ + --set logicalVolume.pool_name=testing1 \ + --set image.simplyblock.tag=remove_snode_init_container \ + --set image.csi.tag=v0.2.4 \ + --set logicalVolume.numDataChunks=1 \ + --set logicalVolume.numParityChunks=1 \ + --set storageclass.volumeBindingMode=Immediate \ + --set cachingnode.create=false \ + --set logicalVolume.encryption=false \ + --set storagenode.ifname=ens18 \ + --set storagenode.create=true \ + --set storagenode.numPartitions=0 \ + --set image.storageNode.tag=v0.1.8 +``` + +Wait for all pods (CSI + storage nodes) to be ready: + +```bash +kubectl wait --for=condition=Ready pods -l app=spdk-csi -n simplyblock --timeout=300s +``` + +Verify storage nodes are online: + +```bash +sbcli-dev sn list +``` + +**Expected**: All storage nodes show `online` status. + +### 1.5 Verify R25.x Cluster + +```bash +# Cluster should be active +sbcli-dev cluster list + +# All storage nodes online +sbcli-dev sn list + +# CSI pods running +kubectl get pods -n simplyblock -l app=spdk-csi +``` + +--- + +## Phase 2: Pre-Upgrade Data Setup + +Create data before the upgrade so we can verify integrity after migration. + +### 2.1 Create Storage Pool + +```bash +sbcli-dev pool add upgrade-test-pool +``` + +### 2.2 Create StorageClass and VolumeSnapshotClass + +```yaml +# storageclass.yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: simplyblock-csi-sc +provisioner: csi.simplyblock.io +parameters: + cluster_id: "" + pool_name: "upgrade-test-pool" + ndcs: "1" + npcs: "0" +reclaimPolicy: Delete +volumeBindingMode: Immediate +``` + +```yaml +# volumesnapshotclass.yaml +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: simplyblock-csi-snapshotclass +driver: csi.simplyblock.io +deletionPolicy: Delete +``` + +```bash +kubectl apply -f storageclass.yaml +kubectl apply -f volumesnapshotclass.yaml +``` + +### 2.3 Create PVCs (One Per Storage Node) + +Create one PVC per storage node to spread data across the cluster: + +```yaml +# For each storage node, create a PVC: +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: upgrade-pvc-0 + namespace: default +spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 10Gi + storageClassName: simplyblock-csi-sc +``` + +Wait for all PVCs to be bound: + +```bash +kubectl get pvc -w +``` + +### 2.4 Run FIO with MD5 Verification on Each PVC + +Run FIO with `verify=md5` to write data and compute checksums: + +```ini +[global] +name=pre-upgrade-fio +filename_format=/spdkvol/fio-data.$jobnum +rw=randrw +rwmixread=50 +bs=4k +iodepth=1 +direct=1 +ioengine=libaio +size=1G +numjobs=1 +time_based +runtime=120 +group_reporting +verify=md5 +verify_dump=1 +verify_fatal=1 +verify_backlog=4096 +verify_backlog_batch=32 + +[job1] +``` + +Deploy as a K8s Job with the FIO ConfigMap mounted, one job per PVC. + +**Expected**: All FIO jobs complete successfully with 0 errors. The MD5 verification +headers are written into the data files on the PVC. + +### 2.5 Create Snapshots of Each PVC + +After FIO completes, snapshot each PVC to preserve the verified data state: + +```yaml +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshot +metadata: + name: snap-upgrade-pvc-0 +spec: + volumeSnapshotClassName: simplyblock-csi-snapshotclass + source: + persistentVolumeClaimName: upgrade-pvc-0 +``` + +Wait for snapshots to be ready: + +```bash +kubectl get volumesnapshot -w +``` + +### 2.6 Create Clones from Snapshots + +Create a clone PVC from each snapshot and run FIO on the clone: + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clone-upgrade-pvc-0 +spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 10Gi + storageClassName: simplyblock-csi-sc + dataSource: + name: snap-upgrade-pvc-0 + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io +``` + +Run FIO on clones with `verify=md5` to confirm the cloned data is intact. + +**Expected**: All clone FIO jobs pass MD5 verification (the cloned data matches the original). + +### 2.7 Capture Pre-Upgrade State + +Record the following before starting the upgrade: + +- [ ] Cluster UUID and secret +- [ ] All storage node UUIDs and their status (`sbcli-dev sn list`) +- [ ] Pool names/IDs +- [ ] PVC names and their bound PV names (`kubectl get pvc`) +- [ ] Volume snapshot names (`kubectl get volumesnapshot`) +- [ ] Clone PVC names +- [ ] FIO job results (all passed with 0 verify errors) +- [ ] `sbcli-dev lvol list` output — note all lvol IDs + +--- + +## Phase 3: Maintenance Window Upgrade (R25 to R26) + +> **WARNING**: Storage nodes are shut down during this phase. Volumes are +> unavailable to workloads. Plan for downtime and notify teams. + +### Step 1 — Ensure FDB Resources Have `helm.sh/resource-policy: keep` in Chart + +There are 8 FDB resources that must survive `helm uninstall`: + +| Kind | Name | +|------|------| +| Deployment | simplyblock-fdb-controller-manager | +| ServiceAccount | simplyblock-fdb-controller-manager | +| ClusterRole | simplyblock-fdb-manager-role | +| ClusterRole | simplyblock-fdb-manager-clusterrole | +| RoleBinding | simplyblock-fdb-manager-rolebinding | +| ClusterRoleBinding | simplyblock-fdb-manager-clusterrolebinding | +| FoundationDBCluster | simplyblock-fdb-cluster | +| ConfigMap | simplyblock-fdb-cluster-config | + +> **Why the ConfigMap?** The `simplyblock-fdb-cluster-config` ConfigMap contains the FDB +> cluster connection file. Admin pods mount it as volume `fdb-cluster-file`. If this +> ConfigMap is deleted during `helm uninstall sbcli`, admin pods will be stuck in +> `ContainerCreating` and all `sbcli`/`sbctl` commands will fail. + +> **IMPORTANT: `kubectl annotate` on live resources is NOT effective for `helm uninstall`.** +> Helm reads annotations from its stored release manifest (in `sh.helm.release.v1.*` +> secrets), not from the live object in etcd. Annotating a live resource with +> `kubectl annotate` only patches etcd — Helm's copy is unchanged and it will still +> delete the resource on uninstall. +> +> **The correct approach** is to add `helm.sh/resource-policy: keep` directly in the +> Helm chart templates so it is baked into the stored manifest. This requires a chart +> change + `helm upgrade` before uninstall. + +#### Option A — Chart template fix (correct way, requires chart change) + +Add to each FDB template in the chart: + +```yaml +metadata: + annotations: + "helm.sh/resource-policy": keep +``` + +Then run `helm upgrade` to persist the annotation into Helm's release secret before +running `helm uninstall`. + +#### Option B — Patch the Helm release secret directly (workaround without chart change) + +If the chart cannot be modified (e.g., upgrading from an older R25 chart), you can +patch the stored Helm release manifest directly: + +```bash +# 1. Get the latest Helm release secret +SECRET_NAME=$(kubectl get secrets -n simplyblock -l owner=helm,name=sbcli \ + --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}') + +# 2. Decode, decompress, patch, recompress, re-encode the release data +kubectl get secret "$SECRET_NAME" -n simplyblock -o jsonpath='{.data.release}' \ + | base64 -d | base64 -d | gzip -d > /tmp/helm-release.json + +# 3. Inject keep annotation into FDB resource manifests in the release +# (This is complex — use the chart fix if possible) +``` + +#### Option C — `kubectl annotate` live resources (limited effectiveness) + +> **WARNING**: This only works if Helm happens to check live objects, which standard +> Helm does NOT do. Listed here for reference but **Option A is strongly recommended**. + +```bash +kubectl annotate deployment simplyblock-fdb-controller-manager -n simplyblock \ + helm.sh/resource-policy=keep --overwrite +kubectl annotate serviceaccount simplyblock-fdb-controller-manager -n simplyblock \ + helm.sh/resource-policy=keep --overwrite +kubectl annotate clusterrole simplyblock-fdb-manager-role \ + helm.sh/resource-policy=keep --overwrite +kubectl annotate clusterrole simplyblock-fdb-manager-clusterrole \ + helm.sh/resource-policy=keep --overwrite +kubectl annotate rolebinding simplyblock-fdb-manager-rolebinding -n simplyblock \ + helm.sh/resource-policy=keep --overwrite +kubectl annotate clusterrolebinding simplyblock-fdb-manager-clusterrolebinding \ + helm.sh/resource-policy=keep --overwrite +kubectl annotate foundationdbcluster simplyblock-fdb-cluster -n simplyblock \ + helm.sh/resource-policy=keep --overwrite +kubectl annotate configmap simplyblock-fdb-cluster-config -n simplyblock \ + helm.sh/resource-policy=keep --overwrite +``` + +**Verify** (check Helm's stored manifest, not just live object): + +```bash +# Check live object (may not reflect what Helm sees) +kubectl get deployment simplyblock-fdb-controller-manager -n simplyblock \ + -o jsonpath='{.metadata.annotations.helm\.sh/resource-policy}' +# Expected: keep + +# Check Helm's stored manifest (this is what actually matters) +helm get manifest sbcli -n simplyblock 2>/dev/null | grep -A5 "simplyblock-fdb-controller-manager" | grep resource-policy +# Expected: "helm.sh/resource-policy": keep +``` + +### Step 2 — Shut Down All Storage Nodes + +Force-shutdown each storage node. Using `--force` combines suspend and shutdown in one +command and avoids failures when some nodes are already in a non-online state: + +```bash +for NODE_ID in $(sbctl sn list | grep -v "offline" | awk '{print $2}'); do + sbctl sn shutdown "$NODE_ID" --force +done +``` + +Wait for all nodes to reach `offline`: + +```bash +sbctl sn list +# Expected: All nodes show "offline" status +``` + +### Step 2.1 — Disable Auto-Restart on All Nodes (Safety Net) + +> **Status**: The R26 operator now skips creating `node_restart` tasks for nodes +> that were already offline before the operator started. This makes Step 2.1 +> optional in most cases. However, if upgrading to an older R26 build or if the +> fix regresses, this step prevents the operator's tasks-runner from creating +> stale `node_restart` tasks that block the explicit `sn restart` in Step 10 +> (there is no `--force` flag for restart). + +```bash +for NODE_ID in $(sbctl sn list --json | jq -r '.[].UUID'); do + sbctl --dev sn set "$NODE_ID" auto_restart_disabled true +done +``` + +If stale restart tasks already exist (e.g. from a previous failed run), cancel +them before proceeding: + +```bash +# List tasks +sbctl cluster list-tasks "$CLUSTER_ID" --limit 0 + +# Cancel any running/new node_restart tasks +for TASK_ID in $(sbctl cluster list-tasks "$CLUSTER_ID" --json --limit 0 \ + | jq -r '.[] | select(.function=="node_restart" and (.status=="running" or .status=="new")) | .id'); do + sbctl cluster cancel-task "$CLUSTER_ID" "$TASK_ID" +done +``` + +### Step 3 — Uninstall the `spdk-csi` Helm Chart + +```bash +helm uninstall spdk-csi --namespace simplyblock --wait +``` + +### Step 3.1 — Delete Orphaned Snapshot Controller + +The `spdk-csi` chart deploys a `simplyblock-snapshot-controller` Deployment in +`kube-system` with `helm.sh/resource-policy: keep`. This means it survives the +`helm uninstall` above but retains stale ownership annotations pointing to the +old `spdk-csi` release. When the new `simplyblock-operator` chart tries to +install its own copy, Helm fails with: + +``` +rendered manifests contain a resource that already exists. Unable to continue +with install: existing resource conflict: namespace: kube-system, name: +simplyblock-snapshot-controller, existing_kind: apps/v1, Kind=Deployment, +new_kind: apps/v1, Kind=Deployment +``` + +**Fix**: Delete the orphaned deployment after uninstalling `spdk-csi`: + +```bash +kubectl delete deployment simplyblock-snapshot-controller -n kube-system --ignore-not-found +``` + +### Step 4 — Uninstall the `sbcli` Helm Chart + +```bash +helm uninstall sbcli --namespace simplyblock --wait +``` + +FDB resources survive because the chart templates include `helm.sh/resource-policy: keep` +annotations (see Step 1). If the chart does NOT have these annotations, FDB will be +deleted by `helm uninstall` and the database will be destroyed. + +**Verify FDB still running** (CRITICAL — if any of these fail, STOP and investigate): + +```bash +# 1. FoundationDBCluster CR must still exist +kubectl get foundationdbcluster simplyblock-fdb-cluster -n simplyblock +# Expected: Shows the cluster resource + +# 2. FDB controller-manager deployment must still exist +kubectl get deployment simplyblock-fdb-controller-manager -n simplyblock +# Expected: Shows the deployment + +# 3. FDB pods must still be running +kubectl get pods -n simplyblock -l foundationdb.org/fdb-cluster-name=simplyblock-fdb-cluster +# Expected: Multiple FDB pods in Running state + +# 4. FDB CRDs must still exist +kubectl get crd foundationdbclusters.apps.foundationdb.org +# Expected: Shows the CRD +``` + +> **If FDB resources are missing**: The chart likely does not have `helm.sh/resource-policy: keep` +> in its templates. This must be fixed in the chart (see Step 1, Option A). Note that +> `kubectl annotate` on live objects does NOT protect against `helm uninstall` — Helm reads +> from its stored release manifest, not from etcd. + +### Step 4.1 — Verify FDB Cluster-Config ConfigMap + +Check that the `simplyblock-fdb-cluster-config` ConfigMap survived the helm uninstall. +Admin pods mount this ConfigMap as volume `fdb-cluster-file` — without it they will be +stuck in `ContainerCreating`. + +```bash +kubectl get configmap simplyblock-fdb-cluster-config -n simplyblock +``` + +If the ConfigMap is missing, recreate it from a running FDB pod: + +```bash +# Extract the cluster file content from any FDB pod +FDB_POD=$(kubectl get pods -n simplyblock \ + -l foundationdb.org/fdb-cluster-name=simplyblock-fdb-cluster \ + -o jsonpath='{.items[0].metadata.name}') + +CLUSTER_FILE=$(kubectl exec -n simplyblock "$FDB_POD" -- \ + cat /var/dynamic-conf/fdb.cluster 2>/dev/null) + +# Recreate the ConfigMap +kubectl create configmap simplyblock-fdb-cluster-config \ + -n simplyblock \ + --from-literal=cluster-file="$CLUSTER_FILE" +``` + +**Verify**: + +```bash +kubectl get configmap simplyblock-fdb-cluster-config -n simplyblock \ + -o jsonpath='{.data.cluster-file}' +# Expected: A non-empty FDB cluster connection string +``` + +### Step 5 — Create the Upgrade Secret + +The upgrade secret tells the operator to adopt the existing cluster instead of creating a new one. + +```bash +kubectl create secret generic simplyblock--upgrade \ + --namespace simplyblock \ + --from-literal=uuid= \ + --from-literal=secret= +``` + +Example: + +```bash +kubectl create secret generic simplyblock-simplyblock-cluster-upgrade \ + --namespace simplyblock \ + --from-literal=uuid=93cdb610-3a72-464c-b223-fe48327fc329 \ + --from-literal=secret=bdMyLkU5k4H0btBZU5H +``` + +> The secret name **must** match `simplyblock--upgrade` where `CR_NAME` is the +> `metadata.name` of the StorageCluster CR you will apply in Step 7. + +### Step 6 — Install the Operator Helm Chart (FDB Disabled) + +> **Prerequisite — cert-manager (TLS-enabled installs only)**: If the operator chart +> enables TLS (e.g., `simplyblock-webappapi-tls` Certificate resources), `cert-manager` +> must be installed before this step. Without it, Certificate CRDs won't exist and the +> helm install will fail, or the TLS secret will never be created and admin pods will +> fail to start. +> +> ```bash +> # Check if cert-manager CRDs exist +> kubectl get crd certificates.cert-manager.io 2>/dev/null +> +> # If missing, install cert-manager +> kubectl apply -f https://github.com/cert-manager/cert-manager/releases/latest/download/cert-manager.yaml +> kubectl wait --for=condition=Available deployment --all -n cert-manager --timeout=120s +> ``` + +Install the new operator chart with FDB creation disabled (FDB is already running): + +```bash +helm upgrade --install simplyblock-operator ./charts/simplyblock-operator \ + --namespace simplyblock \ + --timeout 10m \ + --set operator.enabled=true \ + --set controlplane.foundationdb.enabled=false \ + --set image.simplyblock.repository= \ + --set image.simplyblock.tag= \ + --set image.operator.repository=simplyblock/simplyblock-operator \ + --set image.operator.tag= \ + --set controlplane.csiHostpathDriver.enabled=true \ + --set controlplane.storageclass.name=local-hostpath \ + --set csiConfig.simplybk.ip=http://simplyblock-webappapi.simplyblock:5000 +``` + +Wait for operator pods: + +```bash +kubectl wait --for=condition=Ready pods --all -n simplyblock \ + --timeout=300s --field-selector=status.phase!=Succeeded +``` + +### Step 6.1 — Shut Down Nodes Again (Prevent Auto-Restart) + +After the operator installs, it may try to auto-restart nodes. Shut them down again +to explicitly set `auto_restart_disabled=True`: + +```bash +for NODE_ID in $(sbctl sn list | grep -E "online|in_creation|reaching" | awk '{print $2}'); do + sbctl --dev sn shutdown "$NODE_ID" +done +``` + +### Step 7 — Apply Custom Resources + +Apply the StorageCluster, Pool, and StorageNodeSet CRs. The operator detects the upgrade +secret and adopts the existing cluster. + +> **IMPORTANT — CR names must match existing backend names:** +> - The **Pool CR** `metadata.name` must match the existing pool name in the R25 cluster +> (e.g., if the pool was created as `testing1` via `sbcli-dev pool add testing1`, the +> Pool CR must use `name: testing1`). This allows the operator to adopt the existing pool. +> - The **StorageCluster CR** `metadata.name` must be consistent with the upgrade secret +> name from Step 5 (`simplyblock--upgrade`). + +```yaml +# storagecluster.yaml +apiVersion: storage.simplyblock.io/v1alpha1 +kind: StorageCluster +metadata: + name: simplyblock-cluster # Must match upgrade secret: simplyblock--upgrade + namespace: simplyblock +spec: + fabricType: tcp + isSingleNode: false + enableNodeAffinity: true + strictNodeAntiAffinity: false + stripe: + dataChunks: 1 + parityChunks: 0 + warningThreshold: + capacity: 95 + provisionedCapacity: 97 + criticalThreshold: + capacity: 96 + provisionedCapacity: 98 +--- +apiVersion: storage.simplyblock.io/v1alpha1 +kind: Pool +metadata: + name: # Must match the pool name from R25 (e.g., testing1) + namespace: simplyblock +spec: + clusterName: simplyblock-cluster +--- +apiVersion: storage.simplyblock.io/v1alpha1 +kind: StorageNodeSet +metadata: + name: simplyblock-node + namespace: simplyblock +spec: + clusterName: simplyblock-cluster + clusterImage: ":" + spdkImage: "" + spdkProxyImage: ":" + mgmtIfname: ens18 + dataIfname: + - enp1s0 + maxLogicalVolumeCount: 30 + enableCpuTopology: true + workerNodes: + - + - + - +``` + +```bash +kubectl apply -f storagecluster.yaml -n simplyblock +``` + +Verify adoption (status should reflect existing UUIDs, not `in_creation`): + +```bash +kubectl get storagecluster -n simplyblock -o yaml +kubectl get storagenode -n simplyblock -o yaml +kubectl get pool -n simplyblock -o yaml +``` + +### Step 8 — Run R25 to R26 Data Migration Script + +Run the migration script inside the admin pod to update storage node fields in the database +(`lvstore_ports`, `lvol_poller_mask`, `lvstore_stack_secondary`): + +```bash +ADMIN_POD=$(kubectl get pods -n simplyblock -l app=simplyblock-admin \ + -o jsonpath='{.items[0].metadata.name}') + +kubectl exec -it -n simplyblock $ADMIN_POD -- bash +``` + +Inside the pod, run: + +```python +from simplyblock_core import utils +from simplyblock_core.db_controller import DBController +db_controller = DBController() + +for snode in db_controller.get_storage_nodes(): + print(f"updating storage node object: {snode.get_id()}") + for node in db_controller.get_storage_nodes(): + if snode.get_id() == node.secondary_node_id: + snode.lvstore_stack_secondary = node.get_id() + break + snode.lvstore_ports = { + snode.lvstore: { + "lvol_subsys_port": snode.lvol_subsys_port, + "hublvol_port": snode.hublvol.nvmf_port + } + } + if snode.lvstore_stack_secondary: + sec = db_controller.get_storage_node_by_id(snode.lvstore_stack_secondary) + snode.lvstore_ports[sec.lvstore] = { + "lvol_subsys_port": sec.lvol_subsys_port, + "hublvol_port": sec.hublvol.nvmf_port, + } + if snode.poller_cpu_cores: + snode.lvol_poller_mask = utils.generate_mask([snode.poller_cpu_cores[-1]]) + if len(snode.poller_cpu_cores) > 1: + snode.poller_cpu_cores = snode.poller_cpu_cores[:-1] + snode.pollers_mask = utils.generate_mask(snode.poller_cpu_cores) + + snode.write_to_db() + +print("Creating mini lvol objects") +for lvol in db_controller.get_all_lvols(): + lvol.write_to_db() + +print("Creating mini Snapshots objects") +for snap in db_controller.get_snapshots(): + snap.write_to_db() + +print("done") +``` + +**Expected**: Output ends with `done`. After running, `sbctl sn list` shows `LVS Ports` +column values populated. + +### Step 9 — Patch Backend Objects with CR References + +Register the K8s CR details on each backend object so the operator and backend stay in sync. + +**Storage Cluster**: + +```bash +export CLUSTER_UUID= +export CLUSTER_CR_NAME=simplyblock-cluster + +sbctl --dev cluster set $CLUSTER_UUID cr_plural storageclusters +sbctl --dev cluster set $CLUSTER_UUID cr_namespace simplyblock +sbctl --dev cluster set $CLUSTER_UUID cr_name $CLUSTER_CR_NAME +``` + +**Storage Nodes** (repeat for each): + +```bash +for NODE_ID in $(sbctl sn list | grep -E "offline|in_creation" | awk '{print $2}'); do + sbctl --dev sn set "$NODE_ID" cr_plural storagenodesets + sbctl --dev sn set "$NODE_ID" cr_namespace simplyblock + sbctl --dev sn set "$NODE_ID" cr_name simplyblock-node +done +``` + +### Step 9.1 — Cancel Stale Restart Tasks (If Needed) + +> **Status**: With the R26 operator fix (see Step 2.1), stale tasks should not +> appear. This step is a safety net for older operator builds or regressions. + +If any `node_restart` tasks were created by the operator's tasks-runner between +Step 6 (operator install) and Step 10, they will block `sn restart`. Check and +cancel them: + +```bash +# Check for stale node_restart tasks +sbctl cluster list-tasks "$CLUSTER_ID" --limit 0 + +# Cancel any that are running or new +for TASK_ID in $(sbctl cluster list-tasks "$CLUSTER_ID" --json --limit 0 \ + | jq -r '.[] | select(.function=="node_restart" and (.status=="running" or .status=="new")) | .id'); do + echo "Cancelling stale task: $TASK_ID" + sbctl cluster cancel-task "$CLUSTER_ID" "$TASK_ID" +done +``` + +### Step 10 — Restart Storage Nodes One at a Time + +Restart each storage node with the new SPDK image and proxy image. + +> **IMPORTANT — Maintenance upgrade**: In a maintenance upgrade all nodes start +> offline. The cluster **cannot** become `active` until every node is back online. +> Do **not** wait for cluster `active` between individual node restarts — only +> wait for each node to reach `online`, then proceed to the next. Check cluster +> `active` only after **all** nodes have been restarted. + +```bash +export SPDK_IMAGE= +export SPDK_PROXY_IMAGE= + +for NODE_ID in $(sbctl sn list --json | jq -r '.[].UUID'); do + echo "Restarting node: $NODE_ID" + sbctl -d --dev sn restart "$NODE_ID" \ + --spdk-image "$SPDK_IMAGE" \ + --spdk-proxy-image "$SPDK_PROXY_IMAGE" + + # Wait for this node to come online (up to 10 minutes) + while ! sbctl sn list --json | jq -e ".[] | select(.UUID==\"$NODE_ID\" and .Status==\"online\")" > /dev/null 2>&1; do + sleep 5 + done + echo " Node $NODE_ID is online" + + sleep 10 # brief pause before next node +done +``` + +> **Note — Admin-control pod recycling**: During Step 10, the R26 operator may +> recycle the `simplyblock-admin-control` pods as nodes come back online +> (deployment rollout). If `kubectl exec` commands fail with `error: unable to +> upgrade connection: pod does not exist`, wait a few seconds and retry with the +> new pod name: +> +> ```bash +> ADMIN_POD=$(kubectl get pods -n simplyblock -l app=simplyblock-admin-control \ +> -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) +> ``` + +### Step 10.1 — Wait for Cluster Active and Health Checks + +After all nodes are restarted, wait for the cluster to become `active` and for +all node health checks to settle to `True`. The `health_check` field may +remain `None` or `False` for 30-60 seconds after a node comes online while the +monitoring loop catches up. + +```bash +# Wait for cluster active +while [ "$(sbctl cluster list --json | jq -r '.[0].Status')" != "ACTIVE" ]; do + echo "Waiting for cluster to become active..." + sleep 10 +done +echo "Cluster is active" + +# Wait for all nodes to report health_check=True (up to 2 minutes) +TIMEOUT=120 +while [ $TIMEOUT -gt 0 ]; do + UNHEALTHY=$(sbctl sn list --json | jq '[.[] | select(.Health != "True")] | length') + if [ "$UNHEALTHY" -eq 0 ]; then + echo "All nodes are healthy" + break + fi + echo " $UNHEALTHY node(s) still settling health_check, retrying in 10s..." + sleep 10 + TIMEOUT=$((TIMEOUT - 10)) +done + +if [ $TIMEOUT -le 0 ]; then + echo "WARNING: Some nodes still have health_check != True after 120s" + sbctl sn list +fi +``` + +### Step 11 — Restart Workload Pods + +Once all storage nodes are online and the cluster is active, restart application +pods to re-establish NVMe connections: + +```bash +kubectl rollout restart deployment/ -n +``` + +Or for all deployments in a namespace: + +```bash +kubectl get deployments -n -o name | \ + xargs -I{} kubectl rollout restart {} -n +``` + +--- + +## Phase 4: Post-Upgrade Validation + +### 4.1 Verify Old Data Integrity (MD5) + +Re-run FIO in verify-only mode on the original PVCs to confirm the data written +in Phase 2 is intact after migration: + +```ini +[global] +name=post-upgrade-verify +filename_format=/spdkvol/fio-data.$jobnum +rw=read +bs=4k +iodepth=1 +direct=1 +ioengine=libaio +size=1G +numjobs=1 +verify=md5 +verify_only +verify_dump=1 +verify_fatal=1 + +[job1] +``` + +**Expected**: All FIO verify-only jobs complete with 0 errors. The data written +pre-upgrade is intact. + +### 4.2 Run FIO on Existing (Old) PVCs + +Run a fresh FIO with `verify=md5` on the old PVCs to confirm read/write IO works +post-upgrade: + +```ini +[global] +name=post-upgrade-io +filename_format=/spdkvol/fio-post.$jobnum +rw=randrw +rwmixread=50 +bs=4k +direct=1 +ioengine=libaio +size=1G +numjobs=1 +time_based +runtime=120 +verify=md5 +verify_dump=1 +verify_fatal=1 + +[job1] +``` + +**Expected**: IO completes successfully with 0 verify errors. + +### 4.3 Create New Snapshots and Clones on Old PVCs + +Take new snapshots of the old PVCs (post-upgrade data) and create clones from them: + +- [ ] Create VolumeSnapshot for each old PVC +- [ ] Wait for snapshot to be ready +- [ ] Create clone PVC from the snapshot +- [ ] Run FIO on clone with `verify=md5` + +**Expected**: Snapshots and clones work correctly on the upgraded cluster. + +### 4.4 Create New PVCs (Fresh Provisioning) + +Verify that new volume provisioning works end-to-end on the upgraded cluster: + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: post-upgrade-new-pvc +spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 10Gi + storageClassName: simplyblock-csi-sc +``` + +- [ ] PVC should bind successfully +- [ ] Run FIO with `verify=md5` — should complete with 0 errors + +### 4.5 Snapshot and Clone New PVCs + +- [ ] Create VolumeSnapshot of the new PVC +- [ ] Wait for snapshot ready +- [ ] Create clone PVC from snapshot +- [ ] Run FIO on clone with `verify=md5` + +**Expected**: Full snapshot/clone lifecycle works on newly provisioned volumes. + +### 4.6 Node Outage Test + +Verify HA works post-upgrade by simulating node failures: + +**Single node outage**: + +1. Identify a storage node hosting one of the PVCs +2. Shut down that node: `sbctl sn shutdown ` +3. Verify FIO continues on the PVC (HA should redirect IO) +4. Restart the node: `sbctl sn restart ` +5. Wait for cluster active and node online +6. Verify all FIO jobs pass + +**Multi-node outage** (if running with parity, e.g., ndcs=2 npcs=1): + +1. Shut down two nodes simultaneously +2. Verify IO continues on PVCs with sufficient redundancy +3. Restart nodes one at a time +4. Wait for cluster active + +**Expected**: IO continues without errors during single-node outage. +After node restart, cluster returns to active and all data is intact. + +### 4.7 Final Checklist + +| Check | Command | Expected | +|-------|---------|----------| +| Cluster active | `sbctl cluster list` | `active` | +| All nodes online | `sbctl sn list` | All `online` | +| LVS Ports populated | `sbctl sn list` | Non-empty values | +| Old PVCs bound | `kubectl get pvc` | All `Bound` | +| New PVCs bound | `kubectl get pvc` | All `Bound` | +| Snapshots ready | `kubectl get volumesnapshot` | All `readyToUse: true` | +| Clones bound | `kubectl get pvc` (clone PVCs) | All `Bound` | +| Operator CRs adopted | `kubectl get storagecluster -o yaml` | Shows existing UUIDs | +| FIO verify pass | FIO job logs | `0 verify errors` | +| CR refs patched | `sbctl --dev cluster get ` | `cr_name`, `cr_namespace` set | + +--- + +## Rollback + +If the upgrade fails **before Step 3** (Helm uninstall), re-install the original Helm charts. + +After Step 3, rollback requires: + +1. Restoring from the FDB PVC data +2. Re-installing the original `sbcli` and `spdk-csi` charts +3. Manual recovery of storage node state + +> Full rollback procedures are not covered here. The recommendation is to snapshot/backup +> the FDB PVC before starting the maintenance window. + +--- + +## Operational Notes (Lessons Learned from E2E Runs) + +These notes capture real-world issues found during automated and manual upgrade +testing that operators should be aware of. + +### 1. Do NOT wait for cluster active between node restarts (maintenance path) + +In a maintenance upgrade, all storage nodes start offline. The cluster enters +`suspended` state because it has no quorum. **The cluster cannot become `active` +until all (or most) nodes are back online.** If you wait for cluster `active` +after restarting each individual node, you will hang indefinitely after the +first node. + +**Correct approach**: Restart each node one at a time, wait only for that node +to reach `online` status, then immediately start the next. Only check for +cluster `active` after **all** nodes have been restarted (Step 10.1). + +This does NOT apply to rolling upgrades, where only one node is down at a time +and the cluster stays active throughout. + +### 2. Health check settling delay after restart + +After a node restarts, its `health_check` field in the database transitions +through `None` → `False` → `True` as the monitoring loop catches up. This can +take 20-60 seconds. **Do not assert `health_check == True` immediately** after +a node comes online — poll with a timeout (120 seconds recommended). + +### 3. Admin-control pod recycling during node restarts + +When storage nodes come back online, the R26 operator may trigger a rollout of +the `simplyblock-admin-control` deployment. If your automation uses `kubectl exec` +to run `sbctl` commands via a cached admin pod name, the cached name may become +stale. Symptoms: + +- `error: unable to upgrade connection: pod does not exist` +- `json.JSONDecodeError: Expecting value: line 1 column 1` (empty stdout) + +**Mitigation**: Re-discover the admin pod name if kubectl exec fails, and retry +the command. The E2E framework handles this automatically. + +### 4. Stale `node_restart` tasks blocking `sn restart` + +After the R26 operator installs (Step 6), its tasks-runner may detect offline +nodes and create `node_restart` tasks. When you later run `sn restart` in +Step 10, it may fail with a conflict because the stale task is still +running/pending. + +**Mitigation**: The R26 operator now skips creating restart tasks for nodes that +were already offline. For older builds, use Step 2.1 (disable auto-restart) and +Step 9.1 (cancel stale tasks) before Step 10. + +### 5. StorageNodeSet CR must adopt existing nodes + +The R26 operator's StorageNodeSet reconciler must detect pre-existing storage +nodes from R25 and adopt them (same ports: 8080-8085). If the operator creates +new nodes instead (ports 4420+), the old data is inaccessible. This was a known +operator bug — ensure the operator version includes the StorageNode CR adoption +fix. + +### 6. Preserve resources on failure for debugging + +When an upgrade test fails, avoid cleaning up PVCs, pools, and lvols in the +teardown. Use `--preserve_resources_on_failure true` in the test runner to keep +all K8s resources intact for post-mortem analysis. + +--- + +## Automated Test + +The E2E test `K8sNativeMajorUpgrade` in +[k8s_major_upgrade.py](e2e/e2e_tests/upgrade_tests/k8s_major_upgrade.py) +automates all four phases. It is triggered by the `k8s-native-upgrade.yaml` workflow +with `UPGRADE_TYPE=r25-to-r2x`. diff --git a/e2e/__init__.py b/e2e/__init__.py index d5db4278c6..0acd808dcc 100644 --- a/e2e/__init__.py +++ b/e2e/__init__.py @@ -36,7 +36,9 @@ from e2e_tests.add_node_fio_run import ( TestAddNodesDuringFioRun, - TestAddK8sNodesDuringFioRun + TestAddK8sNodesDuringFioRun, + TestAddNodesDualNodePerHost, + TestAddK8sNodesDualNodePerHost, ) from e2e_tests.k8s_native_add_node import K8sNativeAddNodeTest from e2e_tests.k8s_native_node_migration import K8sNativeNodeMigrationTest @@ -124,6 +126,8 @@ MassCreateDeleteRestart_300x10_K8s, MassCreateDeleteRestart_300x10_6Snap_K8s, MassCreateDeleteRestart_300x10_10Snap_K8s, + MassCreateRapidRestart_6k_3Snap_Docker, + MassCreateRapidRestart_6k_3Snap_K8s, ) from stress_test.device_failure_migration import ( DeviceFailureMigrationNoLoadDocker, @@ -173,8 +177,15 @@ TestLvolSecurityMultiClientConcurrent, ) -from e2e_tests.upgrade_tests.major_upgrade import TestMajorUpgrade, TestMajorUpgradeSingleNode -from e2e_tests.upgrade_tests.k8s_major_upgrade import K8sNativeMajorUpgrade +from e2e_tests.upgrade_tests.major_upgrade import ( + TestMajorUpgrade, + TestMajorUpgradeSingleNode, + TestMajorUpgradeDualNode, +) +from e2e_tests.upgrade_tests.k8s_major_upgrade import ( + K8sNativeMajorUpgrade, + K8sNativeMajorUpgradeDualNode, +) # ── Phase 1 functional E2E tests ───────────────────────────────────── from e2e_tests.test_lvol_basic import TestLvolBasicCRUD @@ -306,6 +317,7 @@ TestSingleNodeOutage, TestSingleNodeFailure, TestAddNodesDuringFioRun, + TestAddNodesDualNodePerHost, TestRestartNodeOnAnotherHost, TestRebootNodeHost, TestMgmtNodeReboot, @@ -323,11 +335,13 @@ TestHASingleNodeOutage, TestSingleNodeResizeLvolCone, TestAddK8sNodesDuringFioRun, + TestAddK8sNodesDualNodePerHost, K8sNativeAddNodeTest, K8sNativeNodeMigrationTest, TestSequentialNodeAdd, TestAddNodeSnapshotCloneOnNewNode, K8sNativeMajorUpgrade, + K8sNativeMajorUpgradeDualNode, # Security E2E tests TestLvolSecurityCombinations, TestLvolDynamicHostManagement, @@ -453,6 +467,8 @@ MassCreateDeleteRestart_300x10_K8s, MassCreateDeleteRestart_300x10_6Snap_K8s, MassCreateDeleteRestart_300x10_10Snap_K8s, + MassCreateRapidRestart_6k_3Snap_Docker, + MassCreateRapidRestart_6k_3Snap_K8s, DeviceFailureMigrationNoLoadDocker, DeviceFailureMigrationUnderLoadDocker, DeviceFailureMigrationPCIeNoLoadDocker, @@ -734,6 +750,8 @@ def get_stress_tests(): MassCreateDeleteRestart_300x10_K8s, MassCreateDeleteRestart_300x10_6Snap_K8s, MassCreateDeleteRestart_300x10_10Snap_K8s, + MassCreateRapidRestart_6k_3Snap_Docker, + MassCreateRapidRestart_6k_3Snap_K8s, DeviceFailureMigrationNoLoadDocker, DeviceFailureMigrationUnderLoadDocker, DeviceFailureMigrationPCIeNoLoadDocker, @@ -793,6 +811,8 @@ def get_monitoring_tests(): MassCreateDeleteRestart_300x10_K8s, MassCreateDeleteRestart_300x10_6Snap_K8s, MassCreateDeleteRestart_300x10_10Snap_K8s, + MassCreateRapidRestart_6k_3Snap_Docker, + MassCreateRapidRestart_6k_3Snap_K8s, DeviceFailureMigrationNoLoadDocker, DeviceFailureMigrationUnderLoadDocker, DeviceFailureMigrationPCIeNoLoadDocker, @@ -850,7 +870,16 @@ def get_backup_tests(): TestBackupInterruptedBackup, TestBackupInterruptedRestore, TestBackupConcurrentIO, - # Backup node-add / node-migration edge cases + ] + + +def get_backup_topology_tests(): + """Backup tests that modify cluster topology (add-node / migration). + + These require extra infrastructure (NEW_NODE_IPS, migrate_to_worker) + and must run in dedicated topology pipelines, not the regular backup suite. + """ + return [ TestBackupAfterNodeAdd, TestBackupWithFioOnNewNode, TestBackupAfterNodeMigration, @@ -878,7 +907,9 @@ def get_upgrade_tests(): tests = [ TestMajorUpgrade, TestMajorUpgradeSingleNode, + TestMajorUpgradeDualNode, K8sNativeMajorUpgrade, + # K8sNativeMajorUpgradeDualNode, # disabled: focus on single-node upgrade first ] return tests diff --git a/e2e/e2e.py b/e2e/e2e.py index 15de64d236..f7a7eb45e8 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -6,7 +6,7 @@ import subprocess import time import traceback -from __init__ import get_all_tests, get_security_tests, get_backup_tests, get_backup_stress_tests, get_parity_tests, ALL_TESTS +from __init__ import get_all_tests, get_security_tests, get_backup_tests, get_backup_topology_tests, get_backup_stress_tests, get_parity_tests, ALL_TESTS from logger_config import setup_logger from exceptions.custom_exception import ( TestNotFoundException, @@ -36,15 +36,18 @@ # starts with a fresh cluster and clean spare nodes. TOPOLOGY_MODIFYING_TESTS = { "TestAddNodesDuringFioRun", + "TestAddNodesDualNodePerHost", "TestSequentialNodeAdd", "TestAddNodeSnapshotCloneOnNewNode", "TestBackupAfterNodeAdd", "TestBackupWithFioOnNewNode", "TestAddK8sNodesDuringFioRun", + "TestAddK8sNodesDualNodePerHost", "K8sNativeAddNodeTest", "K8sNativeNodeMigrationTest", "TestBackupAfterNodeMigration", "TestBackupDuringMigration", + "TestBackupCrossClusterRestore", } def main(): @@ -137,6 +140,8 @@ def main(): test_class_run = get_security_tests() elif args.testname and args.testname.strip().lower() == "backup": test_class_run = get_backup_tests() + elif args.testname and args.testname.strip().lower() == "backup-topology": + test_class_run = get_backup_topology_tests() elif args.testname and args.testname.strip().lower() == "backup-stress": test_class_run = get_backup_stress_tests() elif args.testname and args.testname.strip().lower() == "parity": @@ -148,6 +153,11 @@ def main(): logger.warning("Skipping TestAddNodesDuringFioRun: requires --new-nodes with at least 1 IP.") skipped_cases += 1 continue + if cls.__name__ == "TestAddNodesDualNodePerHost": + if len(new_nodes) == 0: + logger.warning("Skipping TestAddNodesDualNodePerHost: requires --new-nodes with at least 1 IP.") + skipped_cases += 1 + continue if cls.__name__ == "TestRestartNodeOnAnotherHost": if len(new_nodes) == 0: logger.warning("Skipping TestRestartNodeOnAnotherHost: requires --new-nodes with atleast 1 IP.") @@ -160,6 +170,13 @@ def main(): logger.warning("Skipping TestAddK8sNodesDuringFioRun: requires --new-nodes with at least 1 IP.") skipped_cases += 1 continue + if cls.__name__ == "TestAddK8sNodesDualNodePerHost": + if not args.run_k8s: + continue + if len(new_nodes) == 0: + logger.warning("Skipping TestAddK8sNodesDualNodePerHost: requires --new-nodes with at least 1 IP.") + skipped_cases += 1 + continue if cls.__name__ == "K8sNativeAddNodeTest": if not args.run_k8s: continue @@ -204,12 +221,19 @@ def main(): if needle in cls.__name__.lower().replace("_", "") and cls not in seen: if cls.__name__ == "TestAddNodesDuringFioRun" and len(new_nodes) == 0: raise ValueError("TestAddNodesDuringFioRun requires --new-nodes with at least 1 IP.") + if cls.__name__ == "TestAddNodesDualNodePerHost" and len(new_nodes) == 0: + raise ValueError("TestAddNodesDualNodePerHost requires --new-nodes with at least 1 IP.") if cls.__name__ == "TestRestartNodeOnAnotherHost" and len(new_nodes) == 0: raise ValueError("TestRestartNodeOnAnotherHost requires --new-nodes with atleast 1 new IP.") if cls.__name__ == "TestAddK8sNodesDuringFioRun" and len(new_nodes) == 0: if not args.run_k8s: continue raise ValueError("TestAddK8sNodesDuringFioRun requires --new-nodes with at least 1 IP.") + if cls.__name__ == "TestAddK8sNodesDualNodePerHost": + if not args.run_k8s: + continue + if len(new_nodes) == 0: + raise ValueError("TestAddK8sNodesDualNodePerHost requires --new-nodes with at least 1 IP.") if cls.__name__ == "K8sNativeAddNodeTest": if not args.run_k8s: continue diff --git a/e2e/e2e_tests/add_node_fio_run.py b/e2e/e2e_tests/add_node_fio_run.py index a02feaad50..fca7405d83 100644 --- a/e2e/e2e_tests/add_node_fio_run.py +++ b/e2e/e2e_tests/add_node_fio_run.py @@ -318,117 +318,795 @@ def run(self): self.logger.info("TEST CASE PASSED !!!") -class TestAddK8sNodesDuringFioRun(TestClusterBase): +class TestAddNodesDualNodePerHost(TestAddNodesDuringFioRun): + """ + Dual-node-per-host add-node variant: adds new nodes configured with + ``--nodes-per-socket 2``, so each physical host gets 2 logical storage + nodes from a single ``sn add-node`` call. + + Overrides Step 3 of the parent to: + 1. Call ``sn configure --nodes-per-socket 2`` on each new host + 2. Assert that 2 node UUIDs are created per new IP + 3. Wait for all logical nodes (2 per IP) to come online + + Steps 1-2, 4-5, and health checks are inherited — they already iterate + by node_id and work correctly for multiple nodes per host. + """ + def __init__(self, **kwargs): super().__init__(**kwargs) - self.new_nodes = kwargs.get("new_nodes") # List of new worker node IPs - self.k3s_mnode = kwargs.get("k3s_mnode") - self.storage_pool_name = self.pool_name # Taking from base class - self.mount_base = "/mnt/" - self.namespace = kwargs.get("namespace", None) - self.test_name = "add_nodes_during_fio_k8s" - self.logger.info(f"New Nodes to Add: {self.new_nodes}") + self.test_name = "add_nodes_dual_node_per_host" + self.nodes_per_socket = 2 + self.logger.info( + f"Dual-node-per-host add-node mode: each new host will get " + f"{self.nodes_per_socket} logical nodes" + ) + + def run(self): + self.logger.info("Starting Test: Add Nodes (Dual-Node-Per-Host) During FIO Run") + + # Step 1: Create lvols on existing nodes (identical to parent) + fio_threads = [] + lvol_details = {} + self.sbcli_utils.add_storage_pool(self.pool_name) + sleep_n_sec(10) + for i, _ in enumerate(self.storage_nodes): + lvol_name = f"lvl_{generate_random_sequence(4)}{i}" + mount_path = f"{self.mount_base}/{lvol_name}" + log_path = f"{self.log_path}/{lvol_name}.log" + + node_uuid = self.sbcli_utils.get_node_without_lvols() + + self.sbcli_utils.add_lvol(lvol_name, self.pool_name, size="10G", + distr_ndcs=self.ndcs, distr_npcs=self.npcs, + distr_bs=self.bs, distr_chunk_bs=self.chunk_bs, + host_id=node_uuid) + connect_ls = self.sbcli_utils.get_lvol_connect_str(lvol_name=lvol_name) + for connect_str in connect_ls: + self.ssh_obj.exec_command(self.mgmt_nodes[0], connect_str) + + device = self.ssh_obj.get_lvol_vs_device( + node=self.mgmt_nodes[0], + lvol_id=self.sbcli_utils.get_lvol_id(lvol_name) + ) + self.ssh_obj.format_disk(self.mgmt_nodes[0], device) + self.ssh_obj.mount_path(self.mgmt_nodes[0], device, mount_path) + + fio_thread = threading.Thread( + target=self.ssh_obj.run_fio_test, + args=(self.mgmt_nodes[0], None, mount_path, log_path), + kwargs={ + "size": "500M", + "name": f"{lvol_name}_fio", + "rw": "randrw", + "nrfiles": 5, + "iodepth": 1, + "numjobs": 5, + "time_based": True, + "runtime": 600, + }, + ) + fio_thread.start() + fio_threads.append(fio_thread) + + # Snapshot + clone on existing lvols + sleep_n_sec(10) + snapshot_name = f"snap_{lvol_name}" + self.ssh_obj.add_snapshot( + self.mgmt_nodes[0], + self.sbcli_utils.get_lvol_id(lvol_name), + snapshot_name, + ) + snapshot_id = self.ssh_obj.get_snapshot_id( + self.mgmt_nodes[0], snapshot_name=snapshot_name + ) + sleep_n_sec(10) + clone_name = f"clone_{lvol_name}" + self.ssh_obj.add_clone(self.mgmt_nodes[0], snapshot_id, clone_name) + clone_id = self.sbcli_utils.get_lvol_id(clone_name) + + connect_ls = self.sbcli_utils.get_lvol_connect_str(lvol_name=clone_name) + for connect_str in connect_ls: + self.ssh_obj.exec_command(self.mgmt_nodes[0], connect_str) + + cl_mount_path = f"{self.mount_base}/{clone_name}" + cl_log_path = f"{self.log_path}/{clone_name}.log" + + lvol_details[lvol_name] = { + "ID": self.sbcli_utils.get_lvol_id(lvol_name), + "Mount": mount_path, + "Log": log_path, + "Clone": { + "ID": clone_id, + "Snapshot": snapshot_name, + "Log": cl_log_path, + "Mount": cl_mount_path, + }, + } + + device = self.ssh_obj.get_lvol_vs_device( + node=self.mgmt_nodes[0], lvol_id=clone_id + ) + self.ssh_obj.format_disk(self.mgmt_nodes[0], device) + self.ssh_obj.mount_path(self.mgmt_nodes[0], device, cl_mount_path) + + fio_thread = threading.Thread( + target=self.ssh_obj.run_fio_test, + args=(self.mgmt_nodes[0], None, cl_mount_path, cl_log_path), + kwargs={ + "size": "500M", + "name": f"{clone_name}_fio", + "rw": "randrw", + "nrfiles": 5, + "iodepth": 1, + "numjobs": 5, + "time_based": True, + "runtime": 600, + }, + ) + fio_thread.start() + fio_threads.append(fio_thread) + + sleep_n_sec(30) + + sleep_n_sec(30) + + # Step 3: Add new nodes with --nodes-per-socket 2 + self.logger.info( + f"Adding new nodes (nodes_per_socket={self.nodes_per_socket})" + ) + + node_sample = self.sbcli_utils.get_storage_nodes()["results"][0] + max_lvol = node_sample["max_lvol"] + max_prov = int(node_sample["max_prov"] / (1024**3)) + + data_nics = node_sample.get("data_nics", []) + data_nic = data_nics[0]["if_name"] if data_nics else None + self.logger.info(f"Data NIC from existing node: {data_nic}") + + new_nodes_id = [] + timestamp = int(datetime.now().timestamp()) + cluster_details = None + for ip in self.new_nodes: + self.logger.info( + f"Configuring and deploying storage node: {ip} " + f"(nodes_per_socket={self.nodes_per_socket})" + ) + self.ssh_obj.deploy_storage_node( + ip, max_lvol, max_prov, + nodes_per_socket=self.nodes_per_socket, + ) + self.ssh_obj.add_storage_node( + self.mgmt_nodes[0], self.cluster_id, ip, + spdk_image=node_sample["spdk_image"], + partitions=node_sample["num_partitions_per_dev"], + disable_ha_jm=not node_sample["enable_ha_jm"], + enable_test_device=node_sample["enable_test_device"], + spdk_debug=node_sample["spdk_debug"], + data_nic=data_nic, + ) + sleep_n_sec(60) + + # Collect new node IDs — expect nodes_per_socket new UUIDs per IP + new_nodes_before = set(new_nodes_id) + new_nodes_ids_temp = self.sbcli_utils.get_all_node_without_lvols() + new_on_this_ip = [ + nid for nid in new_nodes_ids_temp if nid not in new_nodes_before + ] + self.logger.info( + f"[{ip}] New nodes created: {new_on_this_ip} " + f"(expected {self.nodes_per_socket})" + ) + assert len(new_on_this_ip) == self.nodes_per_socket, ( + f"Expected {self.nodes_per_socket} new nodes on {ip}, " + f"got {len(new_on_this_ip)}: {new_on_this_ip}" + ) + new_nodes_id.extend(new_on_this_ip) + + self.storage_nodes.append(ip) + containers = self.ssh_obj.get_running_containers(node_ip=ip) + self.container_nodes[ip] = containers + + try: + cluster_details = self.sbcli_utils.wait_for_cluster_status( + cluster_id=self.cluster_id, + status="in_expansion", + timeout=60, + ) + except Exception: + self.logger.info( + "Cluster is not in expansion state, Checking if online!!" + ) + + for node in self.storage_nodes: + self.ssh_obj.restart_docker_logging( + node_ip=node, + containers=self.container_nodes[node], + log_dir=os.path.join(self.docker_logs_path, node), + test_name=self.test_name, + ) + + # Step 4: Wait for all new nodes online + sleep_n_sec(60) + self.logger.info( + f"Waiting for {len(new_nodes_id)} new nodes to come online" + ) + + for node in new_nodes_id: + self.sbcli_utils.wait_for_storage_node_status( + node_id=node, status="online", timeout=300 + ) + + cluster_details = self.sbcli_utils.wait_for_cluster_status( + cluster_id=self.cluster_id, status="active", timeout=300 + ) + self.logger.info( + f"Completed cluster expansion for cluster id: {self.cluster_id} " + f"and Cluster status is {cluster_details['status']}" + ) + + sleep_n_sec(120) + + self.validate_migration_for_node(timestamp, 2000, None, 60, no_task_ok=False) + sleep_n_sec(30) + + # Step 5: Create lvols on new nodes and validate + for node in new_nodes_id: + lvol_name = f"lvl_{generate_random_sequence(4)}_nn" + mount_path = f"{self.mount_base}/{lvol_name}" + log_path = f"{self.log_path}/{lvol_name}.log" + + self.sbcli_utils.add_lvol( + lvol_name, self.pool_name, size="10G", + distr_ndcs=self.ndcs, distr_npcs=self.npcs, + distr_bs=self.bs, distr_chunk_bs=self.chunk_bs, + host_id=node, + ) + connect_ls = self.sbcli_utils.get_lvol_connect_str(lvol_name=lvol_name) + for connect_str in connect_ls: + self.ssh_obj.exec_command(self.mgmt_nodes[0], connect_str) + + device = self.ssh_obj.get_lvol_vs_device( + node=self.mgmt_nodes[0], + lvol_id=self.sbcli_utils.get_lvol_id(lvol_name), + ) + self.ssh_obj.format_disk(self.mgmt_nodes[0], device) + self.ssh_obj.mount_path(self.mgmt_nodes[0], device, mount_path) + + fio_thread = threading.Thread( + target=self.ssh_obj.run_fio_test, + args=(self.mgmt_nodes[0], None, mount_path, log_path), + kwargs={ + "size": "500M", + "name": f"{lvol_name}_fio", + "rw": "randrw", + "nrfiles": 5, + "iodepth": 1, + "numjobs": 5, + "time_based": True, + "runtime": 600, + }, + ) + fio_thread.start() + fio_threads.append(fio_thread) + + lvol_details[lvol_name] = { + "ID": self.sbcli_utils.get_lvol_id(lvol_name), + "Mount": mount_path, + "Log": log_path, + "Clone": { + "ID": None, + "Snapshot": None, + "Log": None, + "Mount": None, + }, + } + + sleep_n_sec(10) + + snapshot_name = f"snap_{lvol_name}" + self.ssh_obj.add_snapshot( + self.mgmt_nodes[0], lvol_details[lvol_name]["ID"], snapshot_name + ) + snapshot_id = self.ssh_obj.get_snapshot_id( + self.mgmt_nodes[0], snapshot_name=snapshot_name + ) + sleep_n_sec(10) + + clone_name = f"clone_{lvol_name}" + self.ssh_obj.add_clone(self.mgmt_nodes[0], snapshot_id, clone_name) + clone_id = self.sbcli_utils.get_lvol_id(clone_name) + + connect_ls = self.sbcli_utils.get_lvol_connect_str(lvol_name=clone_name) + for connect_str in connect_ls: + self.ssh_obj.exec_command(self.mgmt_nodes[0], connect_str) + + cl_mount_path = f"{self.mount_base}/{clone_name}" + cl_log_path = f"{self.log_path}/{clone_name}.log" + + lvol_details[lvol_name]["Clone"]["ID"] = clone_id + lvol_details[lvol_name]["Clone"]["Snapshot"] = snapshot_name + lvol_details[lvol_name]["Clone"]["Log"] = cl_log_path + lvol_details[lvol_name]["Clone"]["Mount"] = cl_mount_path + + device = self.ssh_obj.get_lvol_vs_device( + node=self.mgmt_nodes[0], lvol_id=clone_id + ) + self.ssh_obj.format_disk(self.mgmt_nodes[0], device) + self.ssh_obj.mount_path(self.mgmt_nodes[0], device, cl_mount_path) + + fio_thread = threading.Thread( + target=self.ssh_obj.run_fio_test, + args=(self.mgmt_nodes[0], None, cl_mount_path, cl_log_path), + kwargs={ + "size": "500M", + "name": f"{clone_name}_fio", + "rw": "randrw", + "nrfiles": 5, + "iodepth": 1, + "numjobs": 5, + "time_based": True, + "runtime": 600, + }, + ) + fio_thread.start() + fio_threads.append(fio_thread) + + self.common_utils.manage_fio_threads( + node=self.mgmt_nodes[0], threads=fio_threads, timeout=2000 + ) + sleep_n_sec(60) + + for lvol_name, lvol_detail in lvol_details.items(): + self.logger.info(f"Checking fio log for lvol and clone for {lvol_name}") + self.common_utils.validate_fio_test( + node=self.mgmt_nodes[0], log_file=lvol_detail["Log"] + ) + self.common_utils.validate_fio_test( + node=self.mgmt_nodes[0], log_file=lvol_detail["Clone"]["Log"] + ) + + for node in self.sbcli_utils.get_storage_nodes()["results"]: + assert node["status"] == "online", f"{node['id']} is not online" + assert node["health_check"], f"{node['id']} health check failed" + + self.logger.info("TEST CASE PASSED !!!") + + +class TestAddK8sNodesDuringFioRun(TestClusterBase): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.new_nodes = kwargs.get("new_nodes") # List of new worker node IPs + self.k3s_mnode = kwargs.get("k3s_mnode") + self.storage_pool_name = self.pool_name # Taking from base class + self.mount_base = "/mnt/" + self.namespace = kwargs.get("namespace", None) + self.test_name = "add_nodes_during_fio_k8s" + self.logger.info(f"New Nodes to Add: {self.new_nodes}") + + def get_namespace(self): + """Retrieves the namespace using the specified logic.""" + node_ip = self.storage_nodes[0] # Use the first storage node + + # 1. Namespace provided as input + if self.namespace and len(self.namespace) > 0: + self.logger.info(f"Namespace provided as input: {self.namespace}") + return self.namespace + + # 2. Check /var/simplyblock/namespace on an existing node + command = "cat /var/simplyblock/namespace 2>/dev/null" # Suppress errors if file doesn't exist + try: + out, _err = self.ssh_obj.exec_command(node_ip, command) + namespace = out.strip() + if namespace: + self.logger.info(f"Namespace found in /var/simplyblock/namespace: {namespace}") + return namespace + except Exception as e: + self.logger.debug(f"Error reading namespace from file: {e}") + + # 3. Default to 'simplyblk' with a warning + self.logger.warning("Namespace not found in file or input flag. Defaulting to 'simplyblk'") + return "simplyblk" + + def _prepare_worker_node(self, node_ip): + """Prepares a worker node by installing necessary packages and configuring kernel parameters.""" + commands = [ + "sudo rm -f /usr/local/bin/kubectl || true", + "sudo rm -f /usr/bin/kubectl || true", + "sudo yum remove -y kubectl || true", + "sudo yum install -y fio nvme-cli bc", + "sudo modprobe nvme-tcp", + "sudo modprobe nbd", + "total_memory_kb=$(grep MemTotal /proc/meminfo | awk '{print $2}')", + "total_memory_mb=$((total_memory_kb / 1024))", + "hugepages=$(echo \"$total_memory_mb * 0.3 / 1\" | bc)", + "sudo sysctl -w vm.nr_hugepages=$hugepages", + "sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1", + "sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1", + "sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer", + "sudo /usr/local/bin/k3s kubectl get node", + "sudo yum install -y pciutils", + "lspci", + "sudo yum install -y make golang", + "echo 'nvme-tcp' | sudo tee /etc/modules-load.d/nvme-tcp.conf", + "echo 'nbd' | sudo tee /etc/modules-load.d/nbd.conf", + "echo \"vm.nr_hugepages=$hugepages\" | sudo tee /etc/sysctl.d/hugepages.conf", + "sudo sysctl --system" + ] + for command in commands: + self.logger.info(f"Executing command on {node_ip}: {command}") + self.ssh_obj.exec_command(node_ip, command) + + def _add_node_to_cluster(self, node_ip): + """Adds a worker node to the k3s cluster.""" + # 1. Get the token from the k3s master node + token, _ = self.ssh_obj.exec_command(self.k3s_mnode, "sudo cat /var/lib/rancher/k3s/server/node-token") + + token = token.strip() + + # 2. Install k3s and join the cluster + k3s_install_cmd = f"curl -sfL https://get.k3s.io | K3S_URL=https://{self.k3s_mnode}:6443 K3S_TOKEN={token} bash -" + self.logger.info(f"Installing k3s on {node_ip} and joining cluster") + self.ssh_obj.exec_command(node_ip, k3s_install_cmd) + + self.logger.info(f"Waiting for kubectl to be ready on {node_ip}") + sleep_n_sec(30) + + node_name_cmd = "kubectl get nodes -o wide | grep -w %s | awk '{print $1}'" % node_ip + self.logger.info(f"Getting node name to label {node_ip}.") + name, _ = self.ssh_obj.exec_command(self.k3s_mnode, node_name_cmd) + + name = name.strip() + + # 3. Add label to the node + kubectl_label_cmd = f"kubectl label node {name} type=simplyblock-storage-plane" + self.logger.info(f"Adding label to node {node_ip}, name: {name}") + self.ssh_obj.exec_command(self.k3s_mnode, kubectl_label_cmd) + + def _add_node_sbcli(self, node_ip): + """Adds the node to the SimplyBlock cluster using sbcli.""" + node_sample = self.sbcli_utils.get_storage_nodes()["results"][0] + data_nics = node_sample.get("data_nics", []) + data_nic = data_nics[0]["if_name"] if data_nics else None + self.ssh_obj.add_storage_node(self.mgmt_nodes[0], self.cluster_id, node_ip, + spdk_image=node_sample["spdk_image"], + partitions=node_sample["num_partitions_per_dev"], + disable_ha_jm= not node_sample["enable_ha_jm"], + enable_test_device=node_sample["enable_test_device"], + spdk_debug=node_sample["spdk_debug"], + namespace=self.namespace, + data_nic=data_nic) + sleep_n_sec(180) + + def run(self): + self.logger.info("Starting Test: Add Nodes During FIO Run (Kubernetes)") + self.namespace = self.get_namespace() # Getting namespace + self.mgmt_node = self.mgmt_nodes[0] + + # Step 1: Create lvol on existing nodes + fio_threads = [] + lvol_details = {} + self.sbcli_utils.add_storage_pool(self.storage_pool_name) + sleep_n_sec(10) + for i, _ in enumerate(self.storage_nodes): + lvol_name = f"lvl_{generate_random_sequence(4)}{i}" + mount_path = f"{self.mount_base}/{lvol_name}" + log_path = f"{self.log_path}/{lvol_name}.log" + + node_uuid = self.sbcli_utils.get_node_without_lvols() + + self.sbcli_utils.add_lvol(lvol_name, self.storage_pool_name, size="10G", + distr_ndcs=self.ndcs, distr_npcs=self.npcs, + distr_bs=self.bs, distr_chunk_bs=self.chunk_bs, + host_id=node_uuid) + connect_ls = self.sbcli_utils.get_lvol_connect_str(lvol_name=lvol_name) + for connect_str in connect_ls: + self.ssh_obj.exec_command(self.mgmt_nodes[0], connect_str) + + device = self.ssh_obj.get_lvol_vs_device(node=self.mgmt_nodes[0], lvol_id=self.sbcli_utils.get_lvol_id(lvol_name)) + self.ssh_obj.format_disk(self.mgmt_nodes[0], device) + self.ssh_obj.mount_path(self.mgmt_nodes[0], device, mount_path) + + fio_thread = threading.Thread( + target=self.ssh_obj.run_fio_test, + args=(self.mgmt_nodes[0], None, mount_path, log_path), + kwargs={ + "size": "500M", + "name": f"{lvol_name}_fio", + "rw": "randrw", + "nrfiles": 5, + "iodepth": 1, + "numjobs": 5, + "time_based": True, + "runtime": 600, + }, + ) + fio_thread.start() + fio_threads.append(fio_thread) + + lvol_details[lvol_name] = { + "ID": self.sbcli_utils.get_lvol_id(lvol_name), + "Mount": mount_path, + "Log": log_path, + "Clone": { + "ID": None, + "Snapshot": None, + "Log": None, + "Mount": None, + } + } + + sleep_n_sec(10) + + snapshot_name = f"snap_{lvol_name}" + self.ssh_obj.add_snapshot(self.mgmt_nodes[0], lvol_details[lvol_name]["ID"], snapshot_name) + + snapshot_id = self.ssh_obj.get_snapshot_id(self.mgmt_nodes[0], snapshot_name=snapshot_name) + + sleep_n_sec(10) + + clone_name = f"clone_{lvol_name}" + + self.ssh_obj.add_clone(self.mgmt_nodes[0], snapshot_id, clone_name) + + clone_id = self.sbcli_utils.get_lvol_id(clone_name) + + connect_ls = self.sbcli_utils.get_lvol_connect_str(lvol_name=clone_name) + for connect_str in connect_ls: + self.ssh_obj.exec_command(self.mgmt_nodes[0], connect_str) + + cl_mount_path = f"{self.mount_base}/{clone_name}" + cl_log_path = f"{self.log_path}/{clone_name}.log" + + lvol_details[lvol_name]["Clone"]["ID"] = clone_id + lvol_details[lvol_name]["Clone"]["Snapshot"] = snapshot_name + lvol_details[lvol_name]["Clone"]["Log"] = cl_log_path + lvol_details[lvol_name]["Clone"]["Mount"] = cl_mount_path + + device = self.ssh_obj.get_lvol_vs_device(node=self.mgmt_nodes[0], lvol_id=clone_id) + self.ssh_obj.format_disk(self.mgmt_nodes[0], device) + self.ssh_obj.mount_path(self.mgmt_nodes[0], device, cl_mount_path) + + fio_thread = threading.Thread( + target=self.ssh_obj.run_fio_test, + args=(self.mgmt_nodes[0], None, cl_mount_path, cl_log_path), + kwargs={ + "size": "500M", + "name": f"{clone_name}_fio", + "rw": "randrw", + "nrfiles": 5, + "iodepth": 1, + "numjobs": 5, + "time_based": True, + "runtime": 600, + }, + ) + fio_thread.start() + fio_threads.append(fio_thread) + + sleep_n_sec(30) + + + sleep_n_sec(30) + + # Step 2: Add new nodes + self.logger.info("Adding new nodes") + new_nodes_id = [] + timestamp = int(datetime.now().timestamp()) + cluster_details = None + + for ip in self.new_nodes: + self.logger.info(f"Preparing worker node: {ip}") + self._prepare_worker_node(ip) + self.logger.info(f"Adding node {ip} to k3s cluster") + self._add_node_to_cluster(ip) + sleep_n_sec(30) + self.logger.info(f"Adding node {ip} to SimplyBlock cluster") + # self._add_node_sbcli(ip) + sleep_n_sec(180) + new_nodes_ids_temp = self.sbcli_utils.get_all_node_without_lvols() + for node_id in new_nodes_ids_temp: + if node_id not in new_nodes_id: + new_nodes_id.append(node_id) + self.storage_nodes.append(ip) + + try: + cluster_details = self.sbcli_utils.wait_for_cluster_status( + cluster_id=self.cluster_id, + status="in_expansion", + timeout=60 + ) + except Exception: + self.logger.error("Cluster is not in expansion state, Checking if online!!") + + self.runner_k8s_log.restart_logging() + + # Step 3: Resume cluster + sleep_n_sec(300) + for node in new_nodes_id: + self.sbcli_utils.wait_for_storage_node_status( + node_id=node, + status="online", + timeout=300 + ) + + cluster_details = self.sbcli_utils.wait_for_cluster_status( + cluster_id=self.cluster_id, + status="active", + timeout=300 + ) + + # self.logger.info("Expanding the cluster") + # self.ssh_obj.expand_cluster(self.mgmt_nodes[0], cluster_id=self.cluster_id) + + for node in new_nodes_id: + self.sbcli_utils.wait_for_storage_node_status( + node_id=node, + status="online", + timeout=300 + ) + + sleep_n_sec(120) + + self.validate_migration_for_node(timestamp, 2000, None, 60, no_task_ok=False) + sleep_n_sec(30) + + self.logger.info(f"Completed cluster expansion for cluster id: {self.cluster_id} and Cluster status is {cluster_details['status']}") + + # Step 4: Create lvols on new nodes and validate + for node in new_nodes_id: + lvol_name = f"lvl_{generate_random_sequence(4)}_nn" + mount_path = f"{self.mount_base}/{lvol_name}" + log_path = f"{self.log_path}/{lvol_name}.log" + + self.sbcli_utils.add_lvol(lvol_name, self.pool_name, size="10G", + distr_ndcs=self.ndcs, distr_npcs=self.npcs, + distr_bs=self.bs, distr_chunk_bs=self.chunk_bs, + host_id=node) + connect_ls = self.sbcli_utils.get_lvol_connect_str(lvol_name=lvol_name) + for connect_str in connect_ls: + self.ssh_obj.exec_command(self.mgmt_nodes[0], connect_str) + + device = self.ssh_obj.get_lvol_vs_device(node=self.mgmt_nodes[0], lvol_id=self.sbcli_utils.get_lvol_id(lvol_name)) + self.ssh_obj.format_disk(self.mgmt_nodes[0], device) + self.ssh_obj.mount_path(self.mgmt_nodes[0], device, mount_path) + + fio_thread = threading.Thread( + target=self.ssh_obj.run_fio_test, + args=(self.mgmt_nodes[0], None, mount_path, log_path), + kwargs={ + "size": "500M", + "name": f"{lvol_name}_fio", + "rw": "randrw", + "nrfiles": 5, + "iodepth": 1, + "numjobs": 5, + "time_based": True, + "runtime": 600, + }, + ) + fio_thread.start() + fio_threads.append(fio_thread) + + lvol_details[lvol_name] = { + "ID": self.sbcli_utils.get_lvol_id(lvol_name), + "Mount": mount_path, + "Log": log_path, + "Clone": { + "ID": None, + "Snapshot": None, + "Log": None, + "Mount": None, + } + } + + sleep_n_sec(10) + + snapshot_name = f"snap_{lvol_name}" + self.ssh_obj.add_snapshot(self.mgmt_nodes[0], lvol_details[lvol_name]["ID"], snapshot_name) - def get_namespace(self): - """Retrieves the namespace using the specified logic.""" - node_ip = self.storage_nodes[0] # Use the first storage node + snapshot_id = self.ssh_obj.get_snapshot_id(self.mgmt_nodes[0], snapshot_name=snapshot_name) - # 1. Namespace provided as input - if self.namespace and len(self.namespace) > 0: - self.logger.info(f"Namespace provided as input: {self.namespace}") - return self.namespace + sleep_n_sec(10) - # 2. Check /var/simplyblock/namespace on an existing node - command = "cat /var/simplyblock/namespace 2>/dev/null" # Suppress errors if file doesn't exist - try: - out, _err = self.ssh_obj.exec_command(node_ip, command) - namespace = out.strip() - if namespace: - self.logger.info(f"Namespace found in /var/simplyblock/namespace: {namespace}") - return namespace - except Exception as e: - self.logger.debug(f"Error reading namespace from file: {e}") + clone_name = f"clone_{lvol_name}" - # 3. Default to 'simplyblk' with a warning - self.logger.warning("Namespace not found in file or input flag. Defaulting to 'simplyblk'") - return "simplyblk" + self.ssh_obj.add_clone(self.mgmt_nodes[0], snapshot_id, clone_name) - def _prepare_worker_node(self, node_ip): - """Prepares a worker node by installing necessary packages and configuring kernel parameters.""" - commands = [ - "sudo rm -f /usr/local/bin/kubectl || true", - "sudo rm -f /usr/bin/kubectl || true", - "sudo yum remove -y kubectl || true", - "sudo yum install -y fio nvme-cli bc", - "sudo modprobe nvme-tcp", - "sudo modprobe nbd", - "total_memory_kb=$(grep MemTotal /proc/meminfo | awk '{print $2}')", - "total_memory_mb=$((total_memory_kb / 1024))", - "hugepages=$(echo \"$total_memory_mb * 0.3 / 1\" | bc)", - "sudo sysctl -w vm.nr_hugepages=$hugepages", - "sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1", - "sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1", - "sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer", - "sudo /usr/local/bin/k3s kubectl get node", - "sudo yum install -y pciutils", - "lspci", - "sudo yum install -y make golang", - "echo 'nvme-tcp' | sudo tee /etc/modules-load.d/nvme-tcp.conf", - "echo 'nbd' | sudo tee /etc/modules-load.d/nbd.conf", - "echo \"vm.nr_hugepages=$hugepages\" | sudo tee /etc/sysctl.d/hugepages.conf", - "sudo sysctl --system" - ] - for command in commands: - self.logger.info(f"Executing command on {node_ip}: {command}") - self.ssh_obj.exec_command(node_ip, command) + clone_id = self.sbcli_utils.get_lvol_id(clone_name) - def _add_node_to_cluster(self, node_ip): - """Adds a worker node to the k3s cluster.""" - # 1. Get the token from the k3s master node - token, _ = self.ssh_obj.exec_command(self.k3s_mnode, "sudo cat /var/lib/rancher/k3s/server/node-token") + connect_ls = self.sbcli_utils.get_lvol_connect_str(lvol_name=clone_name) + for connect_str in connect_ls: + self.ssh_obj.exec_command(self.mgmt_nodes[0], connect_str) - token = token.strip() + cl_mount_path = f"{self.mount_base}/{clone_name}" + cl_log_path = f"{self.log_path}/{clone_name}.log" - # 2. Install k3s and join the cluster - k3s_install_cmd = f"curl -sfL https://get.k3s.io | K3S_URL=https://{self.k3s_mnode}:6443 K3S_TOKEN={token} bash -" - self.logger.info(f"Installing k3s on {node_ip} and joining cluster") - self.ssh_obj.exec_command(node_ip, k3s_install_cmd) + lvol_details[lvol_name]["Clone"]["ID"] = clone_id + lvol_details[lvol_name]["Clone"]["Snapshot"] = snapshot_name + lvol_details[lvol_name]["Clone"]["Log"] = cl_log_path + lvol_details[lvol_name]["Clone"]["Mount"] = cl_mount_path - self.logger.info(f"Waiting for kubectl to be ready on {node_ip}") - sleep_n_sec(30) + device = self.ssh_obj.get_lvol_vs_device(node=self.mgmt_nodes[0], lvol_id=clone_id) + self.ssh_obj.format_disk(self.mgmt_nodes[0], device) + self.ssh_obj.mount_path(self.mgmt_nodes[0], device, cl_mount_path) - node_name_cmd = "kubectl get nodes -o wide | grep -w %s | awk '{print $1}'" % node_ip - self.logger.info(f"Getting node name to label {node_ip}.") - name, _ = self.ssh_obj.exec_command(self.k3s_mnode, node_name_cmd) + fio_thread = threading.Thread( + target=self.ssh_obj.run_fio_test, + args=(self.mgmt_nodes[0], None, cl_mount_path, cl_log_path), + kwargs={ + "size": "500M", + "name": f"{clone_name}_fio", + "rw": "randrw", + "nrfiles": 5, + "iodepth": 1, + "numjobs": 5, + "time_based": True, + "runtime": 600, + }, + ) + fio_thread.start() + fio_threads.append(fio_thread) - name = name.strip() - - # 3. Add label to the node - kubectl_label_cmd = f"kubectl label node {name} type=simplyblock-storage-plane" - self.logger.info(f"Adding label to node {node_ip}, name: {name}") - self.ssh_obj.exec_command(self.k3s_mnode, kubectl_label_cmd) + self.common_utils.manage_fio_threads( + node=self.mgmt_nodes[0], + threads=fio_threads, + timeout=2000 + ) + sleep_n_sec(60) - def _add_node_sbcli(self, node_ip): - """Adds the node to the SimplyBlock cluster using sbcli.""" - node_sample = self.sbcli_utils.get_storage_nodes()["results"][0] - data_nics = node_sample.get("data_nics", []) - data_nic = data_nics[0]["if_name"] if data_nics else None - self.ssh_obj.add_storage_node(self.mgmt_nodes[0], self.cluster_id, node_ip, - spdk_image=node_sample["spdk_image"], - partitions=node_sample["num_partitions_per_dev"], - disable_ha_jm= not node_sample["enable_ha_jm"], - enable_test_device=node_sample["enable_test_device"], - spdk_debug=node_sample["spdk_debug"], - namespace=self.namespace, - data_nic=data_nic) - sleep_n_sec(180) + + for lvol_name, lvol_detail in lvol_details.items(): + self.logger.info(f"Checking fio log for lvol and clone for {lvol_name}") + self.common_utils.validate_fio_test(node=self.mgmt_nodes[0], log_file=lvol_detail["Log"]) + self.common_utils.validate_fio_test(node=self.mgmt_nodes[0], log_file=lvol_detail["Clone"]["Log"]) + + for node in self.sbcli_utils.get_storage_nodes()["results"]: + assert node["status"] == "online", f"{node['id']} is not online" + assert node["health_check"], f"{node['id']} health check failed" + + self.logger.info("TEST CASE PASSED !!!") + + +class TestAddK8sNodesDualNodePerHost(TestAddK8sNodesDuringFioRun): + """K8s add-node test for dual-node-per-host (nodesPerSocket=2). + + Before adding new workers, the StorageNodeSet is patched with + nodesPerSocket=2 so the operator provisions 2 logical storage + nodes on each new worker. + + Dispatch with: + TEST_CLASS: "TestAddK8sNodesDualNodePerHost" + NEW_NODE_IPS: "IP1 IP2" (spare hosts to add) + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.test_name = "add_k8s_nodes_dual_node_per_host" + self.nodes_per_socket = 2 + + def _patch_nodes_per_socket(self): + """Patch the StorageNodeSet to set nodesPerSocket.""" + patch_cmd = ( + "kubectl patch storagenodesets.storage.simplyblock.io " + f"simplyblock-node -n {self.namespace} --type=merge " + f"-p '{{\"spec\":{{\"nodesPerSocket\":{self.nodes_per_socket}}}}}'" + ) + self.logger.info( + f"Patching StorageNodeSet with nodesPerSocket=" + f"{self.nodes_per_socket}" + ) + self.ssh_obj.exec_command(self.k3s_mnode, patch_cmd) def run(self): - self.logger.info("Starting Test: Add Nodes During FIO Run (Kubernetes)") - self.namespace = self.get_namespace() # Getting namespace + self.logger.info( + "Starting Test: Add K8s Nodes During FIO Run " + f"(Dual-Node, nodesPerSocket={self.nodes_per_socket})" + ) + self.namespace = self.get_namespace() self.mgmt_node = self.mgmt_nodes[0] - # Step 1: Create lvol on existing nodes + # Step 1: Create lvols on existing nodes with FIO + snapshots/clones fio_threads = [] lvol_details = {} self.sbcli_utils.add_storage_pool(self.storage_pool_name) @@ -440,15 +1118,22 @@ def run(self): node_uuid = self.sbcli_utils.get_node_without_lvols() - self.sbcli_utils.add_lvol(lvol_name, self.storage_pool_name, size="10G", - distr_ndcs=self.ndcs, distr_npcs=self.npcs, - distr_bs=self.bs, distr_chunk_bs=self.chunk_bs, - host_id=node_uuid) - connect_ls = self.sbcli_utils.get_lvol_connect_str(lvol_name=lvol_name) + self.sbcli_utils.add_lvol( + lvol_name, self.storage_pool_name, size="10G", + distr_ndcs=self.ndcs, distr_npcs=self.npcs, + distr_bs=self.bs, distr_chunk_bs=self.chunk_bs, + host_id=node_uuid, + ) + connect_ls = self.sbcli_utils.get_lvol_connect_str( + lvol_name=lvol_name + ) for connect_str in connect_ls: self.ssh_obj.exec_command(self.mgmt_nodes[0], connect_str) - device = self.ssh_obj.get_lvol_vs_device(node=self.mgmt_nodes[0], lvol_id=self.sbcli_utils.get_lvol_id(lvol_name)) + device = self.ssh_obj.get_lvol_vs_device( + node=self.mgmt_nodes[0], + lvol_id=self.sbcli_utils.get_lvol_id(lvol_name), + ) self.ssh_obj.format_disk(self.mgmt_nodes[0], device) self.ssh_obj.mount_path(self.mgmt_nodes[0], device, mount_path) @@ -468,7 +1153,7 @@ def run(self): ) fio_thread.start() fio_threads.append(fio_thread) - + lvol_details[lvol_name] = { "ID": self.sbcli_utils.get_lvol_id(lvol_name), "Mount": mount_path, @@ -478,25 +1163,34 @@ def run(self): "Snapshot": None, "Log": None, "Mount": None, - } + }, } sleep_n_sec(10) - + snapshot_name = f"snap_{lvol_name}" - self.ssh_obj.add_snapshot(self.mgmt_nodes[0], lvol_details[lvol_name]["ID"], snapshot_name) + self.ssh_obj.add_snapshot( + self.mgmt_nodes[0], + lvol_details[lvol_name]["ID"], + snapshot_name, + ) - snapshot_id = self.ssh_obj.get_snapshot_id(self.mgmt_nodes[0], snapshot_name=snapshot_name) + snapshot_id = self.ssh_obj.get_snapshot_id( + self.mgmt_nodes[0], snapshot_name=snapshot_name + ) sleep_n_sec(10) clone_name = f"clone_{lvol_name}" - - self.ssh_obj.add_clone(self.mgmt_nodes[0], snapshot_id, clone_name) + self.ssh_obj.add_clone( + self.mgmt_nodes[0], snapshot_id, clone_name + ) clone_id = self.sbcli_utils.get_lvol_id(clone_name) - connect_ls = self.sbcli_utils.get_lvol_connect_str(lvol_name=clone_name) + connect_ls = self.sbcli_utils.get_lvol_connect_str( + lvol_name=clone_name + ) for connect_str in connect_ls: self.ssh_obj.exec_command(self.mgmt_nodes[0], connect_str) @@ -508,9 +1202,13 @@ def run(self): lvol_details[lvol_name]["Clone"]["Log"] = cl_log_path lvol_details[lvol_name]["Clone"]["Mount"] = cl_mount_path - device = self.ssh_obj.get_lvol_vs_device(node=self.mgmt_nodes[0], lvol_id=clone_id) + device = self.ssh_obj.get_lvol_vs_device( + node=self.mgmt_nodes[0], lvol_id=clone_id + ) self.ssh_obj.format_disk(self.mgmt_nodes[0], device) - self.ssh_obj.mount_path(self.mgmt_nodes[0], device, cl_mount_path) + self.ssh_obj.mount_path( + self.mgmt_nodes[0], device, cl_mount_path + ) fio_thread = threading.Thread( target=self.ssh_obj.run_fio_test, @@ -531,15 +1229,25 @@ def run(self): sleep_n_sec(30) - sleep_n_sec(30) - # Step 2: Add new nodes - self.logger.info("Adding new nodes") + # Step 2: Patch StorageNodeSet for dual-node, then add new workers + self.logger.info( + "Patching StorageNodeSet and adding new nodes " + f"(nodesPerSocket={self.nodes_per_socket})" + ) + self._patch_nodes_per_socket() + new_nodes_id = [] timestamp = int(datetime.now().timestamp()) cluster_details = None + # Track existing nodes before expansion + existing_node_ids = set( + n["id"] + for n in self.sbcli_utils.get_storage_nodes()["results"] + ) + for ip in self.new_nodes: self.logger.info(f"Preparing worker node: {ip}") self._prepare_worker_node(ip) @@ -547,56 +1255,91 @@ def run(self): self._add_node_to_cluster(ip) sleep_n_sec(30) self.logger.info(f"Adding node {ip} to SimplyBlock cluster") - # self._add_node_sbcli(ip) sleep_n_sec(180) - new_nodes_ids_temp = self.sbcli_utils.get_all_node_without_lvols() - for node_id in new_nodes_ids_temp: - if node_id not in new_nodes_id: - new_nodes_id.append(node_id) + + # Detect new nodes created on this worker + all_nodes_now = self.sbcli_utils.get_storage_nodes()["results"] + new_on_this_ip = [ + n["id"] for n in all_nodes_now + if n["id"] not in existing_node_ids + and n["id"] not in new_nodes_id + ] + self.logger.info( + f"New nodes on {ip}: {new_on_this_ip} " + f"(expected {self.nodes_per_socket})" + ) + + # If not all nodes created yet, wait longer + if len(new_on_this_ip) < self.nodes_per_socket: + self.logger.info( + f"Waiting for remaining nodes on {ip} " + f"(got {len(new_on_this_ip)}, " + f"need {self.nodes_per_socket})" + ) + sleep_n_sec(120) + all_nodes_now = self.sbcli_utils.get_storage_nodes()[ + "results" + ] + new_on_this_ip = [ + n["id"] for n in all_nodes_now + if n["id"] not in existing_node_ids + and n["id"] not in new_nodes_id + ] + + assert len(new_on_this_ip) >= self.nodes_per_socket, ( + f"Expected {self.nodes_per_socket} new nodes on {ip}, " + f"got {len(new_on_this_ip)}: {new_on_this_ip}" + ) + new_nodes_id.extend(new_on_this_ip) self.storage_nodes.append(ip) - + try: cluster_details = self.sbcli_utils.wait_for_cluster_status( cluster_id=self.cluster_id, status="in_expansion", - timeout=60 + timeout=60, ) except Exception: - self.logger.error("Cluster is not in expansion state, Checking if online!!") + self.logger.error( + "Cluster is not in expansion state, " + "Checking if online!!" + ) self.runner_k8s_log.restart_logging() - # Step 3: Resume cluster + self.logger.info( + f"Total new nodes: {len(new_nodes_id)} " + f"(expected {self.nodes_per_socket * len(self.new_nodes)})" + ) + + # Step 3: Wait for all new nodes online sleep_n_sec(300) for node in new_nodes_id: self.sbcli_utils.wait_for_storage_node_status( - node_id=node, - status="online", - timeout=300 + node_id=node, status="online", timeout=300, ) - + cluster_details = self.sbcli_utils.wait_for_cluster_status( - cluster_id=self.cluster_id, - status="active", - timeout=300 + cluster_id=self.cluster_id, status="active", timeout=300, ) - - # self.logger.info("Expanding the cluster") - # self.ssh_obj.expand_cluster(self.mgmt_nodes[0], cluster_id=self.cluster_id) for node in new_nodes_id: self.sbcli_utils.wait_for_storage_node_status( - node_id=node, - status="online", - timeout=300 + node_id=node, status="online", timeout=300, ) sleep_n_sec(120) - self.validate_migration_for_node(timestamp, 2000, None, 60, no_task_ok=False) + self.validate_migration_for_node( + timestamp, 2000, None, 60, no_task_ok=False + ) sleep_n_sec(30) - self.logger.info(f"Completed cluster expansion for cluster id: {self.cluster_id} and Cluster status is {cluster_details['status']}") + self.logger.info( + f"Completed cluster expansion for cluster id: " + f"{self.cluster_id} and Cluster status is " + f"{cluster_details['status']}" + ) # Step 4: Create lvols on new nodes and validate for node in new_nodes_id: @@ -604,15 +1347,22 @@ def run(self): mount_path = f"{self.mount_base}/{lvol_name}" log_path = f"{self.log_path}/{lvol_name}.log" - self.sbcli_utils.add_lvol(lvol_name, self.pool_name, size="10G", - distr_ndcs=self.ndcs, distr_npcs=self.npcs, - distr_bs=self.bs, distr_chunk_bs=self.chunk_bs, - host_id=node) - connect_ls = self.sbcli_utils.get_lvol_connect_str(lvol_name=lvol_name) + self.sbcli_utils.add_lvol( + lvol_name, self.pool_name, size="10G", + distr_ndcs=self.ndcs, distr_npcs=self.npcs, + distr_bs=self.bs, distr_chunk_bs=self.chunk_bs, + host_id=node, + ) + connect_ls = self.sbcli_utils.get_lvol_connect_str( + lvol_name=lvol_name + ) for connect_str in connect_ls: self.ssh_obj.exec_command(self.mgmt_nodes[0], connect_str) - device = self.ssh_obj.get_lvol_vs_device(node=self.mgmt_nodes[0], lvol_id=self.sbcli_utils.get_lvol_id(lvol_name)) + device = self.ssh_obj.get_lvol_vs_device( + node=self.mgmt_nodes[0], + lvol_id=self.sbcli_utils.get_lvol_id(lvol_name), + ) self.ssh_obj.format_disk(self.mgmt_nodes[0], device) self.ssh_obj.mount_path(self.mgmt_nodes[0], device, mount_path) @@ -632,7 +1382,7 @@ def run(self): ) fio_thread.start() fio_threads.append(fio_thread) - + lvol_details[lvol_name] = { "ID": self.sbcli_utils.get_lvol_id(lvol_name), "Mount": mount_path, @@ -642,25 +1392,34 @@ def run(self): "Snapshot": None, "Log": None, "Mount": None, - } + }, } sleep_n_sec(10) - + snapshot_name = f"snap_{lvol_name}" - self.ssh_obj.add_snapshot(self.mgmt_nodes[0], lvol_details[lvol_name]["ID"], snapshot_name) + self.ssh_obj.add_snapshot( + self.mgmt_nodes[0], + lvol_details[lvol_name]["ID"], + snapshot_name, + ) - snapshot_id = self.ssh_obj.get_snapshot_id(self.mgmt_nodes[0], snapshot_name=snapshot_name) + snapshot_id = self.ssh_obj.get_snapshot_id( + self.mgmt_nodes[0], snapshot_name=snapshot_name + ) sleep_n_sec(10) clone_name = f"clone_{lvol_name}" - - self.ssh_obj.add_clone(self.mgmt_nodes[0], snapshot_id, clone_name) + self.ssh_obj.add_clone( + self.mgmt_nodes[0], snapshot_id, clone_name + ) clone_id = self.sbcli_utils.get_lvol_id(clone_name) - connect_ls = self.sbcli_utils.get_lvol_connect_str(lvol_name=clone_name) + connect_ls = self.sbcli_utils.get_lvol_connect_str( + lvol_name=clone_name + ) for connect_str in connect_ls: self.ssh_obj.exec_command(self.mgmt_nodes[0], connect_str) @@ -672,9 +1431,13 @@ def run(self): lvol_details[lvol_name]["Clone"]["Log"] = cl_log_path lvol_details[lvol_name]["Clone"]["Mount"] = cl_mount_path - device = self.ssh_obj.get_lvol_vs_device(node=self.mgmt_nodes[0], lvol_id=clone_id) + device = self.ssh_obj.get_lvol_vs_device( + node=self.mgmt_nodes[0], lvol_id=clone_id + ) self.ssh_obj.format_disk(self.mgmt_nodes[0], device) - self.ssh_obj.mount_path(self.mgmt_nodes[0], device, cl_mount_path) + self.ssh_obj.mount_path( + self.mgmt_nodes[0], device, cl_mount_path + ) fio_thread = threading.Thread( target=self.ssh_obj.run_fio_test, @@ -696,18 +1459,28 @@ def run(self): self.common_utils.manage_fio_threads( node=self.mgmt_nodes[0], threads=fio_threads, - timeout=2000 + timeout=2000, ) sleep_n_sec(60) - for lvol_name, lvol_detail in lvol_details.items(): - self.logger.info(f"Checking fio log for lvol and clone for {lvol_name}") - self.common_utils.validate_fio_test(node=self.mgmt_nodes[0], log_file=lvol_detail["Log"]) - self.common_utils.validate_fio_test(node=self.mgmt_nodes[0], log_file=lvol_detail["Clone"]["Log"]) + self.logger.info( + f"Checking fio log for lvol and clone for {lvol_name}" + ) + self.common_utils.validate_fio_test( + node=self.mgmt_nodes[0], log_file=lvol_detail["Log"] + ) + self.common_utils.validate_fio_test( + node=self.mgmt_nodes[0], + log_file=lvol_detail["Clone"]["Log"], + ) for node in self.sbcli_utils.get_storage_nodes()["results"]: - assert node["status"] == "online", f"{node['id']} is not online" - assert node["health_check"], f"{node['id']} health check failed" + assert node["status"] == "online", ( + f"{node['id']} is not online" + ) + assert node["health_check"], ( + f"{node['id']} health check failed" + ) self.logger.info("TEST CASE PASSED !!!") \ No newline at end of file diff --git a/e2e/e2e_tests/backup/test_backup_restore.py b/e2e/e2e_tests/backup/test_backup_restore.py index 35fd3dc81a..e32ec7e98a 100644 --- a/e2e/e2e_tests/backup/test_backup_restore.py +++ b/e2e/e2e_tests/backup/test_backup_restore.py @@ -1864,8 +1864,8 @@ def run(self): except Exception: pass - # --- TC-BCK-018: Delete lvol while backup is in-progress; backup must still complete --- - self.logger.info("TC-BCK-018: delete lvol before backup completes, expect backup to finish and restore to work") + # --- TC-BCK-018: Backup then delete source lvol; restore must work --- + self.logger.info("TC-BCK-018: backup lvol, wait for completion, delete source, then restore and verify") tc18_lvol_name, tc18_lvol_id = self._create_lvol() _, tc18_mount = self._connect_and_mount(tc18_lvol_name, tc18_lvol_id) self._run_fio(tc18_mount, runtime=30) @@ -1875,28 +1875,18 @@ def run(self): tc18_snap_name = f"tc18_snap_{_rand_suffix()}" tc18_snap_id = self._create_snapshot(tc18_lvol_id, tc18_snap_name, backup=True) - self.logger.info(f"TC-BCK-018: snapshot {tc18_snap_id} + backup triggered — deleting lvol after backup source resolved") + self.logger.info(f"TC-BCK-018: snapshot {tc18_snap_id} + backup triggered") - # Delete lvol before backup completes (backup reads from snapshot, not live lvol). - # In K8s mode we must wait for the StorageBackup to leave Pending phase, - # otherwise the backup controller can't resolve the source PVC. + # Wait for backup to complete before deleting the source. + # In K8s mode the operator re-resolves PVC references during reconciliation, + # so deleting the PVC mid-backup causes BackupSourceResolutionError. + tc18_bk_id = self._wait_for_backup_by_snap(tc18_snap_name, "TC-BCK-018") + self.logger.info(f"TC-BCK-018: backup {tc18_bk_id} completed") + + # Now delete the source lvol/PVC if self.k8s_test: k8s = self._ensure_k8s_utils() pvc_name = self._k8s_normalize_name(tc18_lvol_name) - # tc18_snap_id is the StorageBackup CRD name (bck-tc18-snap-xxx) - bck_name = tc18_snap_id - # Wait up to 120s for backup to move past Pending - for _ in range(24): - try: - res = k8s.get_resource_json("storagebackup", bck_name) - phase = (res.get("status", {}).get("phase") or "").lower() - if phase and phase != "pending": - self.logger.info( - f"TC-BCK-018: StorageBackup {bck_name} reached phase={phase}, safe to delete PVC") - break - except Exception: - pass - sleep_n_sec(5) k8s.delete_pvc(pvc_name) if pvc_name in self.created_pvcs: self.created_pvcs.remove(pvc_name) @@ -1913,11 +1903,7 @@ def run(self): self.sbcli_utils.delete_lvol(lvol_name=tc18_lvol_name, skip_error=True) if tc18_lvol_name in self.created_lvols: self.created_lvols.remove(tc18_lvol_name) - self.logger.info("TC-BCK-018: lvol deleted; waiting for backup to complete") - - # Backup should still complete because it reads from snapshot, not the live lvol - tc18_bk_id = self._wait_for_backup_by_snap(tc18_snap_name, "TC-BCK-018") - self.logger.info(f"TC-BCK-018: backup {tc18_bk_id} completed despite lvol deletion ✓") + self.logger.info("TC-BCK-018: source lvol deleted after backup completed") # Restore and verify checksums tc18_restored_name = f"tc18_restored_{_rand_suffix()}" @@ -1929,7 +1915,7 @@ def run(self): mount=f"{self.mount_path}/tc18_{_rand_suffix()}", format_disk=False) self._verify_checksums(self.fio_node, tc18_r_mount, tc18_checksums) - self.logger.info("TC-BCK-018: checksums match after restore from in-progress backup ✓") + self.logger.info("TC-BCK-018: checksums match after restore from backup of deleted source ✓") self.logger.info("=== TestBackupRestoreDataIntegrity PASSED ===") @@ -2979,9 +2965,21 @@ class TestBackupCrossClusterRestore(BackupTestBase): Environment variables --------------------- - CLUSTER2_ID UUID of the destination cluster - CLUSTER2_SECRET API secret for the destination cluster - CLUSTER2_API_BASE_URL REST API URL for the destination cluster + CLUSTER2_ID UUID of the destination cluster (optional) + CLUSTER2_SECRET API secret for the destination cluster (optional) + CLUSTER2_API_BASE_URL REST API URL for the destination cluster (optional) + STORAGE_PRIVATE_IPS Storage node IPs in Cluster-1 + NEW_NODE_IPS Spare node IPs for auto-bootstrap of Cluster-2 + + If CLUSTER2_* env vars are NOT set, the test auto-bootstraps a second + cluster. It first looks for spare nodes (IPs in NEW_NODE_IPS or + STORAGE_PRIVATE_IPS that are not already in Cluster-1). If no spare + nodes exist, it splits the total pool in half (min 2 per cluster). + + CI dispatch example (Docker, e2e-bootstrap.yml): + STORAGE_PRIVATE_IPS: "IP1 IP2" # cluster 1 + NEW_NODE_IPS: "IP3 IP4" # spare → cluster 2 + TEST_CLASS: "TestBackupCrossClusterRestore" Covers ------ @@ -2996,35 +2994,414 @@ class TestBackupCrossClusterRestore(BackupTestBase): TC-BCK-076b Cluster-2: `backup source-switch local` restores own source """ + # Minimum storage nodes per cluster for cross-cluster restore + _MIN_NODES_PER_CLUSTER = 2 + def __init__(self, **kwargs): super().__init__(**kwargs) self.test_name = "backup_cross_cluster_restore" self._cluster2_id = os.environ.get("CLUSTER2_ID", "") self._cluster2_secret = os.environ.get("CLUSTER2_SECRET", "") self._cluster2_api_url = os.environ.get("CLUSTER2_API_BASE_URL", "") + self._cluster2_namespace = os.environ.get("CLUSTER2_NAMESPACE", "simplyblock-c2") self._meta_file = "/tmp/cross_cluster_backup_meta.json" # Resources created on Cluster-2 (separate tracking for teardown) self._c2_lvols: list[str] = [] + # Whether we bootstrapped cluster 2 ourselves (for teardown) + self._self_bootstrapped_c2 = False + # K8s-mode: second K8sUtils instance for Cluster-2 (initialised in _check_prerequisites) + self._k8s_c2 = None # ── prerequisite check ──────────────────────────────────────────────────── def _check_prerequisites(self): - missing = [ - v for v, val in [ - ("CLUSTER2_ID", self._cluster2_id), - ("CLUSTER2_SECRET", self._cluster2_secret), - ("CLUSTER2_API_BASE_URL", self._cluster2_api_url), - ] if not val - ] - if missing: + """Ensure Cluster-2 credentials are available. + + K8s mode: + Cluster-2 is pre-deployed by the pipeline in namespace + ``CLUSTER2_NAMESPACE`` (default ``simplyblock-c2``). Credentials + are extracted from the admin pod in that namespace. + + Docker mode: + If CLUSTER2_* env vars are not set, attempt to bootstrap a second + cluster by splitting the available storage nodes in half. + """ + if self._cluster2_id and self._cluster2_secret and self._cluster2_api_url: + if self.k8s_test: + self._init_k8s_c2() + return # env vars already set + + if self.k8s_test: + # In K8s mode, discover Cluster-2 from its namespace admin pod + self._init_k8s_c2() + self._discover_k8s_cluster2() + return + + self.logger.info( + "TC-BCK-070: CLUSTER2_* env vars not set — " + "attempting to bootstrap a second cluster from available nodes") + self._bootstrap_second_cluster() + + # ── K8s mode: Cluster-2 namespace discovery ───────────────────────────── + + def _init_k8s_c2(self): + """Initialise a K8sUtils instance pointing at the Cluster-2 namespace.""" + if self._k8s_c2 is not None: + return + from utils.k8s_utils import K8sUtils + mgmt_node = self.mgmt_nodes[0] + self._k8s_c2 = K8sUtils( + ssh_obj=self.ssh_obj, + mgmt_node=mgmt_node, + namespace=self._cluster2_namespace, + ) + self.logger.info( + f"[K8s] Cluster-2 K8sUtils initialised for namespace " + f"'{self._cluster2_namespace}' on {mgmt_node}") + + def _discover_k8s_cluster2(self): + """Extract Cluster-2 ID and secret from the admin pod in C2 namespace.""" + self.logger.info( + f"TC-BCK-070: discovering Cluster-2 from namespace " + f"'{self._cluster2_namespace}'") + # cluster list via C2 admin pod + out, err = self._k8s_c2.exec_sbcli(f"{self.base_cmd} cluster list") + if not out or "error" in (err or "").lower(): + raise RuntimeError( + f"TC-BCK-070: cannot list clusters in namespace " + f"'{self._cluster2_namespace}': {err}") + + # Extract the cluster UUID (should be the only cluster in this namespace) + import re + uuid_re = re.compile( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + re.IGNORECASE, + ) + for line in out.split("\n"): + m = uuid_re.search(line) + if m: + self._cluster2_id = m.group(0) + break + if not self._cluster2_id: + raise RuntimeError( + f"TC-BCK-070: no cluster UUID found in namespace " + f"'{self._cluster2_namespace}' output: {out[:300]}") + + # Get secret + secret_out, _ = self._k8s_c2.exec_sbcli( + f"{self.base_cmd} cluster get-secret {self._cluster2_id}") + self._cluster2_secret = (secret_out or "").strip().split("\n")[-1].strip() + + # API URL: same mgmt node, both clusters share the management API + self._cluster2_api_url = self.api_base_url + + self.logger.info( + f"TC-BCK-070: Cluster-2 discovered — ID={self._cluster2_id}, " + f"namespace={self._cluster2_namespace}") + + # ── self-bootstrap second cluster ──────────────────────────────────────── + + def _bootstrap_second_cluster(self): + """Create a second cluster on the same mgmt node using spare storage nodes. + + Determines which IPs from ``STORAGE_PRIVATE_IPS`` are NOT already in + Cluster-1, and uses those for Cluster-2. If all IPs are already in + Cluster-1, falls back to splitting: removes the second half from + Cluster-1 and uses them for Cluster-2. + + Requires: + - ``STORAGE_PRIVATE_IPS`` env var listing *all* storage node IPs + - At least ``_MIN_NODES_PER_CLUSTER * 2`` total IPs + """ + # Collect all known storage node IPs from env vars + storage_ips_raw = os.environ.get("STORAGE_PRIVATE_IPS", "") + new_node_ips_raw = os.environ.get("NEW_NODE_IPS", "") + all_ips = [] + seen = set() + for ip in (storage_ips_raw + " " + new_node_ips_raw).split(): + ip = ip.strip() + if ip and ip not in seen: + all_ips.append(ip) + seen.add(ip) + + if not all_ips: raise EnvironmentError( - f"TC-BCK-070: cross-cluster restore requires env vars: " - f"{', '.join(missing)}") + "TC-BCK-070: STORAGE_PRIVATE_IPS (and/or NEW_NODE_IPS) env var " + "required to auto-bootstrap a second cluster") + + # Determine which IPs are already in Cluster-1 + c1_ips = set(self.storage_nodes or []) + spare_ips = [ip for ip in all_ips if ip not in c1_ips] + + total = len(all_ips) + if len(spare_ips) >= self._MIN_NODES_PER_CLUSTER: + # Spare nodes available (e.g. from NEW_NODE_IPS) — use them directly + c2_ips = spare_ips + self.logger.info( + f"TC-BCK-070: using {len(c2_ips)} spare node(s) for Cluster-2: " + f"{c2_ips} (Cluster-1 has: {sorted(c1_ips)})") + elif total >= self._MIN_NODES_PER_CLUSTER * 2: + # All nodes are in Cluster-1 — split in half + split = total // 2 + c1_keep = all_ips[:split] + c2_ips = all_ips[split:] + if len(c1_keep) < self._MIN_NODES_PER_CLUSTER: + raise EnvironmentError( + f"TC-BCK-070: cannot split {total} nodes into 2 clusters " + f"with min {self._MIN_NODES_PER_CLUSTER} each") + self.logger.info( + f"TC-BCK-070: splitting {total} storage nodes — " + f"Cluster-1 keeps: {c1_keep}, Cluster-2 gets: {c2_ips}") + # Remove c2 nodes from Cluster-1 if they are currently members + self._remove_nodes_from_cluster1(c2_ips) + else: + raise EnvironmentError( + f"TC-BCK-070: need at least {self._MIN_NODES_PER_CLUSTER} spare " + f"nodes for Cluster-2 (have {len(spare_ips)} spare out of " + f"{total} total). Pass spare nodes via NEW_NODE_IPS or " + f"STORAGE_PRIVATE_IPS, or set CLUSTER2_* env vars directly.") + + mgmt_ip = self.mgmt_nodes[0] + sbcli_cmd = self.base_cmd + ifname = os.environ.get("IFNAME", "eth0") + data_nic = os.environ.get("BOOTSTRAP_DATA_NIC", "eth1") + max_subsys = os.environ.get("BOOTSTRAP_MAX_SUBSYS", "1024") + ha_type = os.environ.get("HA_TYPE", "ha") + journal_partition = os.environ.get("BOOTSTRAP_JOURNAL_PARTITION", "0") + ha_jm_count = os.environ.get("BOOTSTRAP_HA_JM_COUNT", "3") + ndcs = os.environ.get("NDCS", str(self.ndcs)) + npcs = os.environ.get("NPCS", str(self.npcs)) + extra_cluster_args = os.environ.get("EXTRA_CLUSTER_ARGS", "") + extra_sn_args = os.environ.get("EXTRA_SN_ARGS", "") + spdk_image = os.environ.get("SPDK_IMAGE", "") + branch = os.environ.get("SBCLI_BRANCH", "main") + + # Step 0: ensure SSH connections to Cluster-2 storage nodes + for ip in c2_ips: + self.logger.info(f" [C2] Connecting SSH to {ip}") + try: + self.ssh_obj.connect( + address=ip, + bastion_server_address=self.bastion_server, + ) + except Exception as e: + self.logger.warning(f" [C2] SSH connect to {ip} failed: {e}") + + # Step 1: configure + deploy on each Cluster-2 storage node + for ip in c2_ips: + self.logger.info(f" [C2] Configuring + deploying storage node {ip}") + install_cmd = ( + f"pip install --force-reinstall " + f"git+https://github.com/simplyblock-io/sbcli.git@{branch}" + ) + self.ssh_obj.exec_command(node=ip, command=install_cmd) + sleep_n_sec(5) + configure_cmd = ( + f"{sbcli_cmd} --dev -d sn configure " + f"--max-subsys {max_subsys}" + ) + self.ssh_obj.exec_command(node=ip, command=configure_cmd) + deploy_cmd = f"{sbcli_cmd} sn deploy --ifname {ifname}" + self.ssh_obj.exec_command(node=ip, command=deploy_cmd) + + # Wait for SPDK containers to start + self.logger.info(" [C2] Waiting for SPDK containers to start...") + sleep_n_sec(30) + + # Step 2: create Cluster-2 on mgmt node + self.logger.info(" [C2] Creating second cluster on mgmt node") + create_cmd = ( + f"{sbcli_cmd} --dev -d cluster create" + f" --ha-type {ha_type}" + f" --data-chunks-per-stripe {ndcs}" + f" --parity-chunks-per-stripe {npcs}" + f" --ifname {ifname}" + ) + if extra_cluster_args: + create_cmd += f" {extra_cluster_args}" + self.ssh_obj.exec_command(node=mgmt_ip, command=create_cmd) + + # Extract Cluster-2 ID (the newest cluster that isn't Cluster-1) + out, _ = self.ssh_obj.exec_command( + node=mgmt_ip, + command=f"{sbcli_cmd} cluster list --json 2>/dev/null || " + f"{sbcli_cmd} cluster list" + ) + c2_id = self._extract_second_cluster_id(out) + self.logger.info(f" [C2] Cluster-2 ID: {c2_id}") + + # Step 3: add storage nodes to Cluster-2 + add_base = ( + f"{sbcli_cmd} --dev -d storage-node add-node" + f" --journal-partition {journal_partition}" + f" --ha-jm-count {ha_jm_count}" + f" --data-nics {data_nic}" + ) + if spdk_image: + add_base += f" --spdk-image {spdk_image}" + if extra_sn_args: + add_base += f" {extra_sn_args}" + + for ip in c2_ips: + self.logger.info(f" [C2] Adding storage node {ip} to Cluster-2") + add_cmd = f"{add_base} {c2_id} {ip}:5000 {ifname}" + self.ssh_obj.exec_command(node=mgmt_ip, command=add_cmd) + sleep_n_sec(3) + + # Step 4: activate Cluster-2 + self.logger.info(" [C2] Activating Cluster-2") + self.ssh_obj.exec_command( + node=mgmt_ip, + command=f"{sbcli_cmd} -d cluster activate {c2_id}" + ) + + # Step 5: create pool on Cluster-2 + self.logger.info(" [C2] Creating pool on Cluster-2") + self.ssh_obj.exec_command( + node=mgmt_ip, + command=f"{sbcli_cmd} pool add {self.pool_name} {c2_id}" + ) + + # Step 6: extract Cluster-2 secret + out, _ = self.ssh_obj.exec_command( + node=mgmt_ip, + command=f"{sbcli_cmd} cluster get-secret {c2_id}" + ) + c2_secret = out.strip().split("\n")[0].strip() + self.logger.info(" [C2] Cluster-2 secret obtained") + + # Set instance attributes + self._cluster2_id = c2_id + self._cluster2_secret = c2_secret + # Same mgmt node, same API endpoint + self._cluster2_api_url = self.api_base_url or f"http://{mgmt_ip}" + self._self_bootstrapped_c2 = True + + # Also set env vars so any downstream code can use them + os.environ["CLUSTER2_ID"] = c2_id + os.environ["CLUSTER2_SECRET"] = c2_secret + os.environ["CLUSTER2_API_BASE_URL"] = self._cluster2_api_url + + self.logger.info( + f"TC-BCK-070: Cluster-2 bootstrapped — ID={c2_id}, " + f"API={self._cluster2_api_url}, nodes={c2_ips}") + + def _remove_nodes_from_cluster1(self, ips_to_remove: list[str]): + """Remove storage nodes from Cluster-1 so they can join Cluster-2. + + For each IP, finds the node UUID in Cluster-1, suspends it, + shuts it down, removes it, and runs deploy-cleaner on the host. + """ + sn_data = self.sbcli_utils.get_storage_nodes().get("results", []) + + # Build IP→node_id mapping + ip_to_ids = {} + for node in sn_data: + nid = node.get("id") or node.get("uuid") or "" + nip = node.get("mgmt_ip") or node.get("ip") or "" + if nip and nid: + ip_to_ids.setdefault(nip, []).append(nid) + + for ip in ips_to_remove: + node_ids = ip_to_ids.get(ip, []) + if not node_ids: + self.logger.info(f" [C1] Node {ip} not in Cluster-1, skipping removal") + continue + for nid in node_ids: + self.logger.info(f" [C1] Removing node {nid} ({ip}) from Cluster-1") + try: + self.sbcli_utils.shutdown_node(nid, force=True) + sleep_n_sec(5) + except Exception as e: + self.logger.warning(f" [C1] Shutdown {nid} failed: {e}") + try: + self._sbcli(f"storage-node remove {nid}") + sleep_n_sec(3) + except Exception as e: + self.logger.warning(f" [C1] Remove {nid} failed: {e}") + + # Clean up the host + self.ssh_obj.exec_command( + node=ip, + command=f"{self.base_cmd} sn deploy-cleaner 2>/dev/null || true" + ) + + def _extract_second_cluster_id(self, cluster_list_output: str) -> str: + """Extract the Cluster-2 UUID from ``sbcli cluster list`` output. + + The output may be JSON (``cluster list --json``) or a table. + Returns the first cluster ID that is NOT ``self.cluster_id``. + """ + import json as _json + text = cluster_list_output.strip() + + # Try JSON first + try: + data = _json.loads(text) + if isinstance(data, list): + for entry in data: + cid = (entry.get("id") or entry.get("uuid") + or entry.get("cluster_id") or "") + if cid and cid != self.cluster_id: + return cid + except (_json.JSONDecodeError, ValueError): + pass + + # Fallback: parse table rows for UUID-like strings + import re + uuid_re = re.compile( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + re.IGNORECASE, + ) + for line in text.split("\n"): + match = uuid_re.search(line) + if match: + cid = match.group(0) + if cid != self.cluster_id: + return cid + + raise RuntimeError( + f"Could not find a second cluster ID in output:\n{text[:500]}") + + def _teardown_second_cluster(self): + """Destroy the self-bootstrapped second cluster.""" + if not self._self_bootstrapped_c2 or not self._cluster2_id: + return + self.logger.info(f"Tearing down self-bootstrapped Cluster-2 ({self._cluster2_id})") + mgmt_ip = self.mgmt_nodes[0] + try: + # Deactivate + delete Cluster-2 + self.ssh_obj.exec_command( + node=mgmt_ip, + command=f"{self.base_cmd} cluster deactivate {self._cluster2_id} || true" + ) + sleep_n_sec(5) + self.ssh_obj.exec_command( + node=mgmt_ip, + command=f"{self.base_cmd} cluster delete {self._cluster2_id} || true" + ) + self.logger.info("Cluster-2 deleted") + except Exception as e: + self.logger.warning(f"Cluster-2 teardown error: {e}") # ── Cluster-2 sbcli helper ──────────────────────────────────────────────── def _sbcli_c2(self, subcmd: str) -> tuple[str, str]: - """Run sbcli command targeted at Cluster-2.""" + """Run sbcli command targeted at Cluster-2. + + K8s mode: kubectl exec into the admin pod in C2's namespace. + Docker: SSH to mgmt node with CLUSTER_ID/SECRET/API_BASE_URL env prefix. + """ + if self.k8s_test and self._k8s_c2 is not None: + cmd = ( + f"CLUSTER_ID={self._cluster2_id} " + f"CLUSTER_SECRET={self._cluster2_secret} " + f"API_BASE_URL={self._cluster2_api_url} " + f"{self.base_cmd} {subcmd}" + ) + out, err = self._k8s_c2.exec_sbcli(cmd) + self.logger.debug(f"CMD (k8s-c2): {cmd}\nOUT: {out}\nERR: {err}") + return out, err env_prefix = ( f"CLUSTER_ID={self._cluster2_id} " f"CLUSTER_SECRET={self._cluster2_secret} " @@ -3040,61 +3417,179 @@ def _export_backup_metadata(self, backup_id: str) -> str: Export backup metadata from Cluster-1 using the CLI backup export command, writing a JSON file to self._meta_file. - Returns the path of the metadata file on the mgmt node. + In K8s mode, the export runs inside C1's admin pod, then the file + is transferred into C2's admin pod via ``kubectl cp``. + + Returns the path of the metadata file on the mgmt node (or inside + the admin pod in K8s mode). """ out, err = self._sbcli(f"backup export -o {self._meta_file}") assert not (err and "error" in err.lower()), \ f"TC-BCK-072: backup export failed: {err}" self.logger.info(f"TC-BCK-072: backup export result: {(out or '').strip()}") + + if self.k8s_test and self._k8s_c2 is not None: + # Transfer metadata file from C1 admin pod → C2 admin pod + c1_ns = self.sbcli_utils.k8s.namespace + c1_pod = self.sbcli_utils.k8s.get_admin_pod() + c2_ns = self._cluster2_namespace + c2_pod = self._k8s_c2.get_admin_pod() + local_tmp = f"/tmp/cc_backup_meta_{int(time.time())}.json" + self.logger.info( + f"TC-BCK-072: transferring metadata " + f"{c1_ns}/{c1_pod} → {c2_ns}/{c2_pod}") + # kubectl cp from C1 admin pod to runner + self._k8s_c2._exec_kubectl( + f"kubectl cp {c1_ns}/{c1_pod}:{self._meta_file} {local_tmp}") + # kubectl cp from runner into C2 admin pod + self._k8s_c2._exec_kubectl( + f"kubectl cp {local_tmp} {c2_ns}/{c2_pod}:{self._meta_file}") + self.logger.info("TC-BCK-072: metadata file transferred to C2 admin pod ✓") + return self._meta_file - # ── main run ────────────────────────────────────────────────────────────── + # ── K8s-native CRD cross-cluster restore ──────────────────────────────── - def run(self): - self.logger.info("=== TestBackupCrossClusterRestore START ===") - if self.k8s_test: - self.logger.info( - "TestBackupCrossClusterRestore requires CLI-only operations " - "(export/import/source-switch) — skipping in K8s mode.") - return + def _run_k8s_native_cross_cluster_restore(self, backup_id: str, + orig_checksums: dict): + """Cross-cluster restore via K8s CRDs (BackupImport → BackupRestore). - # TC-BCK-070: check prerequisites - self._check_prerequisites() + Flow: + 1. Get backup UUID from C1's StorageBackup status. + 2. Create BackupImport CR on C2 → wait Done → get storageBackupRef. + 3. Create BackupRestore CR on C2 with pvcTemplate → wait Done. + 4. Verify data integrity via utility pod on restored PVC. + + The controller handles source-switching automatically — no manual + ``backup source-switch`` is needed. + """ + k8s_c1 = self._ensure_k8s_utils() + c2_cluster_name = os.environ.get( + "CLUSTER2_CRD_NAME", self._cluster_name) + + # backup_id in K8s mode is the StorageBackup CRD name; get the + # actual UUID from its status.backupId field. + source_backup_uuid = k8s_c1.get_storage_backup_id(backup_id) + assert source_backup_uuid, ( + f"TC-BCK-072: could not get backupId from StorageBackup " + f"'{backup_id}' status" + ) self.logger.info( - f"TC-BCK-070: prerequisites OK — Cluster-2 ID={self._cluster2_id}") + f"TC-BCK-072: StorageBackup '{backup_id}' → " + f"backupId={source_backup_uuid}") - self.fio_node = self.fio_node[0] - self._ensure_pool_and_sc() + # TC-BCK-073: create BackupImport on C2 + import_name = f"cc-import-{_rand_suffix().lower()}" + self.logger.info( + f"TC-BCK-073: creating BackupImport '{import_name}' on C2 " + f"(source={self._cluster_name}/{source_backup_uuid} " + f"→ target={c2_cluster_name})") + self._k8s_c2.create_backup_import( + name=import_name, + source_cluster_name=self._cluster_name, + source_backup_id=source_backup_uuid, + target_cluster_name=c2_cluster_name, + ) - # ── Cluster-1: write data → snapshot + backup → wait ────────────────── + restore_name = None + try: + # Wait for BackupImport to reach Done + self._k8s_c2.wait_backup_import_done( + import_name, timeout=_RESTORE_COMPLETE_TIMEOUT) + storage_backup_ref = ( + self._k8s_c2.get_backup_import_storage_backup_ref( + import_name)) + assert storage_backup_ref, ( + f"TC-BCK-073: BackupImport '{import_name}' Done but " + f"storageBackupRef is empty") + self.logger.info( + f"TC-BCK-073: BackupImport Done — " + f"storageBackupRef={storage_backup_ref}") + + # TC-BCK-075: create BackupRestore on C2 with pvcTemplate + restore_name = f"cc-restore-{_rand_suffix().lower()}" + restored_pvc = f"cc-rest-{_rand_suffix().lower()}" + c2_sc = os.environ.get( + "CLUSTER2_STORAGE_CLASS", self._storage_class_name) + self.logger.info( + f"TC-BCK-075: creating BackupRestore '{restore_name}' on C2 " + f"(backupRef={storage_backup_ref} → PVC={restored_pvc})") + self._k8s_c2.create_backup_restore( + name=restore_name, + backup_ref_name=storage_backup_ref, + pvc_name=restored_pvc, + pvc_size="5Gi", + cluster_name=c2_cluster_name, + storage_class=c2_sc, + ) - # TC-BCK-071: create lvol on Cluster-1, write known data, create S3 backup - self.logger.info("TC-BCK-071: Cluster-1 — write data and create S3 backup") - lvol_name, lvol_id = self._create_lvol( - name=f"cc_src_{_rand_suffix()}", size="5G") - device, mount = self._connect_and_mount(lvol_name, lvol_id) - self._run_fio(mount, runtime=30) + # Wait for BackupRestore to reach Done (PVC auto-created) + self._k8s_c2.wait_backup_restore_done( + restore_name, timeout=_RESTORE_COMPLETE_TIMEOUT) + self.logger.info( + f"TC-BCK-075: BackupRestore '{restore_name}' Done — " + f"PVC '{restored_pvc}' created on C2") - orig_checksums = self._get_checksums(self.fio_node, mount) - self.logger.info( - f"TC-BCK-071: {len(orig_checksums)} checksum(s) captured on Cluster-1") + # TC-BCK-076: verify data integrity on restored PVC via + # utility pod in C2's namespace + self.logger.info( + "TC-BCK-076: verifying checksums on restored PVC in C2") + pod_name = f"cksum-c2-{_rand_suffix().lower()}" + self._k8s_c2.create_utility_pod(pod_name, restored_pvc) + try: + self._k8s_c2.wait_pod_running(pod_name, timeout=600) + files = self._k8s_c2.find_files_in_pvc(pod_name) + actual = self._k8s_c2.generate_checksums_in_pvc( + pod_name, files) + + expected_by_name = { + os.path.basename(k): v + for k, v in orig_checksums.items() + } + actual_by_name = { + os.path.basename(k): v for k, v in actual.items() + } + assert actual_by_name, ( + "TC-BCK-076: no files in restored PVC for checksum " + "verification" + ) + for fname, cksum in expected_by_name.items(): + assert fname in actual_by_name, ( + f"TC-BCK-076: file {fname} not found in restored PVC" + ) + assert actual_by_name[fname] == cksum, ( + f"TC-BCK-076: checksum mismatch for {fname}: " + f"expected {cksum}, got {actual_by_name[fname]}" + ) + self.logger.info( + "TC-BCK-076: cross-cluster restore checksums match " + "(K8s-native CRD flow)") + finally: + try: + self._k8s_c2.delete_pod(pod_name) + except Exception: + pass - snap_name = f"cc_snap_{_rand_suffix()}" - snap_id = self._create_snapshot(lvol_id, snap_name, backup=True) - self.logger.info(f"TC-BCK-071: snapshot {snap_id} + S3 backup triggered") - sleep_n_sec(5) + finally: + # Best-effort cleanup of C2 CRDs + for kind, crd_name in [ + ("backuprestore", restore_name), + ("backupimport", import_name), + ]: + if crd_name: + try: + self._k8s_c2.delete_resource(kind, crd_name) + except Exception as exc: + self.logger.warning( + f"TC-BCK-076b: cleanup {kind}/{crd_name}: " + f"{exc}") - backups = self._list_backups() - assert backups, "TC-BCK-071: no backups found on Cluster-1 after snapshot" - bk_entry = self._get_backup_for_snapshot(snap_name, backups) or backups[0] - backup_id = ( - bk_entry.get("id") or bk_entry.get("ID") or bk_entry.get("uuid") or "" - ) - assert backup_id, f"TC-BCK-071: could not extract backup_id: {bk_entry}" - self._wait_for_backup(backup_id) - self.logger.info(f"TC-BCK-071: backup {backup_id} is done on Cluster-1 ✓") + # ── CLI cross-cluster restore (Docker mode) ────────────────────────────── - # ── Cluster-2: import → source-switch → restore → verify → switch-back ─ + def _run_cli_cross_cluster_restore(self, backup_id: str, + orig_checksums: dict): + """Cross-cluster restore via CLI (export → import → source-switch → + restore → verify → switch-back).""" # TC-BCK-072: export metadata from Cluster-1 self.logger.info("TC-BCK-072: exporting backup metadata from Cluster-1") @@ -3108,51 +3603,64 @@ def run(self): self.logger.info(f"TC-BCK-073: import result: {out.strip()}") # TC-BCK-074: verify backup is visible on Cluster-2 - self.logger.info("TC-BCK-074: Cluster-2 — backup list should show imported backup") + self.logger.info( + "TC-BCK-074: Cluster-2 — backup list should show imported backup") out2, err2 = self._sbcli_c2("backup list") assert not (err2 and "error" in err2.lower()), \ f"TC-BCK-074: backup list on Cluster-2 failed: {err2}" assert backup_id in out2 or out2.strip(), \ - f"TC-BCK-074: imported backup_id {backup_id} not visible on Cluster-2" - self.logger.info(f"TC-BCK-074: Cluster-2 backup list snippet: {out2[:200]}") + f"TC-BCK-074: imported backup_id {backup_id} not visible on C2" + self.logger.info( + f"TC-BCK-074: Cluster-2 backup list snippet: {out2[:200]}") # TC-BCK-074b: switch Cluster-2's backup source to Cluster-1's S3 self.logger.info( - f"TC-BCK-074b: Cluster-2 — backup source-switch to Cluster-1 ({self.cluster_id})") - out_sw, err_sw = self._sbcli_c2(f"backup source-switch {self.cluster_id}") + f"TC-BCK-074b: Cluster-2 — backup source-switch to " + f"Cluster-1 ({self.cluster_id})") + out_sw, err_sw = self._sbcli_c2( + f"backup source-switch {self.cluster_id}") assert not (err_sw and "error" in err_sw.lower()), \ f"TC-BCK-074b: source-switch to Cluster-1 failed: {err_sw}" - self.logger.info(f"TC-BCK-074b: source switched to Cluster-1 ✓ — {out_sw.strip()}") + self.logger.info( + f"TC-BCK-074b: source switched to Cluster-1 — " + f"{out_sw.strip()}") try: - # TC-BCK-075: restore on Cluster-2 (now sourced from Cluster-1's S3) + # TC-BCK-075: restore on Cluster-2 restored_name = f"cc_rest_{_rand_suffix()}" self.logger.info( - f"TC-BCK-075: Cluster-2 — backup restore {backup_id} → {restored_name}") + f"TC-BCK-075: Cluster-2 — backup restore " + f"{backup_id} → {restored_name}") c2_pool = os.environ.get("CLUSTER2_POOL", self.pool_name) out3, err3 = self._sbcli_c2( - f"backup restore {backup_id} --lvol {restored_name} --pool {c2_pool}") + f"backup restore {backup_id} " + f"--lvol {restored_name} --pool {c2_pool}") assert not (err3 and "error" in err3.lower()), \ f"TC-BCK-075: restore on Cluster-2 failed: {err3}" - self.logger.info(f"TC-BCK-075: restore triggered: {out3.strip()}") + self.logger.info( + f"TC-BCK-075: restore triggered: {out3.strip()}") self._c2_lvols.append(restored_name) # Wait for restore to complete on Cluster-2 - self.logger.info("TC-BCK-075: waiting for Cluster-2 restore to complete…") + self.logger.info( + "TC-BCK-075: waiting for Cluster-2 restore to complete…") deadline = time.time() + _RESTORE_COMPLETE_TIMEOUT while time.time() < deadline: lvol_out, _ = self._sbcli_c2("lvol list") if restored_name in lvol_out: - self.logger.info("TC-BCK-075: restored lvol appeared on Cluster-2 ✓") + self.logger.info( + "TC-BCK-075: restored lvol appeared on Cluster-2") break sleep_n_sec(_POLL_INTERVAL) else: raise TimeoutError( - f"TC-BCK-075: restored lvol {restored_name} did not appear " - f"on Cluster-2 within {_RESTORE_COMPLETE_TIMEOUT}s") + f"TC-BCK-075: restored lvol {restored_name} did not " + f"appear on Cluster-2 within " + f"{_RESTORE_COMPLETE_TIMEOUT}s") - # TC-BCK-076: data integrity — connect on FIO node via Cluster-2 connect string - self.logger.info("TC-BCK-076: connecting restored lvol from Cluster-2") + # TC-BCK-076: data integrity — connect via Cluster-2 + self.logger.info( + "TC-BCK-076: connecting restored lvol from Cluster-2") c2_connect_out, c2_connect_err = self._sbcli_c2( f"volume connect {restored_name}") connect_lines = [ @@ -3160,47 +3668,114 @@ def run(self): for line in c2_connect_out.strip().split("\n") if line.strip() and "nvme connect" in line ] - assert connect_lines, \ - f"TC-BCK-076: no nvme connect strings from Cluster-2: {c2_connect_out}" + assert connect_lines, ( + f"TC-BCK-076: no nvme connect strings from Cluster-2: " + f"{c2_connect_out}") initial_devs = self.ssh_obj.get_devices(node=self.fio_node) for cmd in connect_lines: - self.ssh_obj.exec_command(node=self.fio_node, command=cmd) + self.ssh_obj.exec_command( + node=self.fio_node, command=cmd) sleep_n_sec(3) final_devs = self.ssh_obj.get_devices(node=self.fio_node) new_devs = [d for d in final_devs if d not in initial_devs] - assert new_devs, "TC-BCK-076: no new block device after connecting Cluster-2 lvol" + assert new_devs, ( + "TC-BCK-076: no new block device after connecting " + "Cluster-2 lvol") r_device = f"/dev/{new_devs[0]}" r_mount = f"{self.mount_path}/cc_rest_{_rand_suffix()}" - self.ssh_obj.exec_command(self.fio_node, f"mkdir -p {r_mount}") - self.ssh_obj.mount_path(node=self.fio_node, device=r_device, mount_path=r_mount) + self.ssh_obj.exec_command( + self.fio_node, f"mkdir -p {r_mount}") + self.ssh_obj.mount_path( + node=self.fio_node, device=r_device, + mount_path=r_mount) self.mounted.append((self.fio_node, r_mount)) - self._verify_checksums(self.fio_node, r_mount, orig_checksums) - self.logger.info("TC-BCK-076: cross-cluster restore checksums match ✓") + self._verify_checksums( + self.fio_node, r_mount, orig_checksums) + self.logger.info( + "TC-BCK-076: cross-cluster restore checksums match") finally: - # TC-BCK-076b: switch Cluster-2's backup source back to local (always) - self.logger.info("TC-BCK-076b: Cluster-2 — backup source-switch back to local") - out_back, err_back = self._sbcli_c2("backup source-switch local") + # TC-BCK-076b: switch Cluster-2 source back to local + self.logger.info( + "TC-BCK-076b: Cluster-2 — backup source-switch " + "back to local") + out_back, err_back = self._sbcli_c2( + "backup source-switch local") if err_back and "error" in err_back.lower(): self.logger.warning( - f"TC-BCK-076b: source-switch-back warning: {err_back}") + f"TC-BCK-076b: source-switch-back warning: " + f"{err_back}") else: self.logger.info( - f"TC-BCK-076b: source switched back to local ✓ — {out_back.strip()}") + f"TC-BCK-076b: source switched back to local — " + f"{out_back.strip()}") + + # ── main run ────────────────────────────────────────────────────────────── + + def run(self): + self.logger.info("=== TestBackupCrossClusterRestore START ===") + + # TC-BCK-070: check prerequisites + self._check_prerequisites() + self.logger.info( + f"TC-BCK-070: prerequisites OK — Cluster-2 ID={self._cluster2_id}") + + self.fio_node = self.fio_node[0] + self._ensure_pool_and_sc() + + # ── Cluster-1: write data → snapshot + backup → wait ────────────────── + + # TC-BCK-071: create lvol on Cluster-1, write known data, create S3 backup + self.logger.info("TC-BCK-071: Cluster-1 — write data and create S3 backup") + lvol_name, lvol_id = self._create_lvol( + name=f"cc_src_{_rand_suffix()}", size="5G") + device, mount = self._connect_and_mount(lvol_name, lvol_id) + self._run_fio(mount, runtime=30) + + orig_checksums = self._get_checksums(self.fio_node, mount) + self.logger.info( + f"TC-BCK-071: {len(orig_checksums)} checksum(s) captured on Cluster-1") + + snap_name = f"cc_snap_{_rand_suffix()}" + snap_id = self._create_snapshot(lvol_id, snap_name, backup=True) + self.logger.info(f"TC-BCK-071: snapshot {snap_id} + S3 backup triggered") + sleep_n_sec(5) + + backups = self._list_backups() + assert backups, "TC-BCK-071: no backups found on Cluster-1 after snapshot" + bk_entry = self._get_backup_for_snapshot(snap_name, backups) or backups[0] + backup_id = ( + bk_entry.get("id") or bk_entry.get("ID") or bk_entry.get("uuid") or "" + ) + assert backup_id, f"TC-BCK-071: could not extract backup_id: {bk_entry}" + self._wait_for_backup(backup_id) + self.logger.info(f"TC-BCK-071: backup {backup_id} is done on Cluster-1 ✓") + + # ── Cluster-2: import → restore → verify ──────────────────────────────── + + if self.k8s_test and self._k8s_c2 is not None: + self._run_k8s_native_cross_cluster_restore( + backup_id, orig_checksums) + else: + self._run_cli_cross_cluster_restore( + backup_id, orig_checksums) self.logger.info("=== TestBackupCrossClusterRestore PASSED ===") # ── teardown ────────────────────────────────────────────────────────────── def teardown(self, delete_lvols=True, close_ssh=True, skip_k8s_cleanup=False): - # Safety: ensure Cluster-2's source is switched back to local (always) - try: - self._sbcli_c2("backup source-switch local") - except Exception as e: - self.logger.warning(f"source-switch-back in teardown warning: {e}") + # Safety: ensure Cluster-2's source is switched back to local + # (CLI mode only — K8s-native mode uses CRDs, no manual source-switch) + if not self.k8s_test: + try: + self._sbcli_c2("backup source-switch local") + except Exception as e: + self.logger.warning( + f"source-switch-back in teardown warning: {e}") if delete_lvols: # Best-effort cleanup of Cluster-2 resources @@ -3219,6 +3794,9 @@ def teardown(self, delete_lvols=True, close_ssh=True, skip_k8s_cleanup=False): except Exception: pass + # Tear down self-bootstrapped Cluster-2 (if we created it) + self._teardown_second_cluster() + super().teardown(delete_lvols=delete_lvols, close_ssh=close_ssh, skip_k8s_cleanup=skip_k8s_cleanup) @@ -4572,10 +5150,10 @@ def run(self): self._verify_checksums(self.fio_node, rst_v1_mnt, checksums_v1) self.logger.info("TC-BCK-171: v1 restore data integrity PASSED") - # TC-BCK-172: restore v2, verify + # TC-BCK-172: restore v2, verify (must use 10G since lvol was resized) self.logger.info("TC-BCK-172: Restoring v2 …") rst_v2 = f"rszrst2{_rand_suffix()}" - self._restore_backup(bk_v2, rst_v2) + self._restore_backup(bk_v2, rst_v2, restore_size="10G") self._wait_for_restore(rst_v2) rst_v2_id = self._get_lvol_id(rst_v2) assert rst_v2_id @@ -4643,7 +5221,7 @@ def run(self): # TC-BCK-175: backup list with cluster-id filter self.logger.info("TC-BCK-175: Testing --cluster-id filter …") - out, err = self._sbcli(f"-d backup list --cluster-id {self.cluster_id}") + out, err = self._sbcli(f"backup list --cluster-id {self.cluster_id}") assert not (err and "error" in err.lower()), \ f"backup list --cluster-id failed: {err}" assert bk_id in (out or "") or snap_name in (out or "") or lvol_name in (out or ""), \ diff --git a/e2e/e2e_tests/cluster_test_base.py b/e2e/e2e_tests/cluster_test_base.py index e4b8f96a3f..c66f508009 100644 --- a/e2e/e2e_tests/cluster_test_base.py +++ b/e2e/e2e_tests/cluster_test_base.py @@ -897,7 +897,7 @@ def _generate_checksums_dual(self, lvol_name, files=None, directory=None): files = k8s.find_files_in_pvc(pod_name) return k8s.generate_checksums_in_pvc(pod_name, files) finally: - k8s.delete_pod(pod_name) + k8s.delete_pod(pod_name, wait=True) if pod_name in self._k8s_utility_pods: self._k8s_utility_pods.remove(pod_name) else: @@ -973,10 +973,23 @@ def _k8s_default_teardown(self): self._k8s_configmaps.clear() for snap_name in list(self._k8s_volume_snapshots): try: - k8s.delete_volume_snapshot(snap_name) + k8s.delete_volume_snapshot(snap_name, wait=True) except Exception as e: self.logger.warning(f"[k8s-teardown] VolumeSnapshot error {snap_name}: {e}") self._k8s_volume_snapshots.clear() + # Catch-all: delete any remaining test VolumeSnapshots that may not + # have been tracked (e.g. snapshot-1, snapshot-2 naming pattern). + try: + ns = k8s.namespace + k8s._exec_kubectl( + f"kubectl get volumesnapshot -n {ns} --no-headers " + f"-o custom-columns=NAME:.metadata.name 2>/dev/null " + f"| grep -E '^(snap-|snapshot-)' " + f"| xargs -r kubectl delete volumesnapshot -n {ns} " + f"--ignore-not-found --wait=true --timeout=120s" + ) + except Exception as e: + self.logger.warning(f"[k8s-teardown] catch-all snapshot cleanup error: {e}") for pvc_name in list(self._k8s_pvcs): try: k8s.delete_pvc(pvc_name) diff --git a/e2e/e2e_tests/k8s_native_add_node.py b/e2e/e2e_tests/k8s_native_add_node.py index 16da717f44..a02d48ae3e 100644 --- a/e2e/e2e_tests/k8s_native_add_node.py +++ b/e2e/e2e_tests/k8s_native_add_node.py @@ -366,54 +366,88 @@ def run(self): sleep_n_sec(30) - # ── Step 5: Expand cluster by creating StorageNode CRs ────────── - self.logger.info("Step 5: Creating StorageNode CRs for new workers") + # ── Step 5 & 6: Expand cluster one node at a time ───────────── + # Adding multiple nodes simultaneously can fail; add each node + # sequentially and wait for it to come online before proceeding. + self.logger.info( + f"Step 5: Adding {len(self.new_worker_nodes)} new worker(s) " + f"sequentially" + ) timestamp = int(datetime.now().timestamp()) + all_known_ids = set(initial_node_ids) + new_node_ids = [] - self.k8s_utils.patch_storage_node_add_workers( - new_workers=self.new_worker_nodes, - ) - sleep_n_sec(10) + for idx, worker in enumerate(self.new_worker_nodes): + step_label = f"5.{idx + 1}/{len(self.new_worker_nodes)}" + self.logger.info( + f"Step {step_label}: Creating StorageNode CR for '{worker}'" + ) - # ── Step 6: Wait for expansion ─────────────────────────────────── - self.logger.info("Step 6: Waiting for expansion to complete") + # Create StorageNode CR for this single worker + self.k8s_utils.patch_storage_node_add_workers( + new_workers=[worker], + ) - # Wait for new snode-spdk pods - expected_pods = initial_node_count + len(self.new_worker_nodes) - self.logger.info( - f"Waiting for {expected_pods} snode-spdk pods (was {initial_node_count})" - ) - self.k8s_utils.wait_spdk_pods_ready( - expected_count=expected_pods, timeout=900 - ) + # Wait for the operator to populate the per-node-config ConfigMap + # for this worker. Without this, the DaemonSet pod starts with + # MAX_LVOL=0 and crashes in s-node-api-config-generator. + self.k8s_utils.wait_for_per_node_config(worker, timeout=120) - # Wait for cluster to enter expansion state (may already be past it) - try: + # Delete any stale storage-node pods on this worker so the + # DaemonSet recreates them with the correct configuration + self.k8s_utils.delete_storage_node_pods_on_worker(worker) + + # Wait for the expected number of snode-spdk pods + expected_pods = initial_node_count + idx + 1 + self.logger.info( + f"Waiting for {expected_pods} snode-spdk pods " + f"(was {initial_node_count})" + ) + self.k8s_utils.wait_spdk_pods_ready( + expected_count=expected_pods, timeout=900 + ) + + # Wait for cluster to enter/pass expansion state + try: + self.sbcli_utils.wait_for_cluster_status( + cluster_id=self.cluster_id, + status="in_expansion", + timeout=300, + ) + except Exception: + self.logger.info( + "Cluster may already be past in_expansion state" + ) + + # Discover and wait for the new node to come online + sleep_n_sec(30) + all_nodes_now = self.sbcli_utils.get_storage_nodes()["results"] + for n in all_nodes_now: + if n["id"] not in all_known_ids: + self.logger.info( + f"New storage node discovered: {n['id']}" + ) + self.sbcli_utils.wait_for_storage_node_status( + node_id=n["id"], + status="online", + timeout=600, + ) + new_node_ids.append(n["id"]) + all_known_ids.add(n["id"]) + + # Wait for cluster to return to active before adding next node self.sbcli_utils.wait_for_cluster_status( cluster_id=self.cluster_id, - status="in_expansion", - timeout=300, - ) - except Exception: - self.logger.info("Cluster may already be past in_expansion state") - - # Find new node IDs - sleep_n_sec(60) - all_nodes_now = self.sbcli_utils.get_storage_nodes()["results"] - new_node_ids = [ - n["id"] for n in all_nodes_now if n["id"] not in initial_node_ids - ] - self.logger.info(f"New storage node IDs: {new_node_ids}") - - # Wait for new nodes to come online - for node_id in new_node_ids: - self.sbcli_utils.wait_for_storage_node_status( - node_id=node_id, - status="online", + status="active", timeout=600, ) + self.logger.info( + f"Worker '{worker}' added and online " + f"({idx + 1}/{len(self.new_worker_nodes)})" + ) + + self.logger.info(f"All new storage node IDs: {new_node_ids}") - # Wait for cluster to be active cluster_details = self.sbcli_utils.wait_for_cluster_status( cluster_id=self.cluster_id, status="active", diff --git a/e2e/e2e_tests/k8s_native_node_migration.py b/e2e/e2e_tests/k8s_native_node_migration.py index 091de4871e..cad2ec53a4 100644 --- a/e2e/e2e_tests/k8s_native_node_migration.py +++ b/e2e/e2e_tests/k8s_native_node_migration.py @@ -391,6 +391,20 @@ def run(self): migration_timestamp = int(datetime.now().timestamp()) + # Ensure the storage-node pod on the migration target is healthy + # BEFORE creating the StorageNodeOps CR. If worker-5 was labelled + # during cluster setup the DaemonSet will have scheduled a pod, but + # the operator may not have populated the per-node-config ConfigMap + # entry yet — causing the init container to crash with MAX_LVOL=0. + # Fix the pod now so the operator finds a healthy pod when it starts + # the migration and can resolve DNS immediately. + self.k8s_utils.wait_for_per_node_config( + self.migrate_to_worker, timeout=120 + ) + self.k8s_utils.delete_storage_node_pods_on_worker( + self.migrate_to_worker + ) + ops_name, storage_node_cr = self.k8s_utils.patch_storage_node_migrate( node_uuid=migrate_node_uuid, target_worker=self.migrate_to_worker, diff --git a/e2e/e2e_tests/test_add_node_edge_cases.py b/e2e/e2e_tests/test_add_node_edge_cases.py index baac581eb3..6b8ba9ef34 100644 --- a/e2e/e2e_tests/test_add_node_edge_cases.py +++ b/e2e/e2e_tests/test_add_node_edge_cases.py @@ -114,7 +114,11 @@ def _add_node_k8s(self, worker_name: str, initial_pod_count: int): k8s_utils = K8sUtils(ssh_obj=self.ssh_obj, mgmt_node=mgmt_node) k8s_utils.patch_storage_node_add_workers(new_workers=[worker_name]) - sleep_n_sec(10) + + # Wait for operator to populate per-node-config ConfigMap for this + # worker before restarting pods — prevents MAX_LVOL=0 crash. + k8s_utils.wait_for_per_node_config(worker_name, timeout=120) + k8s_utils.delete_storage_node_pods_on_worker(worker_name) expected_pods = initial_pod_count + 1 self.logger.info(f"Waiting for {expected_pods} snode-spdk pods") @@ -430,7 +434,8 @@ def run(self): mgmt_node = self.mgmt_nodes[0] if self.mgmt_nodes else "" k8s_utils = K8sUtils(ssh_obj=self.ssh_obj, mgmt_node=mgmt_node) k8s_utils.patch_storage_node_add_workers(new_workers=[nodes_to_add[0]]) - sleep_n_sec(10) + k8s_utils.wait_for_per_node_config(nodes_to_add[0], timeout=120) + k8s_utils.delete_storage_node_pods_on_worker(nodes_to_add[0]) k8s_utils.wait_spdk_pods_ready( expected_count=initial_pod_count + 1, timeout=900 ) diff --git a/e2e/e2e_tests/test_api_parity_audit.py b/e2e/e2e_tests/test_api_parity_audit.py index 845f82fa8f..f08708208e 100644 --- a/e2e/e2e_tests/test_api_parity_audit.py +++ b/e2e/e2e_tests/test_api_parity_audit.py @@ -29,6 +29,9 @@ def __init__(self, **kwargs): self.test_name = "api_parity_audit" self.logger = setup_logger(__name__) self.findings = [] + # Pool created for the duration of the audit + self._audit_pool_name = "parity_audit_pool" + self._audit_pool_id = None def setup(self): super().setup() @@ -43,6 +46,19 @@ def setup(self): def run(self): self.logger.info("=== API Parity Audit: CLI / v1 / v2 ===") + # Phase 0: ensure a pool exists for the entire audit + try: + self._ensure_audit_pool() + except Exception: + self.logger.error( + f"[pool.setup] failed to create audit pool: " + f"{traceback.format_exc()}" + ) + self._finding( + "error", "audit_crash", "pool.setup", + detail=traceback.format_exc(limit=3), + ) + # Phase 1: read-only audits (safe) audits_readonly = [ ("cluster.list", self._audit_cluster_list), @@ -85,6 +101,9 @@ def run(self): self._finding("error", "audit_crash", name, detail=traceback.format_exc(limit=3)) + # Cleanup the audit pool + self._cleanup_audit_pool() + # Generate report report_dir = self.docker_logs_path or os.path.join("logs") os.makedirs(report_dir, exist_ok=True) @@ -143,7 +162,10 @@ def run(self): elif cat == "interface_error": self.logger.info( f" [{sev}] {op} iface={f.get('interface')} " - f"detail={f.get('detail', '')}" + f"detail={f.get('detail', '')} " + f"api_call={f.get('api_call', '')} " + f"http_status={f.get('http_status', '')} " + f"response={f.get('response_preview', '')}" ) elif cat == "audit_crash": detail = f.get("detail", "") @@ -161,6 +183,47 @@ def run(self): f"See report: {html_path}" ) + # ── pool lifecycle (ensure pool exists for all audits) ───────── + + def _ensure_audit_pool(self): + """Create the audit pool if it doesn't already exist.""" + pools = self.sbcli_utils.list_storage_pools() + if pools and self._audit_pool_name in pools: + self._audit_pool_id = pools[self._audit_pool_name] + self.logger.info( + f"[pool.setup] Audit pool already exists: {self._audit_pool_id}" + ) + return + + self.logger.info( + f"[pool.setup] Creating audit pool '{self._audit_pool_name}'" + ) + self.sbcli_utils.add_storage_pool(self._audit_pool_name) + sleep_n_sec(5) + + pools = self.sbcli_utils.list_storage_pools() + if pools and self._audit_pool_name in pools: + self._audit_pool_id = pools[self._audit_pool_name] + self.logger.info( + f"[pool.setup] Audit pool created: {self._audit_pool_id}" + ) + else: + self.logger.error( + f"[pool.setup] Audit pool '{self._audit_pool_name}' " + f"not found after creation" + ) + + def _cleanup_audit_pool(self): + """Remove the audit pool after all audits are done.""" + try: + self.sbcli_utils.delete_storage_pool(self._audit_pool_name) + sleep_n_sec(2) + self.logger.info( + f"[pool.cleanup] Deleted audit pool '{self._audit_pool_name}'" + ) + except Exception as exc: + self.logger.warning(f"[pool.cleanup] failed: {exc}") + # ── finding helpers ─────────────────────────────────────────────── def _finding(self, severity, category, operation, **kwargs): @@ -175,21 +238,38 @@ def _finding(self, severity, category, operation, **kwargs): def _run_cli(self, cmd): """Run ``sbctl `` on the first management node via SSH. - Returns parsed JSON on success, or ``None`` on failure. + Returns a dict with keys: + - ``data``: parsed JSON on success, or ``None`` on failure + - ``stdout``: raw stdout text + - ``stderr``: raw stderr text + - ``command``: the full command that was run """ - if not self.mgmt_nodes: - return None full_cmd = f"{self.base_cmd} {cmd}" + result = { + "data": None, + "stdout": "", + "stderr": "", + "command": full_cmd, + } + + if not self.mgmt_nodes: + result["stderr"] = "no management nodes available" + return result + try: stdout, stderr = self.ssh_obj.exec_command( self.mgmt_nodes[0], full_cmd ) + result["stdout"] = stdout or "" + result["stderr"] = stderr or "" + if stdout and stdout.strip(): # CLI may print non-JSON header lines; find the JSON part lines = stdout.strip().splitlines() # Try full output first try: - return json.loads(stdout.strip()) + result["data"] = json.loads(stdout.strip()) + return result except json.JSONDecodeError: pass # Try last line @@ -197,13 +277,15 @@ def _run_cli(self, cmd): line = line.strip() if line.startswith(("{", "[")): try: - return json.loads(line) + result["data"] = json.loads(line) + return result except json.JSONDecodeError: continue - return None + return result except Exception as exc: self.logger.warning(f"CLI '{cmd}' failed: {exc}") - return None + result["stderr"] = str(exc) + return result # ── normalizers ─────────────────────────────────────────────────── @@ -237,6 +319,28 @@ def _normalize_cli(data): return data[0] return data + # ── helpers for recording interface errors with context ──────── + + def _check_interface_data(self, operation, iface, data, api_call="", + http_status=None, raw_response=""): + """Record an error finding if *data* is None / empty, with API details.""" + if data is None: + detail = "returned None" + if iface == "cli" and isinstance(raw_response, dict): + stderr = raw_response.get("stderr", "") + if stderr: + detail = f"returned None; stderr: {stderr[:300]}" + self._finding( + "error", "interface_error", operation, + interface=iface, + detail=detail, + api_call=api_call, + http_status=http_status, + response_preview=str(raw_response)[:500] if raw_response else "", + ) + return False + return True + # ── comparison engine ───────────────────────────────────────────── def _compare_dicts(self, operation, cli_data, v1_data, v2_data, @@ -310,9 +414,23 @@ def _as_list(d): v2_list = _as_list(v2_data) if not (len(cli_list) == len(v1_list) == len(v2_list)): + # Collect sample IDs from each list for debugging + def _ids(lst, limit=5): + ids = [] + for item in lst[:limit]: + if isinstance(item, dict): + ids.append( + item.get(id_field, item.get("uuid", + item.get("id", "?"))) + ) + return ids + self._finding( "error", "count_mismatch", operation, cli=len(cli_list), v1=len(v1_list), v2=len(v2_list), + cli_ids=_ids(cli_list), + v1_ids=_ids(v1_list), + v2_ids=_ids(v2_list), ) # Build lookup by id_field @@ -343,10 +461,32 @@ def _by_id(lst): def _audit_cluster_list(self): self.logger.info("[audit] cluster.list") - cli = self._run_cli("cluster list --json") + cli_result = self._run_cli("cluster list --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_request(api_url="/cluster") v2_status, v2_body = self.v2.list_clusters() + # Log API calls for debugging + self.logger.info( + f" CLI: {cli_result['command']} → data={'yes' if cli else 'None'}" + f"{' stderr=' + cli_result['stderr'][:200] if cli_result['stderr'] else ''}" + ) + self.logger.info(f" v1: GET /cluster → {type(v1).__name__}") + self.logger.info(f" v2: GET /clusters → status={v2_status}") + + self._check_interface_data( + "cluster.list", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "cluster.list", "v1", self._normalize_v1(v1), + api_call="GET /api/v1/cluster", + ) + self._check_interface_data( + "cluster.list", "v2", v2_body, + api_call="GET /api/v2/clusters", http_status=v2_status, + ) + self._compare_lists( "cluster.list", cli, self._normalize_v1(v1), v2_body, @@ -357,10 +497,24 @@ def _audit_cluster_list(self): def _audit_cluster_get(self): self.logger.info("[audit] cluster.get") cid = self.cluster_id - cli = self._run_cli(f"cluster get {cid} --json") + cli_result = self._run_cli(f"cluster get {cid} --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_request(api_url=f"/cluster/{cid}") v2_status, v2_body = self.v2.get_cluster(cid) + self._check_interface_data( + "cluster.get", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "cluster.get", "v1", self._normalize_v1(v1), + api_call=f"GET /api/v1/cluster/{cid}", + ) + self._check_interface_data( + "cluster.get", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{cid}", http_status=v2_status, + ) + self._compare_dicts( "cluster.get", self._normalize_cli(cli), @@ -373,10 +527,25 @@ def _audit_cluster_get(self): def _audit_cluster_capacity(self): self.logger.info("[audit] cluster.capacity") cid = self.cluster_id - cli = self._run_cli(f"cluster get-capacity {cid} --json") + cli_result = self._run_cli(f"cluster get-capacity {cid} --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_cluster_capacity() v2_status, v2_body = self.v2.get_cluster_capacity(cid) + self._check_interface_data( + "cluster.capacity", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "cluster.capacity", "v1", self._normalize_v1(v1), + api_call=f"GET /api/v1/cluster/capacity/{cid}", + ) + self._check_interface_data( + "cluster.capacity", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{cid}/capacity", + http_status=v2_status, + ) + self._compare_dicts( "cluster.capacity", self._normalize_cli(cli), @@ -388,10 +557,25 @@ def _audit_cluster_capacity(self): def _audit_cluster_iostats(self): self.logger.info("[audit] cluster.iostats") cid = self.cluster_id - cli = self._run_cli(f"cluster get-io-stats {cid} --json") + cli_result = self._run_cli(f"cluster get-io-stats {cid} --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_io_stats(cid) v2_status, v2_body = self.v2.get_cluster_iostats(cid) + self._check_interface_data( + "cluster.iostats", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "cluster.iostats", "v1", self._normalize_v1(v1), + api_call=f"GET /api/v1/cluster/iostats/{cid}", + ) + self._check_interface_data( + "cluster.iostats", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{cid}/iostats", + http_status=v2_status, + ) + # IO stats are time-varying, so only compare field presence self._compare_dicts( "cluster.iostats", @@ -403,7 +587,8 @@ def _audit_cluster_iostats(self): def _audit_cluster_logs(self): self.logger.info("[audit] cluster.logs") cid = self.cluster_id - cli = self._run_cli(f"cluster get-logs {cid} --json --limit 5") + cli_result = self._run_cli(f"cluster get-logs {cid} --json --limit 5") + cli = cli_result["data"] v1 = self.sbcli_utils.get_cluster_logs(cluster_id=cid) v2_status, v2_body = self.v2.get_cluster_logs(cid, limit=5) @@ -411,17 +596,41 @@ def _audit_cluster_logs(self): cli_norm = self._normalize_cli(cli) v1_norm = self._normalize_v1(v1) - for iface, data in [("cli", cli_norm), ("v1", v1_norm), ("v2", v2_body)]: - if data is None: - self._finding("warning", "interface_error", "cluster.logs", - interface=iface, detail="returned None") + self._check_interface_data( + "cluster.logs", "cli", cli_norm, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "cluster.logs", "v1", v1_norm, + api_call=f"GET /api/v1/cluster/logs/{cid}", + ) + self._check_interface_data( + "cluster.logs", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{cid}/logs?limit=5", + http_status=v2_status, + ) def _audit_node_list(self): self.logger.info("[audit] node.list") - cli = self._run_cli("sn list --json") + cli_result = self._run_cli("sn list --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_storage_nodes() v2_status, v2_body = self.v2.list_nodes() + self._check_interface_data( + "node.list", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "node.list", "v1", self._normalize_v1(v1), + api_call="GET /api/v1/storagenode", + ) + self._check_interface_data( + "node.list", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{self.cluster_id}/storage-nodes", + http_status=v2_status, + ) + self._compare_lists( "node.list", cli, self._normalize_v1(v1), v2_body, @@ -432,15 +641,30 @@ def _audit_node_list(self): def _audit_node_get(self): self.logger.info("[audit] node.get") if not self.sn_nodes: - self._finding("info", "not_tested", "node.get", + self._finding("error", "not_tested", "node.get", detail="no storage nodes available") return node_id = self.sn_nodes[0] - cli = self._run_cli(f"sn get {node_id} --json") + cli_result = self._run_cli(f"sn get {node_id} --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_storage_node_details(node_id) v2_status, v2_body = self.v2.get_node(node_id) + self._check_interface_data( + "node.get", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "node.get", "v1", self._normalize_v1(v1), + api_call=f"GET /api/v1/storagenode/{node_id}", + ) + self._check_interface_data( + "node.get", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{self.cluster_id}/storage-nodes/{node_id}", + http_status=v2_status, + ) + self._compare_dicts( "node.get", self._normalize_cli(cli), @@ -453,15 +677,30 @@ def _audit_node_get(self): def _audit_node_capacity(self): self.logger.info("[audit] node.capacity") if not self.sn_nodes: - self._finding("info", "not_tested", "node.capacity", + self._finding("error", "not_tested", "node.capacity", detail="no storage nodes available") return node_id = self.sn_nodes[0] - cli = self._run_cli(f"sn get-capacity {node_id} --json") + cli_result = self._run_cli(f"sn get-capacity {node_id} --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_request(api_url=f"/storagenode/capacity/{node_id}") v2_status, v2_body = self.v2.get_node_capacity(node_id) + self._check_interface_data( + "node.capacity", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "node.capacity", "v1", self._normalize_v1(v1), + api_call=f"GET /api/v1/storagenode/capacity/{node_id}", + ) + self._check_interface_data( + "node.capacity", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{self.cluster_id}/storage-nodes/{node_id}/capacity", + http_status=v2_status, + ) + self._compare_dicts( "node.capacity", self._normalize_cli(cli), @@ -472,33 +711,58 @@ def _audit_node_capacity(self): def _audit_node_ports(self): self.logger.info("[audit] node.ports") if not self.sn_nodes: - self._finding("info", "not_tested", "node.ports", + self._finding("error", "not_tested", "node.ports", detail="no storage nodes available") return node_id = self.sn_nodes[0] - cli = self._run_cli(f"sn port-list {node_id} --json") + cli_result = self._run_cli(f"sn port-list {node_id} --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_request(api_url=f"/storagenode/port/{node_id}") v2_status, v2_body = self.v2.list_node_nics(node_id) - # Just check that all return data - for iface, data in [("cli", cli), ("v1", v1), ("v2", v2_body)]: - if data is None: - self._finding("warning", "interface_error", "node.ports", - interface=iface, detail="returned None") + # Check that all return data + self._check_interface_data( + "node.ports", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "node.ports", "v1", v1, + api_call=f"GET /api/v1/storagenode/port/{node_id}", + ) + self._check_interface_data( + "node.ports", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{self.cluster_id}/storage-nodes/{node_id}/nics", + http_status=v2_status, + ) def _audit_device_list(self): self.logger.info("[audit] device.list") if not self.sn_nodes: - self._finding("info", "not_tested", "device.list", + self._finding("error", "not_tested", "device.list", detail="no storage nodes available") return node_id = self.sn_nodes[0] - cli = self._run_cli(f"sn list-devices {node_id} --json") + cli_result = self._run_cli(f"sn list-devices {node_id} --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_device_details(node_id) v2_status, v2_body = self.v2.list_devices(node_id) + self._check_interface_data( + "device.list", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "device.list", "v1", self._normalize_v1(v1), + api_call=f"GET /api/v1/device/{node_id}", + ) + self._check_interface_data( + "device.list", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{self.cluster_id}/storage-nodes/{node_id}/devices", + http_status=v2_status, + ) + self._compare_lists( "device.list", cli, self._normalize_v1(v1), v2_body, @@ -509,7 +773,7 @@ def _audit_device_list(self): def _audit_device_get(self): self.logger.info("[audit] device.get") if not self.sn_nodes: - self._finding("info", "not_tested", "device.get", + self._finding("error", "not_tested", "device.get", detail="no storage nodes available") return @@ -518,15 +782,30 @@ def _audit_device_get(self): v1_devices = self.sbcli_utils.get_device_details(node_id) dev_list = self._normalize_v1(v1_devices) if not dev_list or not isinstance(dev_list, list) or len(dev_list) == 0: - self._finding("info", "not_tested", "device.get", + self._finding("error", "not_tested", "device.get", detail="no devices on first node") return device_id = dev_list[0].get("uuid", dev_list[0].get("id")) - cli = self._run_cli(f"sn get-device {device_id} --json") + cli_result = self._run_cli(f"sn get-device {device_id} --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_request(api_url=f"/device/{device_id}") v2_status, v2_body = self.v2.get_device(node_id, device_id) + self._check_interface_data( + "device.get", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "device.get", "v1", self._normalize_v1(v1), + api_call=f"GET /api/v1/device/{device_id}", + ) + self._check_interface_data( + "device.get", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{self.cluster_id}/storage-nodes/{node_id}/devices/{device_id}", + http_status=v2_status, + ) + self._compare_dicts( "device.get", self._normalize_cli(cli), @@ -538,10 +817,25 @@ def _audit_device_get(self): def _audit_pool_list(self): self.logger.info("[audit] pool.list") - cli = self._run_cli("pool list --json") + cli_result = self._run_cli("pool list --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_request(api_url="/pool") v2_status, v2_body = self.v2.list_pools() + self._check_interface_data( + "pool.list", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "pool.list", "v1", self._normalize_v1(v1), + api_call="GET /api/v1/pool", + ) + self._check_interface_data( + "pool.list", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{self.cluster_id}/storage-pools", + http_status=v2_status, + ) + self._compare_lists( "pool.list", cli, self._normalize_v1(v1), v2_body, @@ -553,17 +847,32 @@ def _audit_pool_get(self): self.logger.info("[audit] pool.get") pools = self.sbcli_utils.list_storage_pools() if not pools: - self._finding("info", "not_tested", "pool.get", - detail="no pools available") + self._finding("error", "not_tested", "pool.get", + detail="no pools available — pool creation may have failed") return pool_name = list(pools.keys())[0] pool_id = pools[pool_name] - cli = self._run_cli(f"pool get {pool_id} --json") + cli_result = self._run_cli(f"pool get {pool_id} --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_pool_by_id(pool_id) v2_status, v2_body = self.v2.get_pool(pool_id) + self._check_interface_data( + "pool.get", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "pool.get", "v1", self._normalize_v1(v1), + api_call=f"GET /api/v1/pool/{pool_id}", + ) + self._check_interface_data( + "pool.get", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{self.cluster_id}/storage-pools/{pool_id}", + http_status=v2_status, + ) + self._compare_dicts( "pool.get", self._normalize_cli(cli), @@ -577,27 +886,52 @@ def _audit_pool_iostats(self): self.logger.info("[audit] pool.iostats") pools = self.sbcli_utils.list_storage_pools() if not pools: - self._finding("info", "not_tested", "pool.iostats", - detail="no pools available") + self._finding("error", "not_tested", "pool.iostats", + detail="no pools available — pool creation may have failed") return pool_id = list(pools.values())[0] - cli = self._run_cli(f"pool get-io-stats {pool_id} --json") + cli_result = self._run_cli(f"pool get-io-stats {pool_id} --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_request(api_url=f"/pool/iostats/{pool_id}") v2_status, v2_body = self.v2.get_pool_iostats(pool_id) # IO stats are time-varying; just check field presence - for iface, data in [("cli", cli), ("v1", v1), ("v2", v2_body)]: - if data is None: - self._finding("warning", "interface_error", "pool.iostats", - interface=iface, detail="returned None") + self._check_interface_data( + "pool.iostats", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "pool.iostats", "v1", v1, + api_call=f"GET /api/v1/pool/iostats/{pool_id}", + ) + self._check_interface_data( + "pool.iostats", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{self.cluster_id}/storage-pools/{pool_id}/iostats", + http_status=v2_status, + ) def _audit_mgmt_list(self): self.logger.info("[audit] mgmt.list") - cli = self._run_cli("cp list --json") + cli_result = self._run_cli("cp list --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_management_nodes() v2_status, v2_body = self.v2.list_management_nodes() + self._check_interface_data( + "mgmt.list", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "mgmt.list", "v1", self._normalize_v1(v1), + api_call="GET /api/v1/mgmtnode", + ) + self._check_interface_data( + "mgmt.list", "v2", v2_body, + api_call="GET /api/v2/management-nodes", + http_status=v2_status, + ) + self._compare_lists( "mgmt.list", cli, self._normalize_v1(v1), v2_body, @@ -609,14 +943,15 @@ def _audit_mgmt_list(self): def _audit_pool_crud(self): self.logger.info("[audit] pool.crud") - pool_name = "parity_audit_pool" + pool_name = "parity_crud_pool" # Create via v1 self.sbcli_utils.add_storage_pool(pool_name) - sleep_n_sec(3) + sleep_n_sec(5) # List via all 3 and check pool appears - cli = self._run_cli("pool list --json") + cli_result = self._run_cli("pool list --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_request(api_url="/pool") v2_status, v2_body = self.v2.list_pools() @@ -643,11 +978,18 @@ def _find_pool(lst, name): ignore_fields={"updated_at", "created_at", "secret"}, ) else: - for iface, p in [("cli", cli_pool), ("v1", v1_pool), ("v2", v2_pool)]: + for iface, p, api_call in [ + ("cli", cli_pool, cli_result["command"]), + ("v1", v1_pool, "GET /api/v1/pool"), + ("v2", v2_pool, f"GET /api/v2/clusters/{self.cluster_id}/storage-pools"), + ]: if p is None: - self._finding("error", "interface_error", "pool.crud", - interface=iface, - detail=f"pool '{pool_name}' not found after create") + self._finding( + "error", "interface_error", "pool.crud", + interface=iface, + detail=f"pool '{pool_name}' not found after create", + api_call=api_call, + ) # Cleanup try: @@ -659,11 +1001,11 @@ def _find_pool(lst, name): def _audit_volume_crud(self): self.logger.info("[audit] volume.crud") - # Ensure a pool exists + # Use the audit pool created at Phase 0 pools = self.sbcli_utils.list_storage_pools() if not pools: - self._finding("info", "not_tested", "volume.crud", - detail="no pools available") + self._finding("error", "not_tested", "volume.crud", + detail="no pools available — pool creation failed") return pool_name = list(pools.keys())[0] @@ -684,14 +1026,30 @@ def _audit_volume_crud(self): if not vol_id: self._finding("error", "interface_error", "volume.crud", interface="v1", - detail=f"volume '{vol_name}' not found after create") + detail=f"volume '{vol_name}' not found after create", + api_call=f"POST /api/v1/lvol (name={vol_name}, pool={pool_name}, size=1G)") return # Get via all 3 - cli = self._run_cli(f"lvol get {vol_id} --json") + cli_result = self._run_cli(f"lvol get {vol_id} --json") + cli = cli_result["data"] v1 = self.sbcli_utils.get_lvol_details(vol_id) v2_status, v2_body = self.v2.get_volume(pool_id, vol_id) + self._check_interface_data( + "volume.get", "cli", cli, + api_call=cli_result["command"], raw_response=cli_result, + ) + self._check_interface_data( + "volume.get", "v1", self._normalize_v1(v1), + api_call=f"GET /api/v1/lvol/{vol_id}", + ) + self._check_interface_data( + "volume.get", "v2", v2_body, + api_call=f"GET /api/v2/clusters/{self.cluster_id}/storage-pools/{pool_id}/volumes/{vol_id}", + http_status=v2_status, + ) + self._compare_dicts( "volume.get", self._normalize_cli(cli), @@ -704,7 +1062,8 @@ def _audit_volume_crud(self): # List volumes via v2 and check count v2_list_status, v2_list_body = self.v2.list_volumes(pool_id) v1_list = self.sbcli_utils.list_lvols() - cli_list = self._run_cli("lvol list --json") + cli_list_result = self._run_cli("lvol list --json") + cli_list = cli_list_result["data"] # Just check the volume appears in all three def _has_vol(data, name): @@ -717,11 +1076,17 @@ def _has_vol(data, name): ) return False - for iface, data in [("cli", cli_list), ("v1", v1_list), ("v2", v2_list_body)]: + for iface, data, api_call in [ + ("cli", cli_list, cli_list_result["command"]), + ("v1", v1_list, "GET /api/v1/lvol"), + ("v2", v2_list_body, + f"GET /api/v2/clusters/{self.cluster_id}/storage-pools/{pool_id}/volumes"), + ]: if not _has_vol(data, vol_name): - self._finding("warning", "interface_error", "volume.list_after_create", + self._finding("error", "interface_error", "volume.list_after_create", interface=iface, - detail=f"volume '{vol_name}' not in list") + detail=f"volume '{vol_name}' not in list", + api_call=api_call) # Cleanup try: @@ -736,8 +1101,8 @@ def _audit_snapshot_crud(self): # Need a volume to snapshot pools = self.sbcli_utils.list_storage_pools() if not pools: - self._finding("info", "not_tested", "snapshot.crud", - detail="no pools available") + self._finding("error", "not_tested", "snapshot.crud", + detail="no pools available — pool creation failed") return pool_name = list(pools.keys())[0] @@ -754,7 +1119,8 @@ def _audit_snapshot_crud(self): if not vol_id: self._finding("error", "interface_error", "snapshot.crud", interface="v1", - detail=f"could not create volume '{vol_name}'") + detail=f"could not create volume '{vol_name}'", + api_call=f"POST /api/v1/lvol (name={vol_name}, pool={pool_name}, size=1G)") return # Create snapshot via v1 @@ -762,7 +1128,8 @@ def _audit_snapshot_crud(self): sleep_n_sec(5) # List snapshots via all 3 - cli = self._run_cli("snapshot list --json") + cli_result = self._run_cli("snapshot list --json") + cli = cli_result["data"] v1 = self.sbcli_utils.list_snapshots() v2_status, v2_body = self.v2.list_snapshots(pool_id) @@ -777,11 +1144,17 @@ def _has_snap(data, name): ) return False - for iface, data in [("cli", cli), ("v1", v1), ("v2", v2_body)]: + for iface, data, api_call in [ + ("cli", cli, cli_result["command"]), + ("v1", v1, "GET /api/v1/snapshot"), + ("v2", v2_body, + f"GET /api/v2/clusters/{self.cluster_id}/storage-pools/{pool_id}/snapshots"), + ]: if not _has_snap(data, snap_name): - self._finding("warning", "interface_error", "snapshot.list_after_create", + self._finding("error", "interface_error", "snapshot.list_after_create", interface=iface, - detail=f"snapshot '{snap_name}' not in list") + detail=f"snapshot '{snap_name}' not in list", + api_call=api_call) # Cleanup try: diff --git a/e2e/e2e_tests/upgrade_tests/k8s_major_upgrade.py b/e2e/e2e_tests/upgrade_tests/k8s_major_upgrade.py index 422cd1e9b6..7205d65eb7 100644 --- a/e2e/e2e_tests/upgrade_tests/k8s_major_upgrade.py +++ b/e2e/e2e_tests/upgrade_tests/k8s_major_upgrade.py @@ -5,7 +5,7 @@ **R25 → R26 (maintenance window)**: Full Helm-to-Operator migration following the production upgrade guide: - 1. Annotate FDB resources with ``helm.sh/resource-policy: keep`` + 1. Patch Helm release secret to add ``helm.sh/resource-policy: keep`` to FDB resources 2. Shut down all storage nodes (suspend + shutdown) 3. Uninstall old Helm chart(s) 4. Create upgrade secret with existing cluster UUID/secret @@ -28,6 +28,7 @@ from __future__ import annotations +import json import os import random import string @@ -99,6 +100,7 @@ def _rand_seq(length: int = 6) -> str: ("rolebinding", "simplyblock-fdb-manager-rolebinding"), ("clusterrolebinding", "simplyblock-fdb-manager-clusterrolebinding"), ("foundationdbcluster", "simplyblock-fdb-cluster"), + ("configmap", "simplyblock-fdb-cluster-config"), ] # Default CR names matching the k8s-native-e2e.yaml workflow @@ -182,6 +184,7 @@ def __init__(self, **kwargs): self.pvc_details: dict[str, dict] = {} self.snapshot_details: dict[str, dict] = {} self.clone_details: dict[str, dict] = {} + self.pre_upgrade_checksums: dict[str, dict] = {} self.logger.info( f"K8s native upgrade: {self.base_version} -> {self.target_version} " @@ -281,7 +284,15 @@ def _is_maintenance_window_upgrade(self) -> bool: # ── FIO config ───────────────────────────────────────────────────────────── - def _build_fio_config(self, name: str, runtime: int = None) -> tuple[str, str]: + def _build_fio_config( + self, name: str, runtime: int = None, + ) -> tuple[str, str, dict]: + """Build FIO main + warmup configs. + + Returns ``(main_config, warmup_config, metadata)`` where *metadata* + contains ``run_id``, ``randseed``, ``bs``, and ``fio_size`` so the + caller can later reconstruct a verify-only config for the same files. + """ bs = f"{2 ** random.randint(2, 7)}k" run_id = _rand_seq(6) randseed = random.randint(1, 2**63) @@ -330,7 +341,38 @@ def _build_fio_config(self, name: str, runtime: int = None) -> tuple[str, str]: f"[job1]\n" ) - return main_config, warmup_config + metadata = { + "run_id": run_id, + "randseed": randseed, + "bs": bs, + "fio_size": self.fio_size, + "num_jobs": self.fio_num_jobs, + } + + return main_config, warmup_config, metadata + + def _build_verify_only_fio_config(self, name: str, meta: dict) -> str: + """Build a verify-only FIO config that replays the exact files/seed + from a previous write run, confirming data integrity without writing.""" + return ( + f"[global]\n" + f"name={name}-verify\n" + f"filename_format=/spdkvol/fio-{meta['run_id']}.$jobnum\n" + f"rw=read\n" + f"bs={meta['bs']}\n" + f"iodepth=1\n" + f"direct=1\n" + f"ioengine=libaio\n" + f"size={meta['fio_size']}\n" + f"numjobs={meta['num_jobs']}\n" + f"verify=md5\n" + f"verify_only\n" + f"verify_dump=1\n" + f"verify_fatal=1\n" + f"randseed={meta['randseed']}\n" + f"\n" + f"[job1]\n" + ) def _save_fio_pod_logs(self, job_name: str, resource_name: str): try: @@ -391,7 +433,10 @@ def _create_pvcs_with_fio(self, count: int, runtime: int = None): } for pvc_name, detail in self.pvc_details.items(): - fio_config, warmup_config = self._build_fio_config(pvc_name, runtime=runtime) + fio_config, warmup_config, fio_meta = self._build_fio_config( + pvc_name, runtime=runtime, + ) + detail["fio_meta"] = fio_meta avoid = self.k8s_utils.get_pvc_primary_k8s_node(pvc_name, self.sbcli_utils) self.k8s_utils.create_fio_job( job_name=detail["job_name"], @@ -406,8 +451,8 @@ def _create_pvcs_with_fio(self, count: int, runtime: int = None): self.k8s_utils.log_fio_pvc_mapping(self.pvc_details) - def _create_snapshots_and_clones(self, runtime: int = None): - """Create snapshots + clones with FIO on each clone.""" + def _create_snapshots_and_clones(self, runtime: int = None, skip_clone_fio: bool = False): + """Create snapshots + clones, optionally with FIO on each clone.""" for pvc_name, detail in self.pvc_details.items(): snap_name = f"snap-{pvc_name}" clone_name = f"clone-{pvc_name}" @@ -431,14 +476,17 @@ def _create_snapshots_and_clones(self, runtime: int = None): ) self.k8s_utils.wait_pvc_bound(clone_name, timeout=300) - fio_config, warmup_config = self._build_fio_config(clone_name, runtime=runtime) - avoid = self.k8s_utils.get_pvc_primary_k8s_node(clone_name, self.sbcli_utils) - self.k8s_utils.create_fio_job( - job_name=clone_job, pvc_name=clone_name, - configmap_name=clone_cm, fio_config=fio_config, - image=self.FIO_IMAGE, avoid_node=avoid, - warmup_config=warmup_config, - ) + if not skip_clone_fio: + fio_config, warmup_config, _clone_meta = self._build_fio_config( + clone_name, runtime=runtime, + ) + avoid = self.k8s_utils.get_pvc_primary_k8s_node(clone_name, self.sbcli_utils) + self.k8s_utils.create_fio_job( + job_name=clone_job, pvc_name=clone_name, + configmap_name=clone_cm, fio_config=fio_config, + image=self.FIO_IMAGE, avoid_node=avoid, + warmup_config=warmup_config, + ) self.clone_details[clone_name] = { "snap_name": snap_name, "job_name": clone_job, @@ -461,6 +509,160 @@ def _validate_all_fio(self, timeout: int): self._save_fio_pod_logs(detail["job_name"], clone_name) self.k8s_utils.validate_fio_job(detail["job_name"], timeout=timeout) + def _cleanup_fio_jobs_only(self): + """Delete FIO jobs and configmaps but leave PVCs/snapshots/clones intact. + + Unlike ``k8s_utils.cleanup_stale_fio_resources()`` which also removes + clone PVCs, snapshots, and test PVCs, this only removes the FIO + workload resources so PVCs are freed for utility pod mounting. + """ + ns = self.k8s_utils.namespace + cmds = [ + # Delete FIO jobs by label + f"kubectl delete jobs -n {ns} -l app=fio-benchmark --ignore-not-found", + # Delete FIO configmaps + ( + f"kubectl get configmaps -n {ns} --no-headers " + f"-o custom-columns=NAME:.metadata.name 2>/dev/null " + f"| grep -E '^(fiocfg-|fio-cfg-)' " + f"| xargs -r kubectl delete configmap -n {ns} --ignore-not-found" + ), + ] + for cmd in cmds: + try: + self.k8s_utils._exec_kubectl(cmd) + except Exception as exc: + self.logger.warning(f"FIO job cleanup step failed: {exc}") + self.logger.info("FIO jobs and configmaps cleaned up (PVCs preserved)") + + def _capture_pvc_checksums(self, pvc_names: list[str]) -> dict[str, dict]: + """Capture MD5 checksums for all files on the given PVCs. + + Returns ``{pvc_name: {filepath: md5hash, ...}, ...}``. + """ + all_checksums = {} + for pvc_name in pvc_names: + pod_name = f"cksum-{pvc_name}"[:63] + self.logger.info(f"Capturing checksums for PVC {pvc_name}") + try: + self.k8s_utils.create_utility_pod(pod_name, pvc_name) + self.k8s_utils.wait_pod_running(pod_name) + files = self.k8s_utils.find_files_in_pvc(pod_name) + if files: + checksums = self.k8s_utils.generate_checksums_in_pvc( + pod_name, files, + ) + all_checksums[pvc_name] = checksums + self.logger.info( + f" {pvc_name}: captured {len(checksums)} file checksums" + ) + else: + self.logger.warning(f" {pvc_name}: no files found on volume") + all_checksums[pvc_name] = {} + except Exception as exc: + self.logger.warning( + f" Failed to capture checksums for {pvc_name}: {exc}" + ) + all_checksums[pvc_name] = {} + finally: + try: + self.k8s_utils.delete_pod(pod_name, wait=True) + except Exception: + pass + return all_checksums + + def _verify_pvc_checksums( + self, pre_checksums: dict[str, dict], label: str = "post-upgrade", + ): + """Verify that current PVC data matches previously captured checksums. + + Raises ``AssertionError`` if any checksum mismatch is found. + """ + mismatches = [] + for pvc_name, expected in pre_checksums.items(): + if not expected: + self.logger.warning( + f" Skipping {pvc_name} — no pre-upgrade checksums captured" + ) + continue + + pod_name = f"verify-cksum-{pvc_name}"[:63] + self.logger.info(f"Verifying checksums for PVC {pvc_name} ({label})") + try: + self.k8s_utils.create_utility_pod(pod_name, pvc_name) + self.k8s_utils.wait_pod_running(pod_name) + actual = self.k8s_utils.generate_checksums_in_pvc( + pod_name, list(expected.keys()), + ) + for filepath, exp_hash in expected.items(): + act_hash = actual.get(filepath) + if act_hash != exp_hash: + msg = ( + f"MISMATCH {pvc_name}:{filepath} " + f"expected={exp_hash} actual={act_hash}" + ) + self.logger.error(msg) + mismatches.append(msg) + else: + self.logger.info( + f" {filepath}: {exp_hash} ✓" + ) + except Exception as exc: + self.logger.warning( + f" Failed to verify checksums for {pvc_name}: {exc}" + ) + finally: + try: + self.k8s_utils.delete_pod(pod_name, wait=True) + except Exception: + pass + + if mismatches: + raise AssertionError( + f"Data integrity check failed ({label}): " + + "; ".join(mismatches) + ) + self.logger.info(f"All checksums verified ({label})") + + def _run_fio_on_clones(self, runtime: int = 60): + """Run FIO on clone PVCs (after cleaning parent data from clone).""" + clone_jobs = [] + for clone_name, detail in self.clone_details.items(): + clone_job = detail["job_name"] + clone_cm = detail["configmap_name"] + + fio_config, warmup_config, _meta = self._build_fio_config( + clone_name, runtime=runtime, + ) + avoid = self.k8s_utils.get_pvc_primary_k8s_node( + clone_name, self.sbcli_utils, + ) + self.k8s_utils.create_fio_job( + job_name=clone_job, pvc_name=clone_name, + configmap_name=clone_cm, fio_config=fio_config, + image=self.FIO_IMAGE, avoid_node=avoid, + warmup_config=warmup_config, + ) + clone_jobs.append((clone_job, clone_name)) + sleep_n_sec(5) + + # Wait for clone FIO with tolerance + fio_timeout = runtime + 240 # runtime + 4 min buffer + for job_name, clone_name in clone_jobs: + try: + self._save_fio_pod_logs(job_name, clone_name) + self.k8s_utils.validate_fio_job(job_name, timeout=fio_timeout) + self.logger.info(f"Clone FIO completed: {clone_name}") + except Exception as exc: + self.logger.warning( + f"Clone FIO did not complete for {clone_name}: {exc}. " + "Continuing — non-fatal." + ) + + # Clean up clone FIO jobs (preserve clone PVCs for checksums) + self._cleanup_fio_jobs_only() + sleep_n_sec(5) + def _run_post_upgrade_verification(self): """Create new PVC + FIO + snapshot + clone post-upgrade.""" self.logger.info("Post-upgrade: Creating new PVC to verify provisioning") @@ -474,7 +676,7 @@ def _run_post_upgrade_verification(self): ) self.k8s_utils.wait_pvc_bound(post_pvc, timeout=300) - fio_cfg, warmup_cfg = self._build_fio_config(post_pvc, runtime=120) + fio_cfg, warmup_cfg, _post_meta = self._build_fio_config(post_pvc, runtime=120) avoid = self.k8s_utils.get_pvc_primary_k8s_node(post_pvc, self.sbcli_utils) self.k8s_utils.create_fio_job( job_name=post_job, pvc_name=post_pvc, configmap_name=post_cm, @@ -500,7 +702,7 @@ def _run_post_upgrade_verification(self): ) self.k8s_utils.wait_pvc_bound(post_clone, timeout=300) - clone_fio, clone_warmup = self._build_fio_config(post_clone, runtime=120) + clone_fio, clone_warmup, _clone_meta = self._build_fio_config(post_clone, runtime=120) self.k8s_utils.create_fio_job( job_name=post_clone_job, pvc_name=post_clone, configmap_name=post_clone_cm, fio_config=clone_fio, @@ -512,15 +714,409 @@ def _run_post_upgrade_verification(self): self._save_fio_pod_logs(post_clone_job, post_clone) self.k8s_utils.validate_fio_job(post_clone_job, timeout=600) - def _assert_all_nodes_healthy(self): + def _assert_all_nodes_healthy(self, timeout=120, interval=10): + """Assert all storage nodes are online with health_check=True. + + After a maintenance upgrade the health_check field may remain + None or False for a short period while the monitoring loop + catches up. Retry up to *timeout* seconds before failing. + """ + from time import time as _now + + deadline = _now() + timeout + while True: + storage_node_list = self.sbcli_utils.get_storage_nodes()["results"] + unhealthy = [] + for node in storage_node_list: + if node["status"] != "online": + unhealthy.append( + f"Node {node['id']} not online (status={node['status']})" + ) + elif not node.get("health_check", False): + unhealthy.append( + f"Node {node['id']} health_check={node.get('health_check')}" + ) + + if not unhealthy: + self.logger.info("All storage nodes online and healthy") + return + + if _now() >= deadline: + for msg in unhealthy: + self.logger.error(msg) + raise AssertionError( + f"{len(unhealthy)} node(s) unhealthy after {timeout}s: " + + "; ".join(unhealthy) + ) + + self.logger.info( + f"Waiting for {len(unhealthy)} node(s) to become healthy, " + f"retrying in {interval}s …" + ) + sleep_n_sec(interval) + + # ── Phase 2.7: Capture pre-upgrade state ────────────────────────────────── + + def _capture_pre_upgrade_state(self): + """Log complete cluster state before starting the upgrade (Phase 2.7).""" + self.logger.info("=" * 40 + " PRE-UPGRADE STATE CAPTURE " + "=" * 40) + + # Cluster + self.logger.info(f"Cluster UUID: {self.cluster_id}") + self.logger.info("Cluster Secret: ***") + + # Storage nodes + storage_nodes = self.sbcli_utils.get_storage_nodes()["results"] + self.logger.info(f"Storage nodes ({len(storage_nodes)}):") + for node in storage_nodes: + self.logger.info( + f" Node {node['id']} — status={node['status']}, " + f"hostname={node.get('hostname', 'N/A')}" + ) + + # Pools + try: + pools = self.sbcli_utils.list_storage_pools() + self.logger.info(f"Storage pools: {pools}") + except Exception as e: + self.logger.warning(f"Could not list pools: {e}") + + # PVCs + self.logger.info(f"Pre-upgrade PVCs ({len(self.pvc_details)}):") + for pvc_name, detail in self.pvc_details.items(): + pv_name = self.k8s_utils.get_pvc_pv_name(pvc_name) or "N/A" + self.logger.info( + f" PVC {pvc_name} -> PV {pv_name} " + f"(SC={detail['storage_class']}, fs={detail['fs_type']})" + ) + + # Snapshots + self.logger.info(f"Pre-upgrade snapshots ({len(self.snapshot_details)}):") + for snap_name, detail in self.snapshot_details.items(): + self.logger.info(f" Snapshot {snap_name} (source PVC: {detail['pvc_name']})") + + # Clones + self.logger.info(f"Pre-upgrade clones ({len(self.clone_details)}):") + for clone_name, detail in self.clone_details.items(): + self.logger.info( + f" Clone {clone_name} (from snapshot: {detail['snap_name']})" + ) + + # Lvols + try: + self.sbcli_utils.list_lvols() + except Exception as e: + self.logger.warning(f"Could not list lvols: {e}") + + self.logger.info("=" * 40 + " END PRE-UPGRADE STATE " + "=" * 40) + + # ── Phase 4.1–4.3: Verify old data post-upgrade ────────────────────────── + + def _verify_old_data_post_upgrade(self): + """Verify pre-upgrade data survives the upgrade (Phases 4.1–4.3). + + 4.1 — FIO verify-only on old PVCs (confirms data integrity) + 4.2 — Fresh randrw FIO on old PVCs (confirms IO works) + 4.3 — New snapshots + clones on old PVCs post-upgrade + """ + self.logger.info( + "Post-upgrade Phase 4.1: Verify old data integrity (FIO verify-only)" + ) + + # 4.1 — Verify-only FIO on each pre-upgrade PVC + verify_jobs: list[tuple[str, str]] = [] + for pvc_name, detail in self.pvc_details.items(): + fio_meta = detail.get("fio_meta") + if not fio_meta: + self.logger.warning( + f"No FIO metadata for PVC {pvc_name}, skipping verify-only" + ) + continue + + verify_job = f"verify-{pvc_name}" + verify_cm = f"fio-verify-cfg-{pvc_name}" + + verify_config = self._build_verify_only_fio_config(pvc_name, fio_meta) + avoid = self.k8s_utils.get_pvc_primary_k8s_node( + pvc_name, self.sbcli_utils, + ) + self.k8s_utils.create_fio_job( + job_name=verify_job, pvc_name=pvc_name, + configmap_name=verify_cm, fio_config=verify_config, + image=self.FIO_IMAGE, avoid_node=avoid, + ) + verify_jobs.append((verify_job, pvc_name)) + sleep_n_sec(5) + + for job_name, pvc_name in verify_jobs: + self.logger.info(f"Validating verify-only FIO for PVC: {pvc_name}") + self._save_fio_pod_logs(job_name, f"{pvc_name}-verify") + self.k8s_utils.validate_fio_job(job_name, timeout=600) + + self.logger.info( + "Post-upgrade Phase 4.1 PASSED: All old PVC data verified intact" + ) + + # 4.2 — Fresh randrw FIO on old PVCs + self.logger.info( + "Post-upgrade Phase 4.2: Fresh FIO on old PVCs (confirm IO works)" + ) + fresh_jobs: list[tuple[str, str]] = [] + for pvc_name, detail in self.pvc_details.items(): + fresh_job = f"post-io-{pvc_name}" + fresh_cm = f"fio-post-io-cfg-{pvc_name}" + + fio_config, warmup_config, _meta = self._build_fio_config( + pvc_name, runtime=120, + ) + avoid = self.k8s_utils.get_pvc_primary_k8s_node( + pvc_name, self.sbcli_utils, + ) + self.k8s_utils.create_fio_job( + job_name=fresh_job, pvc_name=pvc_name, + configmap_name=fresh_cm, fio_config=fio_config, + image=self.FIO_IMAGE, avoid_node=avoid, + warmup_config=warmup_config, + ) + fresh_jobs.append((fresh_job, pvc_name)) + sleep_n_sec(5) + + for job_name, pvc_name in fresh_jobs: + self.logger.info(f"Validating fresh FIO for PVC: {pvc_name}") + self._save_fio_pod_logs(job_name, f"{pvc_name}-post-io") + self.k8s_utils.validate_fio_job(job_name, timeout=600) + + self.logger.info( + "Post-upgrade Phase 4.2 PASSED: Fresh IO on old PVCs succeeded" + ) + + # 4.3 — New snapshots + clones on old PVCs + self.logger.info( + "Post-upgrade Phase 4.3: New snapshots and clones on old PVCs" + ) + post_clone_jobs: list[tuple[str, str]] = [] + for pvc_name, detail in self.pvc_details.items(): + post_snap = f"post-snap-{pvc_name}" + post_clone = f"post-clone-{pvc_name}" + post_clone_job = f"fio-{post_clone}" + post_clone_cm = f"fio-cfg-{post_clone}" + + self.k8s_utils.create_volume_snapshot( + name=post_snap, pvc_name=pvc_name, + snapshot_class=self.SNAPSHOT_CLASS_NAME, + ) + self.k8s_utils.wait_volume_snapshot_ready(post_snap, timeout=300) + + clone_sc = detail.get("storage_class", self.STORAGE_CLASS_NAME) + self.k8s_utils.create_clone_pvc( + name=post_clone, size=self.pvc_size, + storage_class=clone_sc, snapshot_name=post_snap, + ) + self.k8s_utils.wait_pvc_bound(post_clone, timeout=300) + + clone_fio, clone_warmup, _meta = self._build_fio_config( + post_clone, runtime=120, + ) + avoid = self.k8s_utils.get_pvc_primary_k8s_node( + post_clone, self.sbcli_utils, + ) + self.k8s_utils.create_fio_job( + job_name=post_clone_job, pvc_name=post_clone, + configmap_name=post_clone_cm, fio_config=clone_fio, + image=self.FIO_IMAGE, avoid_node=avoid, + warmup_config=clone_warmup, + ) + post_clone_jobs.append((post_clone_job, post_clone)) + sleep_n_sec(5) + + for job_name, clone_name in post_clone_jobs: + self.logger.info(f"Validating post-upgrade clone FIO: {clone_name}") + self._save_fio_pod_logs(job_name, clone_name) + self.k8s_utils.validate_fio_job(job_name, timeout=600) + + self.logger.info( + "Post-upgrade Phase 4.3 PASSED: Snapshots + clones on old PVCs work" + ) + + # ── Phase 4.6: Node outage test ─────────────────────────────────────────── + + def _run_node_outage_test(self): + """Verify HA works post-upgrade by shutting down a non-primary node + while FIO is running (Phase 4.6).""" + self.logger.info("Post-upgrade Phase 4.6: Node outage test") + storage_node_list = self.sbcli_utils.get_storage_nodes()["results"] + if len(storage_node_list) < 2: + self.logger.warning( + "Only 1 storage node — skipping node outage test " + "(need at least 2 nodes for HA validation)" + ) + return + + # Create a PVC and start a long FIO job + outage_pvc = f"outage-pvc-{_rand_seq(4)}" + outage_job = f"fio-{outage_pvc}" + outage_cm = f"fio-cfg-{outage_pvc}" + + self.k8s_utils.create_pvc( + name=outage_pvc, size=self.pvc_size, + storage_class=self.STORAGE_CLASS_NAME, + ) + self.k8s_utils.wait_pvc_bound(outage_pvc, timeout=300) + + fio_config, warmup_config, _meta = self._build_fio_config( + outage_pvc, runtime=300, + ) + avoid = self.k8s_utils.get_pvc_primary_k8s_node( + outage_pvc, self.sbcli_utils, + ) + self.k8s_utils.create_fio_job( + job_name=outage_job, pvc_name=outage_pvc, + configmap_name=outage_cm, fio_config=fio_config, + image=self.FIO_IMAGE, avoid_node=avoid, + warmup_config=warmup_config, + ) + + # Wait for FIO to start running + self.logger.info("Waiting for FIO to establish baseline before node outage") + sleep_n_sec(30) + + # Find the primary node for this PVC and pick a different one to shut down + primary_node_id = None + try: + vol_handle = self.k8s_utils.get_pvc_volume_handle(outage_pvc) + if vol_handle: + lvol_id = vol_handle.split(":")[-1] if ":" in vol_handle else vol_handle + lvol_details = self.sbcli_utils.get_lvol_details(lvol_id) + primary_node_id = lvol_details.get("node_id") + except Exception as e: + self.logger.warning(f"Could not determine primary node: {e}") + + # Pick a non-primary node to shut down + victim_node = None for node in storage_node_list: - assert node["status"] == "online", ( - f"Node {node['id']} not online (status={node['status']})" + if node["id"] != primary_node_id and node["status"] == "online": + victim_node = node + break + + if not victim_node: + self.logger.warning( + "Could not find a non-primary node to shut down, " + "skipping node outage test" + ) + self._save_fio_pod_logs(outage_job, outage_pvc) + self.k8s_utils.validate_fio_job(outage_job, timeout=600) + return + + victim_id = victim_node["id"] + self.logger.info( + f"Shutting down non-primary node {victim_id} " + f"(primary={primary_node_id})" + ) + + # Shut down the victim node + try: + self.sbcli_utils.suspend_node(victim_id) + except Exception as e: + self.logger.warning(f"Suspend failed for {victim_id}: {e}") + sleep_n_sec(10) + try: + self.sbcli_utils.shutdown_node(victim_id) + except Exception as e: + self.logger.warning(f"Shutdown failed for {victim_id}: {e}") + + self.sbcli_utils.wait_for_storage_node_status( + node_id=victim_id, + status=["offline", "unavailable"], + timeout=300, + ) + self.logger.info(f"Node {victim_id} is offline, FIO should continue") + + # Verify FIO is still running + sleep_n_sec(30) + + # Restart the victim node + self.logger.info(f"Restarting node {victim_id}") + try: + self.sbcli_utils.restart_node(victim_id) + except Exception as e: + self.logger.warning(f"Restart failed for {victim_id}: {e}") + + self.sbcli_utils.wait_for_storage_node_status( + node_id=victim_id, status="online", timeout=600, + ) + self.logger.info(f"Node {victim_id} is back online") + + self.sbcli_utils.wait_for_cluster_status( + cluster_id=self.cluster_id, status="active", timeout=600, + ) + + # Validate FIO completed successfully + self._save_fio_pod_logs(outage_job, outage_pvc) + self.k8s_utils.validate_fio_job(outage_job, timeout=600) + self.logger.info("Post-upgrade Phase 4.6 PASSED: Node outage test succeeded") + + # ── Phase 4.7: Final checklist ──────────────────────────────────────────── + + def _run_final_checklist(self, is_maintenance_upgrade: bool = False): + """Run the final validation checklist (Phase 4.7).""" + self.logger.info("Post-upgrade Phase 4.7: Final checklist") + + # Cluster active + cluster_details = self.sbcli_utils.wait_for_cluster_status( + cluster_id=self.cluster_id, status="active", timeout=120, + ) + self.logger.info(f" Cluster status: {cluster_details['status']} ✓") + + # All nodes online + self._assert_all_nodes_healthy() + self.logger.info(" All storage nodes online ✓") + + # All PVCs bound + for pvc_name in self.pvc_details: + out, _ = self.k8s_utils._exec_kubectl( + f"kubectl get pvc {pvc_name} -o jsonpath='{{.status.phase}}'" + ) + phase = (out or "").strip().replace("'", "") + assert phase == "Bound", ( + f"PVC {pvc_name} not Bound (phase={phase})" ) - assert node.get("health_check", True), ( - f"Node {node['id']} health check failed" + self.logger.info(f" All {len(self.pvc_details)} pre-upgrade PVCs Bound ✓") + + # Snapshots ready + for snap_name in self.snapshot_details: + out, _ = self.k8s_utils._exec_kubectl( + f"kubectl get volumesnapshot {snap_name} " + f"-o jsonpath='{{.status.readyToUse}}'" ) + ready = (out or "").strip().replace("'", "") + assert ready == "true", ( + f"Snapshot {snap_name} not ready (readyToUse={ready})" + ) + self.logger.info( + f" All {len(self.snapshot_details)} snapshots readyToUse ✓" + ) + + # CR refs patched (R25→R26 only) + if is_maintenance_upgrade: + try: + out, _ = self.k8s_utils._exec_kubectl( + f"kubectl get storagecluster {self.cluster_cr_name} " + f"-n {_NAMESPACE} -o jsonpath='{{.status.uuid}}'" + ) + cr_uuid = (out or "").strip().replace("'", "") + if cr_uuid: + self.logger.info( + f" StorageCluster CR adopted with UUID={cr_uuid} ✓" + ) + else: + self.logger.warning( + " StorageCluster CR UUID not populated in status" + ) + except Exception as e: + self.logger.warning(f" Could not verify CR adoption: {e}") + + self.logger.info("Post-upgrade Phase 4.7 PASSED: Final checklist complete") # ══════════════════════════════════════════════════════════════════════════ # ROLLING UPGRADE (R26+, no maintenance window) @@ -583,6 +1179,14 @@ def _run_rolling_upgrade(self, storage_node_list: list[dict]): actual_pool = self.sbcli_utils.add_storage_pool(pool_name) if actual_pool and actual_pool != pool_name: pool_name = actual_pool + + # Pool CR name must match the existing backend pool name so the + # operator can adopt it during the upgrade. + self.pool_cr_name = pool_name + self.logger.info( + f"Pool CR name set to '{self.pool_cr_name}' (matching backend pool)" + ) + sleep_n_sec(10) self._create_storage_classes(self.cluster_id, pool_name) @@ -592,6 +1196,9 @@ def _run_rolling_upgrade(self, storage_node_list: list[dict]): self.logger.info("Step 4: Creating snapshots and clones") self._create_snapshots_and_clones() + # Phase 2.7: Capture pre-upgrade state + self._capture_pre_upgrade_state() + self.logger.info("Step 5: Waiting 60s for FIO to establish baseline") sleep_n_sec(60) @@ -649,20 +1256,60 @@ def _run_rolling_upgrade(self, storage_node_list: list[dict]): self._validate_all_fio(fio_timeout) self.logger.info("All pre-upgrade FIO jobs validated successfully") - self.logger.info("Step 9: Post-upgrade new PVC verification") + # Phase 4.1–4.3: Verify old data survives the upgrade + self.logger.info("Step 9: Verifying old data integrity post-upgrade") + self._verify_old_data_post_upgrade() + + # Phase 4.4–4.5: New PVC provisioning + snapshot/clone + self.logger.info("Step 10: Post-upgrade new PVC verification") self._run_post_upgrade_verification() + # Phase 4.6: Node outage test + self._run_node_outage_test() + + # Phase 4.7: Final checklist + self._run_final_checklist(is_maintenance_upgrade=False) + # ══════════════════════════════════════════════════════════════════════════ # MAINTENANCE WINDOW UPGRADE (R25→R26) # ══════════════════════════════════════════════════════════════════════════ def _annotate_fdb_keep(self): - """Step 1: Add helm.sh/resource-policy: keep to FDB resources.""" - self.logger.info("Migration Step 1: Annotating FDB resources with keep policy") + """Step 1: Add helm.sh/resource-policy: keep to FDB resources. + + IMPORTANT: ``kubectl annotate`` on live resources does NOT protect + against ``helm uninstall``. Helm reads annotations from its stored + release manifest, not from the live object in etcd. + + Primary approach (Option A): Edit the chart template files on disk + to add ``helm.sh/resource-policy: keep`` annotations, then run + ``helm upgrade --reuse-values`` to persist the annotations into + Helm's stored release manifest. - # First try upgrading the existing release to add the annotation via helm - # (as described in the guide). Fall back to direct annotation if the - # sbcli release doesn't exist (operator-deployed clusters). + Fallback (Option B): If the chart path is not available, patch the + Helm release secret directly (decode base64→gzip→JSON, inject + annotations, re-encode). + """ + self.logger.info("Migration Step 1: Adding keep policy to FDB resources") + + success = False + + # Option A: Edit chart files on disk + helm upgrade --reuse-values + r25_chart_path = os.environ.get("R25_CHART_PATH", "") + if r25_chart_path and os.path.isdir(r25_chart_path): + success = self._inject_keep_annotations_via_helm_upgrade(r25_chart_path) + else: + self.logger.info( + f"R25_CHART_PATH not set or not found ('{r25_chart_path}'), " + f"trying Helm release secret patch" + ) + + # Option B: Patch Helm release secret directly (fallback) + if not success: + self.logger.info("Falling back to Helm release secret patching") + self._patch_helm_release_keep_annotations(self.helm_release_sbcli) + + # Also annotate live resources (belt-and-suspenders, not sufficient alone) for kind, name in _FDB_KEEP_RESOURCES: ns_flag = f"-n {_NAMESPACE}" if kind not in ("clusterrole", "clusterrolebinding") else "" cmd = ( @@ -670,41 +1317,217 @@ def _annotate_fdb_keep(self): f"helm.sh/resource-policy=keep --overwrite 2>/dev/null || true" ) self.k8s_utils._exec_kubectl(cmd) - self.logger.info("FDB resources annotated with keep policy") + self.logger.info("FDB keep annotation step complete") - def _shutdown_all_nodes(self, storage_node_list: list[dict]): - """Step 2 / 6.1: Suspend + shutdown all storage nodes.""" - self.logger.info(f"Shutting down all {len(storage_node_list)} storage nodes") - for node in storage_node_list: - node_id = node["id"] - self.logger.info(f" Suspending node {node_id}") - try: - self.sbcli_utils.suspend_node(node_id) - except Exception as e: - self.logger.warning(f" Suspend failed for {node_id}: {e}") + def _inject_keep_annotations_via_helm_upgrade(self, chart_path: str) -> bool: + """Edit FDB template files on disk and run helm upgrade --reuse-values. - sleep_n_sec(10) + This is the correct way to add keep annotations: modify the chart + templates so Helm stores the annotation in its release manifest, + then ``helm uninstall`` will see it and skip deletion. - for node in storage_node_list: - node_id = node["id"] - self.logger.info(f" Shutting down node {node_id}") - try: - self.sbcli_utils.shutdown_node(node_id) - except Exception as e: - self.logger.warning(f" Shutdown failed for {node_id}: {e}") + Returns True if successful, False otherwise. + """ + fdb_template = os.path.join(chart_path, "templates", "foundationdb.yaml") + if not os.path.isfile(fdb_template): + self.logger.warning(f"FDB template not found at {fdb_template}") + return False - # Wait for all nodes to reach offline - self.logger.info("Waiting for all nodes to reach offline status") - for node in storage_node_list: - self.sbcli_utils.wait_for_storage_node_status( - node_id=node["id"], - status=["offline", "unavailable"], - timeout=300, - ) - self.logger.info("All storage nodes are offline") + self.logger.info(f"Editing FDB template: {fdb_template}") - def _uninstall_helm_releases(self): - """Steps 3-4: Uninstall old Helm charts.""" + try: + with open(fdb_template, "r") as f: + content = f.read() + + if "helm.sh/resource-policy" in content: + self.logger.info("FDB template already has resource-policy annotations") + else: + # Inject "helm.sh/resource-policy: keep" annotation after each + # "metadata:" block. The template has multiple YAML documents + # separated by "---". Each resource has a "metadata:" line + # followed by " name: ...". We add an annotations block. + import re + # Match "metadata:\n name: " and inject annotation + fdb_names = {name for _, name in _FDB_KEEP_RESOURCES} + for name in fdb_names: + # Pattern: metadata:\n name: (with optional labels after) + pattern = rf'(metadata:\n)( name: {re.escape(name)}\n)' + replacement = ( + r'\1 annotations:\n' + r' "helm.sh/resource-policy": keep\n' + r'\2' + ) + content, count = re.subn(pattern, replacement, content) + if count > 0: + self.logger.info(f" Injected keep annotation for: {name}") + + with open(fdb_template, "w") as f: + f.write(content) + + # Run helm upgrade --reuse-values to persist annotations + self.logger.info( + f"Running helm upgrade --reuse-values for '{self.helm_release_sbcli}'" + ) + cmd = ( + f"helm upgrade {self.helm_release_sbcli} {chart_path} " + f"--namespace {_NAMESPACE} --reuse-values --timeout 5m" + ) + out, _ = self.k8s_utils._exec_kubectl(cmd) + self.logger.info(f"Helm upgrade result: {out[:500] if out else '(empty)'}") + + # Verify the annotation is in the stored manifest + verify_cmd = ( + f"helm get manifest {self.helm_release_sbcli} -n {_NAMESPACE} " + f"2>/dev/null | grep -c 'resource-policy' || echo '0'" + ) + verify_out, _ = self.k8s_utils._exec_kubectl(verify_cmd) + annotation_count = int(verify_out.strip() or "0") + self.logger.info( + f"Verified: {annotation_count} resource-policy annotations " + f"in stored manifest" + ) + return annotation_count > 0 + + except Exception as e: + self.logger.warning(f"Failed to inject keep annotations via helm upgrade: {e}") + return False + + def _patch_helm_release_keep_annotations(self, release_name: str): + """Patch the Helm release secret to inject resource-policy: keep. + + Helm stores release data in secrets named sh.helm.release.v1..v. + The data is: base64 → base64 → gzip → JSON. We decode, inject the keep + annotation into matching FDB resource manifests, and re-encode. + """ + if not release_name: + self.logger.warning("No Helm release name provided, skipping secret patch") + return + + fdb_resource_names = {name for _, name in _FDB_KEEP_RESOURCES} + + # Find the latest Helm release secret + cmd = ( + f"kubectl get secrets -n {_NAMESPACE} " + f"-l owner=helm,name={release_name} " + f"--sort-by=.metadata.creationTimestamp " + f"-o jsonpath='{{.items[-1].metadata.name}}' 2>/dev/null || true" + ) + out, _ = self.k8s_utils._exec_kubectl(cmd) + secret_name = out.strip().strip("'") + if not secret_name: + self.logger.warning(f"No Helm release secret found for '{release_name}', skipping patch") + return + + self.logger.info(f"Patching Helm release secret: {secret_name}") + + # Read the release data from the secret + cmd = ( + f"kubectl get secret {secret_name} -n {_NAMESPACE} " + f"-o jsonpath='{{.data.release}}' 2>/dev/null || true" + ) + out, _ = self.k8s_utils._exec_kubectl(cmd) + raw = out.strip().strip("'") + if not raw: + self.logger.warning("Could not read Helm release secret data, skipping patch") + return + + try: + import base64 + import gzip + + # Helm release encoding: base64 → base64 → gzip → JSON + decoded = gzip.decompress(base64.b64decode(base64.b64decode(raw))) + release = json.loads(decoded) + + manifest = release.get("manifest", "") + if not manifest: + self.logger.warning("No manifest in Helm release, skipping patch") + return + + # Split manifest into individual YAML documents + docs = manifest.split("\n---\n") + patched = False + + new_docs = [] + for doc in docs: + # Check if this document is an FDB resource by looking for its name + matched_name = None + for fname in fdb_resource_names: + if f"name: {fname}" in doc: + matched_name = fname + break + + if matched_name and "helm.sh/resource-policy" not in doc: + # Inject the keep annotation + if " annotations:" in doc: + doc = doc.replace( + " annotations:\n", + ' annotations:\n "helm.sh/resource-policy": keep\n', + 1, + ) + else: + doc = doc.replace( + "metadata:\n", + 'metadata:\n annotations:\n "helm.sh/resource-policy": keep\n', + 1, + ) + patched = True + self.logger.info(f" Injected keep annotation for: {matched_name}") + + new_docs.append(doc) + + if not patched: + self.logger.info("No FDB resources needed patching (already have keep annotation or not found in manifest)") + return + + release["manifest"] = "\n---\n".join(new_docs) + + # Re-encode: JSON → gzip → base64 → base64 + compressed = gzip.compress(json.dumps(release).encode()) + encoded = base64.b64encode(base64.b64encode(compressed)).decode() + + # Patch the secret + patch_json = json.dumps({"data": {"release": encoded}}) + cmd = ( + f"kubectl patch secret {secret_name} -n {_NAMESPACE} " + f"--type=merge -p '{patch_json}'" + ) + self.k8s_utils._exec_kubectl(cmd) + self.logger.info("Helm release secret patched successfully") + + except Exception as e: + self.logger.warning(f"Failed to patch Helm release secret: {e}") + self.logger.warning("FDB resources may be deleted by helm uninstall") + + def _shutdown_all_nodes(self, storage_node_list: list[dict]): + """Step 2 / 6.1: Force-shutdown all storage nodes. + + Suspend is skipped because it fails with "Offline storage nodes + found, cannot suspend node without --force" when any node is + already offline (e.g. during Step 6.1 after operator install). + Using ``shutdown --force`` bypasses the suspended-state check. + """ + self.logger.info(f"Shutting down all {len(storage_node_list)} storage nodes (force)") + for node in storage_node_list: + node_id = node["id"] + self.logger.info(f" Shutting down node {node_id} (force=True)") + try: + self.sbcli_utils.shutdown_node(node_id, force=True) + except Exception as e: + self.logger.warning(f" Shutdown failed for {node_id}: {e}") + + # Wait for all nodes to reach offline + self.logger.info("Waiting for all nodes to reach offline status") + for node in storage_node_list: + self.sbcli_utils.wait_for_storage_node_status( + node_id=node["id"], + status=["offline", "unavailable"], + timeout=300, + ) + self.logger.info("All storage nodes are offline") + + def _uninstall_helm_releases(self): + """Steps 3-4: Uninstall old Helm charts.""" if self.helm_release_spdk_csi: self.logger.info( f"Migration Step 3: Uninstalling helm release '{self.helm_release_spdk_csi}'" @@ -715,7 +1538,14 @@ def _uninstall_helm_releases(self): ) sleep_n_sec(10) + # Delete resources that survived helm uninstall due to + # helm.sh/resource-policy: keep (e.g. simplyblock-snapshot-controller) + self._cleanup_kept_spdk_csi_resources() + if self.helm_release_sbcli: + # Capture FDB cluster-config BEFORE uninstall (in case keep fails) + fdb_cm_data = self._capture_fdb_cluster_config() + self.logger.info( f"Migration Step 4: Uninstalling helm release '{self.helm_release_sbcli}'" ) @@ -725,6 +1555,89 @@ def _uninstall_helm_releases(self): ) sleep_n_sec(10) + # Verify FDB cluster-config ConfigMap survived; recreate if missing + self._ensure_fdb_cluster_config(fdb_cm_data) + + def _capture_fdb_cluster_config(self) -> str: + """Capture the FDB cluster file content before helm uninstall. + + Returns the cluster file data string, or empty string if unavailable. + """ + try: + out, _ = self.k8s_utils._exec_kubectl( + f"kubectl get configmap simplyblock-fdb-cluster-config " + f"-n {_NAMESPACE} -o jsonpath='{{.data.cluster-file}}' " + f"2>/dev/null || true" + ) + data = (out or "").replace("'", "").strip() + if data: + self.logger.info( + f"Captured FDB cluster-config data ({len(data)} chars)") + return data + except Exception as e: + self.logger.warning(f"Failed to capture FDB cluster-config: {e}") + return "" + + def _ensure_fdb_cluster_config(self, fdb_cm_data: str): + """Ensure the FDB cluster-config ConfigMap exists after helm uninstall. + + The admin-control pods mount this ConfigMap as a volume. If it was + deleted during ``helm uninstall sbcli`` despite resource-policy:keep, + recreate it from the previously captured data. If no captured data is + available, attempt to extract it from a running FDB pod. + """ + # Check if ConfigMap still exists + out, _ = self.k8s_utils._exec_kubectl( + f"kubectl get configmap simplyblock-fdb-cluster-config " + f"-n {_NAMESPACE} --no-headers 2>/dev/null || true" + ) + if "simplyblock-fdb-cluster-config" in (out or ""): + self.logger.info("FDB cluster-config ConfigMap survived helm uninstall") + return + + self.logger.warning( + "FDB cluster-config ConfigMap was deleted during helm uninstall — " + "recreating it") + + # Try captured data first + if not fdb_cm_data: + # Fallback: extract from a running FDB pod + try: + out, _ = self.k8s_utils._exec_kubectl( + f"kubectl get pods -n {_NAMESPACE} " + f"-l foundationdb.org/fdb-cluster-name=simplyblock-fdb-cluster " + f"--no-headers -o custom-columns=NAME:.metadata.name " + f"2>/dev/null | head -1" + ) + fdb_pod = (out or "").strip() + if fdb_pod: + out2, _ = self.k8s_utils._exec_kubectl( + f"kubectl exec {fdb_pod} -n {_NAMESPACE} " + f"-c foundationdb -- cat /var/fdb/data/fdb.cluster " + f"2>/dev/null || true" + ) + fdb_cm_data = (out2 or "").strip() + if fdb_cm_data: + self.logger.info( + f"Extracted FDB cluster file from pod {fdb_pod}") + except Exception as e: + self.logger.warning(f"Failed to extract FDB data from pods: {e}") + + if not fdb_cm_data: + self.logger.error( + "Cannot recreate FDB cluster-config ConfigMap — no data " + "available. Admin pods will fail to start.") + return + + # Recreate the ConfigMap + # Escape single quotes in the data for the kubectl command + escaped = fdb_cm_data.replace("'", "'\\''") + self.k8s_utils._exec_kubectl( + f"kubectl create configmap simplyblock-fdb-cluster-config " + f"-n {_NAMESPACE} --from-literal=cluster-file='{escaped}'" + ) + self.logger.info("Recreated FDB cluster-config ConfigMap") + def _create_upgrade_secret(self): """Step 5: Create the upgrade secret so the operator adopts the existing cluster.""" secret_name = f"simplyblock-{self.cluster_cr_name}-upgrade" @@ -747,12 +1660,133 @@ def _create_upgrade_secret(self): out, err = self.k8s_utils._exec_kubectl(cmd) self.logger.info(f"Upgrade secret created: {out}") + def _ensure_cert_manager(self): + """Install cert-manager if not already present (required for TLS). + + R25 clusters don't have cert-manager since TLS wasn't supported. + The target operator chart validates cert-manager CRDs when + tls.enabled=true, so we install it here before helm install. + + If a stale/broken cert-manager release exists, uninstall it first + and retry the install up to 3 times. + """ + self.logger.info("Checking if cert-manager is installed") + out, _ = self.k8s_utils._exec_kubectl( + "kubectl get crd certificates.cert-manager.io 2>/dev/null || true" + ) + if "certificates.cert-manager.io" in (out or ""): + self.logger.info("cert-manager CRDs already present") + return + + # Uninstall stale cert-manager if present from a previous failed run + self.logger.info("Removing any stale cert-manager release") + self.k8s_utils._exec_kubectl( + "helm uninstall cert-manager -n cert-manager " + "--no-hooks --timeout 60s 2>/dev/null || true" + ) + + self.logger.info("Installing cert-manager (TLS prerequisite)") + self.k8s_utils._exec_kubectl( + "helm repo add jetstack https://charts.jetstack.io 2>/dev/null || true" + ) + self.k8s_utils._exec_kubectl("helm repo update") + + last_err = None + for attempt in range(1, 4): + self.logger.info(f"cert-manager install attempt {attempt}/3") + out, err = self.k8s_utils._exec_kubectl( + "helm upgrade --install cert-manager jetstack/cert-manager " + "--namespace cert-manager --create-namespace " + "--version v1.13.0 --set installCRDs=true" + ) + if err and "Error" in err: + last_err = err + self.logger.warning( + f"cert-manager install attempt {attempt} failed: {err[:200]}" + ) + self.k8s_utils._exec_kubectl( + "helm uninstall cert-manager -n cert-manager " + "--no-hooks --timeout 60s 2>/dev/null || true" + ) + sleep_n_sec(10) + continue + last_err = None + break + + if last_err: + raise RuntimeError( + f"cert-manager install failed after 3 attempts: {last_err[:500]}" + ) + + self.k8s_utils._exec_kubectl( + "kubectl wait --for=condition=Ready pods --all " + "-n cert-manager --timeout=120s" + ) + self.logger.info("cert-manager installed and ready") + + def _cleanup_kept_spdk_csi_resources(self): + """Delete resources that survived ``helm uninstall spdk-csi``. + + The old spdk-csi chart sets ``helm.sh/resource-policy: keep`` on + certain resources (e.g. simplyblock-snapshot-controller Deployment + in kube-system). ``helm uninstall`` honours that policy and leaves + them behind. These orphaned resources still carry the old Helm + ownership annotations (``meta.helm.sh/release-name: spdk-csi``), + which prevents the new ``simplyblock-operator`` chart from creating + its own version of the same resource. + + The fix is straightforward: delete the orphans so the new chart + can recreate them cleanly. + """ + self.logger.info("Cleaning up resources kept by spdk-csi resource-policy") + + # Known resources that the spdk-csi chart marks with resource-policy: keep + # Format: (resource_type, name, namespace_or_None) + kept_resources = [ + ("deployment", "simplyblock-snapshot-controller", "kube-system"), + ] + + deleted = 0 + for rtype, name, ns in kept_resources: + ns_flag = f"-n {ns}" if ns else "" + # Check if it exists and belongs to spdk-csi + check_cmd = ( + f"kubectl get {rtype} {name} {ns_flag} " + f"-o jsonpath='{{.metadata.annotations.meta\\.helm\\.sh/release-name}}' " + f"2>/dev/null || true" + ) + out, _ = self.k8s_utils._exec_kubectl(check_cmd) + release = (out or "").replace("'", "").strip() + if release == "spdk-csi": + self.logger.info( + f" Deleting {rtype}/{name} in {ns or 'cluster-scope'} " + f"(orphaned from spdk-csi with resource-policy: keep)" + ) + self.k8s_utils._exec_kubectl( + f"kubectl delete {rtype} {name} {ns_flag} " + f"--ignore-not-found" + ) + deleted += 1 + elif release: + self.logger.info( + f" {rtype}/{name} in {ns or 'cluster-scope'} belongs to " + f"release '{release}', not spdk-csi — skipping" + ) + else: + self.logger.info( + f" {rtype}/{name} in {ns or 'cluster-scope'} not found or " + f"has no release annotation — skipping" + ) + + self.logger.info(f"Deleted {deleted} orphaned spdk-csi resource(s)") + def _install_operator_chart(self): """Step 6: Install the operator Helm chart with FDB disabled.""" self.logger.info("Migration Step 6: Installing operator chart (FDB disabled)") tls_flags = "" if self.tls_enabled: + self._ensure_cert_manager() tls_flags = "--set tls.enabled=true --set tls.mutual_enabled=true" csi_flags = "" @@ -789,6 +1823,45 @@ def _install_operator_chart(self): f"--timeout=300s --field-selector=status.phase!=Succeeded" ) sleep_n_sec(15) + + # Wait specifically for admin-control pods to be Ready + self.logger.info("Waiting for admin-control pods to be Ready") + for attempt in range(60): + out, _ = self.k8s_utils._exec_kubectl( + f"kubectl get pods -n {_NAMESPACE} " + f"-l app=simplyblock-admin-control " + f"--no-headers 2>/dev/null || true" + ) + lines = [ln for ln in (out or "").strip().split("\n") if ln.strip()] + ready_count = sum( + 1 for ln in lines + if "Running" in ln and ln.split()[1].split("/")[0] == ln.split()[1].split("/")[1] + ) + if ready_count > 0: + self.logger.info( + f" {ready_count} admin-control pod(s) Ready") + break + # Check for ContainerCreating with volume mount failures + if any("ContainerCreating" in ln for ln in lines) and attempt % 10 == 9: + self.logger.warning( + f" Admin pods still ContainerCreating after {(attempt+1)*5}s — " + f"checking events for volume mount issues") + for ln in lines: + pod_name = ln.split()[0] if ln.split() else "" + if pod_name and "ContainerCreating" in ln: + ev_out, _ = self.k8s_utils._exec_kubectl( + f"kubectl get events -n {_NAMESPACE} " + f"--field-selector involvedObject.name={pod_name} " + f"--sort-by='.lastTimestamp' 2>/dev/null " + f"| tail -5 || true" + ) + if ev_out: + self.logger.warning(f" Events for {pod_name}:\n{ev_out}") + sleep_n_sec(5) + else: + self.logger.error( + "Admin-control pods did not become Ready within 300s") + self.k8s_utils.get_admin_pod(refresh=True) self.logger.info("Operator chart installed") @@ -926,8 +1999,97 @@ def _patch_backend_cr_references(self, storage_node_list: list[dict]): ) self.logger.info(f"Storage node {node_id} CR refs patched") + def _disable_auto_restart_all_nodes(self, storage_node_list: list[dict]): + """Set auto_restart_disabled=true on all nodes. + + Called before installing the R26 operator so its tasks-runner + won't create node_restart tasks for offline nodes. + """ + self.logger.info( + "Disabling auto-restart on all storage nodes " + "(prevent operator restart tasks)" + ) + sbcli = "sbcli-dev" if self.upgrade_type == "r25-to-r2x" else "sbctl" + for node in storage_node_list: + node_id = node["id"] + try: + self.k8s_utils.exec_sbcli( + f"{sbcli} --dev sn set {node_id} auto_restart_disabled true" + ) + except Exception as e: + self.logger.warning( + f"Failed to disable auto-restart for {node_id}: {e}" + ) + + def _cancel_stale_restart_tasks(self): + """Cancel any running/new node_restart tasks before our explicit restart. + + The R26 operator's tasks-runner may have created restart tasks for + offline nodes between Step 6 (operator install) and Step 6.1 + (second shutdown). These stale tasks block ``sn restart``. + """ + self.logger.info("Checking for stale node_restart tasks to cancel") + sbcli = "sbctl" + try: + stdout, _ = self.k8s_utils.exec_sbcli( + f"{sbcli} cluster list-tasks {self.cluster_id} --json --limit 0" + ) + if not stdout or not stdout.strip(): + self.logger.info("No tasks found") + return + + tasks = json.loads(stdout) + stale = [ + t for t in tasks + if t.get("function") == "node_restart" + and t.get("status") in ("running", "new") + ] + if not stale: + self.logger.info("No stale node_restart tasks found") + return + + self.logger.info(f"Found {len(stale)} stale node_restart tasks — cancelling") + for t in stale: + task_id = t.get("id") or t.get("task_id") or t.get("uuid") + target = t.get("target_id", "") + self.logger.info(f" Cancelling task {task_id} ({target})") + try: + self.k8s_utils.exec_sbcli( + f"{sbcli} cluster cancel-task {self.cluster_id} {task_id}" + ) + except Exception as e: + self.logger.warning(f" cancel-task failed for {task_id}: {e}") + + # Verify all cancelled + sleep_n_sec(5) + stdout2, _ = self.k8s_utils.exec_sbcli( + f"{sbcli} cluster list-tasks {self.cluster_id} --json --limit 0" + ) + if stdout2 and stdout2.strip(): + tasks2 = json.loads(stdout2) + remaining = [ + t for t in tasks2 + if t.get("function") == "node_restart" + and t.get("status") in ("running", "new") + ] + if remaining: + self.logger.warning( + f"{len(remaining)} node_restart tasks still running after cancel" + ) + else: + self.logger.info("All stale node_restart tasks cancelled successfully") + except Exception as e: + self.logger.warning(f"Failed to list/cancel stale tasks: {e}") + def _restart_nodes_sequentially(self, storage_node_list: list[dict]): - """Step 10: Restart each storage node one at a time with new SPDK image.""" + """Step 10: Restart each storage node one at a time with new SPDK image. + + In the maintenance upgrade path all nodes start offline, so the + cluster cannot become ``active`` until every node is back online. + We therefore restart all nodes first (waiting only for each + individual node to reach ``online``), then let the caller check + cluster-active status after the loop. + """ self.logger.info( f"Migration Step 10: Restarting {len(storage_node_list)} nodes sequentially" ) @@ -939,14 +2101,15 @@ def _restart_nodes_sequentially(self, storage_node_list: list[dict]): f" Restarting node {node_id} ({idx + 1}/{len(storage_node_list)})" ) - restart_ts = int(datetime.now().timestamp()) - spdk_flag = "" if self.target_spdk_image: spdk_flag = f" --spdk-image {self.target_spdk_image}" + proxy_flag = "" + if self.target_spdk_proxy_image: + proxy_flag = f" --spdk-proxy-image {self.target_spdk_proxy_image}" self.k8s_utils.exec_sbcli( - f"{sbcli} -d --dev sn restart {node_id}{spdk_flag}" + f"{sbcli} -d --dev sn restart {node_id}{spdk_flag}{proxy_flag}" ) # Wait for node online @@ -955,19 +2118,8 @@ def _restart_nodes_sequentially(self, storage_node_list: list[dict]): ) self.logger.info(f" Node {node_id} is back online") - # Wait for cluster active before next node - self.sbcli_utils.wait_for_cluster_status( - cluster_id=self.cluster_id, status="active", timeout=600, - ) - - # Wait for migration tasks - sleep_n_sec(30) - self.validate_migration_for_node( - restart_ts, 1200, node_id, 60, no_task_ok=True - ) - if idx < len(storage_node_list) - 1: - sleep_n_sec(30) + sleep_n_sec(10) self.logger.info("All storage nodes restarted successfully") @@ -981,28 +2133,80 @@ def _run_maintenance_upgrade(self, storage_node_list: list[dict]): ) # Pre-upgrade: short FIO to write + verify data, then stop - self.logger.info("Pre-upgrade Step 2: Creating StorageClass / VolumeSnapshotClass") - pool_name = self.pool_name - actual_pool = self.sbcli_utils.add_storage_pool(pool_name) + self.logger.info("Pre-upgrade Step 2: Create pool (R25)") + + # R25 pool name must match logicalVolume.pool_name in the spdk-csi + # helm chart (default: "testing1"). The chart's logicalVolume config + # auto-created StorageClass "simplyblock-csi-sc" referencing this pool. + pool_name = "testing1" + # R25 has no operator — create pool directly via sbcli CLI, not Pool CRD + actual_pool = self.sbcli_utils.add_storage_pool_direct( + pool_name, sbcli_cmd="sbcli-dev" + ) if actual_pool and actual_pool != pool_name: pool_name = actual_pool + + # Pool CR name must match the existing backend pool name so the + # operator can adopt it during the upgrade. + self.pool_cr_name = pool_name + self.logger.info( + f"Pool CR name set to '{self.pool_cr_name}' (matching backend pool)" + ) + sleep_n_sec(10) - self._create_storage_classes(self.cluster_id, pool_name) - pre_fio_runtime = 120 # 2 minutes — just write + verify data + # R25: StorageClass "simplyblock-csi-sc" was auto-created by the + # spdk-csi chart's logicalVolume config during helm install. + # Do NOT create StorageClasses here — use the chart-created one. + # Map XFS SC to the same chart-created SC (R25 chart has no XFS variant). + self.XFS_STORAGE_CLASS_NAME = self.STORAGE_CLASS_NAME + self.logger.info( + f"Using chart-created StorageClass '{self.STORAGE_CLASS_NAME}' " + f"(from logicalVolume config, pool_name={pool_name})" + ) + + pre_fio_runtime = 60 # 1 minute — just write data before upgrade self.FIO_RUNTIME = pre_fio_runtime self.logger.info("Pre-upgrade Step 3: Creating PVCs and running short FIO") self._create_pvcs_with_fio(len(storage_node_list), runtime=pre_fio_runtime) + # Wait for pre-upgrade FIO to complete (max 5 mins). + # FIO failure is non-fatal — the goal is to test the upgrade itself, + # not the old version's IO path. + self.logger.info( + "Pre-upgrade: Waiting up to 5 mins for FIO to complete " + "(non-fatal if it fails)" + ) + fio_timeout = 300 # 5 minutes max wait + try: + self._validate_all_fio(fio_timeout) + self.logger.info("Pre-upgrade FIO completed and validated") + except Exception as fio_err: + self.logger.warning( + f"Pre-upgrade FIO did not complete successfully: {fio_err}. " + "Continuing with upgrade — this is non-fatal." + ) + + # Clean up FIO jobs/pods (preserve PVCs for snapshots + checksums) + self.logger.info("Pre-upgrade: Cleaning up FIO jobs") + self._cleanup_fio_jobs_only() + sleep_n_sec(10) + self.logger.info("Pre-upgrade Step 4: Creating snapshots and clones") - self._create_snapshots_and_clones(runtime=pre_fio_runtime) + self._create_snapshots_and_clones(skip_clone_fio=True) - # Wait for pre-upgrade FIO to complete - self.logger.info("Pre-upgrade: Waiting for FIO to complete before maintenance") - fio_timeout = pre_fio_runtime + 300 - self._validate_all_fio(fio_timeout) - self.logger.info("Pre-upgrade FIO completed and validated") + # Run FIO on clones (writes fresh data to clones) + self.logger.info("Pre-upgrade Step 4.1: Running FIO on clones") + self._run_fio_on_clones(runtime=60) + + # Capture MD5 checksums on all PVCs and clones before upgrade + self.logger.info("Pre-upgrade Step 5: Capturing MD5 checksums before upgrade") + all_volume_names = list(self.pvc_details.keys()) + list(self.clone_details.keys()) + self.pre_upgrade_checksums = self._capture_pvc_checksums(all_volume_names) + + # Phase 2.7: Capture pre-upgrade state + self._capture_pre_upgrade_state() # ── Begin maintenance window ── self.logger.info("=" * 40 + " MAINTENANCE WINDOW START " + "=" * 40) @@ -1014,6 +2218,11 @@ def _run_maintenance_upgrade(self, storage_node_list: list[dict]): self.logger.info("Migration Step 2: Shutting down all storage nodes") self._shutdown_all_nodes(storage_node_list) + # Step 2.1: Disable auto-restart — commented out, dev fixed the + # operator's tasks-runner to not create restart tasks for offline nodes. + # Uncomment if the product fix regresses. + # self._disable_auto_restart_all_nodes(storage_node_list) + # Steps 3-4: Uninstall old Helm charts self.logger.info("Migration Steps 3-4: Uninstalling old Helm releases") self._uninstall_helm_releases() @@ -1041,9 +2250,25 @@ def _run_maintenance_upgrade(self, storage_node_list: list[dict]): # Step 9: Patch backend CR references self._patch_backend_cr_references(storage_node_list) + # Step 9.1: Cancel stale restart tasks — commented out, dev fixed + # the operator to not create restart tasks during upgrade. + # Uncomment if stale tasks reappear. + # self._cancel_stale_restart_tasks() + # Step 10: Restart storage nodes one at a time self._restart_nodes_sequentially(storage_node_list) + # Wait for all SPDK pods ready and all nodes online before proceeding + self.logger.info("Waiting for all SPDK pods to be ready") + self.k8s_utils.wait_spdk_pods_ready( + expected_count=len(storage_node_list), timeout=600, + ) + self.logger.info("Waiting for all storage nodes to be online") + for node in storage_node_list: + self.sbcli_utils.wait_for_storage_node_status( + node_id=node["id"], status="online", timeout=600, + ) + # ── End maintenance window ── self.logger.info("=" * 40 + " MAINTENANCE WINDOW END " + "=" * 40) @@ -1055,10 +2280,31 @@ def _run_maintenance_upgrade(self, storage_node_list: list[dict]): ) self._assert_all_nodes_healthy() - # Post-upgrade: run new FIO to verify IO works after migration - self.logger.info("Post-upgrade: Verifying IO works after migration") + # Verify MD5 checksums match post-upgrade + if hasattr(self, "pre_upgrade_checksums") and self.pre_upgrade_checksums: + self.logger.info( + "Post-upgrade: Verifying MD5 checksums match pre-upgrade data" + ) + self._verify_pvc_checksums(self.pre_upgrade_checksums, "post-upgrade") + else: + self.logger.warning( + "No pre-upgrade checksums available — skipping MD5 verification" + ) + + # Phase 4.1–4.3: Verify old data survives the upgrade + self.logger.info("Post-upgrade: Verifying old data integrity") + self._verify_old_data_post_upgrade() + + # Phase 4.4–4.5: New PVC provisioning + snapshot/clone + self.logger.info("Post-upgrade: Verifying new provisioning works") self._run_post_upgrade_verification() + # Phase 4.6: Node outage test + self._run_node_outage_test() + + # Phase 4.7: Final checklist + self._run_final_checklist(is_maintenance_upgrade=True) + # ── Main test flow ───────────────────────────────────────────────────────── def run(self): @@ -1097,3 +2343,306 @@ def run(self): f"{self.base_version} -> {self.target_version}" ) self.logger.info("TEST CASE PASSED !!!") + + +class K8sNativeMajorUpgradeDualNode(K8sNativeMajorUpgrade): + """K8s-native major upgrade for dual-node-per-host (nodesPerSocket=2). + + Each worker runs 2 logical storage nodes. Rolling restarts are grouped + by worker so both nodes on a host are restarted together before moving + to the next host. + + Dispatch the upgrade pipeline with: + EXTRA_SN_ARGS: "--nodes-per-socket 2" (bootstrap creates 2 nodes/host) + TEST_CLASS: "K8sNativeMajorUpgradeDualNode" + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.test_name = "k8s_native_major_upgrade_dual_node" + self.nodes_per_socket = 2 + + # ── Helpers ──────────────────────────────────────────────────────── + + def _build_ip_to_node_ids(self): + """Build IP -> [node_id, ...] mapping from the storage-node API.""" + sn_results = self.sbcli_utils.get_storage_nodes().get("results", []) + ip_to_ids = {} + for r in sn_results: + nid = r.get("id") or r.get("uuid") or r.get("node_id") + ip = r.get("ip") or r.get("mgmt_ip") or r.get("management_ip") + if nid and ip: + ip_to_ids.setdefault(ip, []).append(nid) + return ip_to_ids + + def _get_unique_worker_ips(self, storage_node_list): + """Extract unique worker IPs from node list, preserving order.""" + seen = set() + unique = [] + for r in storage_node_list: + ip = r.get("ip") or r.get("mgmt_ip") or r.get("management_ip") + if ip and ip not in seen: + seen.add(ip) + unique.append(ip) + return unique + + # ── Maintenance-window upgrade overrides ─────────────────────────── + + def _apply_custom_resources(self, storage_node_list): + """Override to include nodesPerSocket in StorageNodeSet spec.""" + self.logger.info( + "Migration Step 7: Applying custom resources " + f"(nodesPerSocket={self.nodes_per_socket})" + ) + + # Build worker nodes YAML from environment or K8s + worker_yaml = "" + worker_nodes_env = os.environ.get("WORKER_NODES", "") + if worker_nodes_env: + for node in worker_nodes_env.split(","): + node = node.strip() + if node: + worker_yaml += f" - {node}\n" + else: + self.logger.warning( + "WORKER_NODES env not set, attempting to derive from K8s" + ) + out, _ = self.k8s_utils._exec_kubectl( + "kubectl get nodes -l node-role.kubernetes.io/worker " + "-o jsonpath='{.items[*].metadata.name}' 2>/dev/null || " + "kubectl get nodes --no-headers " + "-o custom-columns=NAME:.metadata.name" + ) + for node_name in (out or "").replace("'", "").split(): + node_name = node_name.strip() + if node_name: + worker_yaml += f" - {node_name}\n" + + sb_repo = self.simplyblock_repo + sb_tag = self.target_docker_image + spdk_image = self.target_spdk_image + mgmt_ifc = os.environ.get("MGMT_IFC", "ens18") + data_nics = os.environ.get("DATA_NICS", "enp1s0") + max_lvol = os.environ.get("MAX_LVOL", "30") + + cr_yaml = f""" +apiVersion: storage.simplyblock.io/v1alpha1 +kind: StorageCluster +metadata: + name: {self.cluster_cr_name} + namespace: {_NAMESPACE} +spec: + fabricType: tcp + isSingleNode: false + enableNodeAffinity: true + strictNodeAntiAffinity: false + stripe: + dataChunks: {self.ndcs} + parityChunks: {self.npcs} + warningThreshold: + capacity: 95 + provisionedCapacity: 97 + criticalThreshold: + capacity: 96 + provisionedCapacity: 98 +--- +apiVersion: storage.simplyblock.io/v1alpha1 +kind: Pool +metadata: + name: {self.pool_cr_name} + namespace: {_NAMESPACE} +spec: + clusterName: {self.cluster_cr_name} +--- +apiVersion: storage.simplyblock.io/v1alpha1 +kind: StorageNodeSet +metadata: + name: {self.node_cr_name} + namespace: {_NAMESPACE} +spec: + clusterName: {self.cluster_cr_name} + clusterImage: "{sb_repo}:{sb_tag}" + spdkImage: "{spdk_image}" + spdkProxyImage: "{sb_repo}:{sb_tag}" + mgmtIfname: {mgmt_ifc} + dataIfname: + - {data_nics} + maxLogicalVolumeCount: {max_lvol} + enableCpuTopology: true + nodesPerSocket: {self.nodes_per_socket} + workerNodes: +{worker_yaml}""" + + apply_cmd = f"cat <<'CREOF' | kubectl apply -f -\n{cr_yaml}\nCREOF" + out, err = self.k8s_utils._exec_kubectl(apply_cmd) + self.logger.info(f"CRs applied: {out}") + sleep_n_sec(10) + + def _restart_nodes_sequentially(self, storage_node_list): + """Override to group restarts by worker (both nodes per host together).""" + ip_to_node_ids = self._build_ip_to_node_ids() + unique_ips = self._get_unique_worker_ips(storage_node_list) + + self.logger.info( + f"Migration Step 10: Restarting nodes on {len(unique_ips)} workers " + f"(nodesPerSocket={self.nodes_per_socket})" + ) + + sbcli = "sbctl" + for worker_idx, host_ip in enumerate(unique_ips, 1): + nids = ip_to_node_ids.get(host_ip, []) + self.logger.info( + f" Worker {worker_idx}/{len(unique_ips)} ({host_ip}): " + f"restarting {len(nids)} nodes: {nids}" + ) + + # Restart all nodes on this worker + for node_id in nids: + spdk_flag = "" + if self.target_spdk_image: + spdk_flag = f" --spdk-image {self.target_spdk_image}" + proxy_flag = "" + if self.target_spdk_proxy_image: + proxy_flag = ( + f" --spdk-proxy-image {self.target_spdk_proxy_image}" + ) + self.k8s_utils.exec_sbcli( + f"{sbcli} -d --dev sn restart " + f"{node_id}{spdk_flag}{proxy_flag}" + ) + + # Wait for all nodes on this worker to come online + for node_id in nids: + self.sbcli_utils.wait_for_storage_node_status( + node_id=node_id, status="online", timeout=600, + ) + self.logger.info(f" Node {node_id} is back online") + + if worker_idx < len(unique_ips): + sleep_n_sec(10) + + self.logger.info("All storage nodes restarted successfully") + + # ── Rolling upgrade override ─────────────────────────────────────── + + def _run_rolling_upgrade(self, storage_node_list): + """Override to group rolling restarts by worker node.""" + self.FIO_RUNTIME = 3600 # 1 hour — FIO runs throughout + + # Pre-upgrade: create PVCs, FIO, snapshots, clones (same as parent) + self.logger.info("Step 2: Creating StorageClass and VolumeSnapshotClass") + pool_name = self.pool_name + actual_pool = self.sbcli_utils.add_storage_pool(pool_name) + if actual_pool and actual_pool != pool_name: + pool_name = actual_pool + + self.pool_cr_name = pool_name + self.logger.info( + f"Pool CR name set to '{self.pool_cr_name}' (matching backend pool)" + ) + + sleep_n_sec(10) + self._create_storage_classes(self.cluster_id, pool_name) + + self.logger.info("Step 3: Creating PVCs and starting FIO Jobs") + self._create_pvcs_with_fio(len(storage_node_list)) + + self.logger.info("Step 4: Creating snapshots and clones") + self._create_snapshots_and_clones() + + # Capture pre-upgrade state + self._capture_pre_upgrade_state() + + self.logger.info("Step 5: Waiting 60s for FIO to establish baseline") + sleep_n_sec(60) + + # Helm upgrade + self.logger.info("Step 6: Running helm upgrade for control plane") + self._helm_upgrade() + sleep_n_sec(30) + + # Rolling restart — grouped by worker + self.logger.info( + "Step 7: Rolling storage node restart " + "(dual-node, grouped by worker)" + ) + storage_node_list = self.sbcli_utils.get_storage_nodes()["results"] + ip_to_node_ids = self._build_ip_to_node_ids() + unique_ips = self._get_unique_worker_ips(storage_node_list) + total_nodes = len(storage_node_list) + + for worker_idx, host_ip in enumerate(unique_ips, 1): + nids = ip_to_node_ids.get(host_ip, []) + self.logger.info( + f"Step 7.{worker_idx}: Restarting worker {host_ip} " + f"({worker_idx}/{len(unique_ips)}, " + f"{len(nids)} nodes: {nids})" + ) + + restart_ts = int(datetime.now().timestamp()) + + # Create StorageNodeOps for each node on this worker + ops_names = [] + for node_id in nids: + ops_name, _ = self.k8s_utils.patch_storage_node_restart( + node_uuid=node_id, + spdk_image=self.target_spdk_image or None, + spdk_proxy_image=self.target_spdk_proxy_image or None, + ) + ops_names.append(ops_name) + + # Wait for all StorageNodeOps to complete + for ops_name in ops_names: + self.k8s_utils.wait_storage_node_ops_done( + ops_name, timeout=600 + ) + + self.k8s_utils.wait_spdk_pods_ready( + expected_count=total_nodes, timeout=600 + ) + + # Wait for all nodes on this worker to come online + for node_id in nids: + self.sbcli_utils.wait_for_storage_node_status( + node_id=node_id, status="online", timeout=600, + ) + self.logger.info( + f"All {len(nids)} nodes on worker {host_ip} are back online" + ) + + sleep_n_sec(30) + for node_id in nids: + self.validate_migration_for_node( + restart_ts, 1200, node_id, 60, no_task_ok=True + ) + + if worker_idx < len(unique_ips): + sleep_n_sec(30) + + self.logger.info("All storage nodes restarted successfully") + self.runner_k8s_log.restart_logging() + + # Post-upgrade validation (same as parent) + self.logger.info("Step 8: Post-upgrade validation") + self._assert_all_nodes_healthy() + self.sbcli_utils.wait_for_cluster_status( + cluster_id=self.cluster_id, status="active", timeout=300, + ) + + fio_timeout = self.FIO_RUNTIME + 300 + self._validate_all_fio(fio_timeout) + self.logger.info("All pre-upgrade FIO jobs validated successfully") + + # Verify old data survives upgrade + self.logger.info("Step 9: Verifying old data integrity post-upgrade") + self._verify_old_data_post_upgrade() + + # New PVC provisioning + snapshot/clone + self.logger.info("Step 10: Post-upgrade new PVC verification") + self._run_post_upgrade_verification() + + # Node outage test + self._run_node_outage_test() + + # Final checklist + self._run_final_checklist(is_maintenance_upgrade=False) diff --git a/e2e/e2e_tests/upgrade_tests/major_upgrade.py b/e2e/e2e_tests/upgrade_tests/major_upgrade.py index 8cedd3e98a..be2c61811a 100644 --- a/e2e/e2e_tests/upgrade_tests/major_upgrade.py +++ b/e2e/e2e_tests/upgrade_tests/major_upgrade.py @@ -967,11 +967,12 @@ def run(self): ) sleep_n_sec(self.step_sleep) - # Restart with target spdk image - self.logger.info(f"[SN {snode}] Restarting with spdk-image={self.spdk_image}") + # Restart with target spdk image and proxy image + proxy_flag = f" --spdk-proxy-image {self.target_docker_image}" if self.target_docker_image else "" + self.logger.info(f"[SN {snode}] Restarting with spdk-image={self.spdk_image}, spdk-proxy-image={self.target_docker_image or '(default)'}") self.ssh_obj.exec_command( self.mgmt_nodes[0], - f"{self.sbctl_cmd} --dev -d sn restart {node_id} --spdk-image {self.spdk_image}", + f"{self.sbctl_cmd} --dev -d sn restart {node_id} --spdk-image {self.spdk_image}{proxy_flag}", raise_on_error=True, ) try: @@ -1076,3 +1077,599 @@ def __init__(self, **kwargs): self.fio_during_upgrade = False self.test_name = "test_major_upgrade_single" self.logger.info("Single-node upgrade mode: FIO will NOT run during the upgrade window") + + +class TestMajorUpgradeDualNode(TestMajorUpgrade): + """ + Dual-node-per-host upgrade variant: handles clusters where each physical + host runs 2 storage nodes (``--nodes-per-socket 2``). + + Key differences from the single-node-per-host parent: + + * ``node_ctx`` is keyed by **node_id** (UUID), not by IP address. + * An ``ip_to_node_ids`` mapping (IP → list of node_ids) is built from the + storage-node API so we can iterate logical nodes per physical host. + * The rolling upgrade loop (Step 10) iterates **unique IPs**: for each host + it suspends/shuts-down/deploys/restarts ALL logical nodes on that host, + while physical operations (env update, ``sn deploy``, Docker logging) run + only once per IP. + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.test_name = "test_major_upgrade_dual_node" + self.nodes_per_socket = 2 + self.logger.info( + f"Dual-node-per-host upgrade mode: expecting {self.nodes_per_socket} " + "logical nodes per physical host" + ) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _build_ip_to_node_ids(self): + """Build IP → [node_id, ...] mapping from the storage-node API. + + For dual-node-per-host, one IP maps to 2 node_ids. + Returns dict[str, list[str]]. + """ + sn_results = self.sbcli_utils.get_storage_nodes().get("results", []) + ip_to_ids = {} + for r in sn_results: + nid = r.get("id") or r.get("uuid") or r.get("node_id") + ip = r.get("ip") or r.get("mgmt_ip") or r.get("management_ip") + if nid and ip: + ip_to_ids.setdefault(ip, []).append(nid) + return ip_to_ids + + # ------------------------------------------------------------------ + # Overridden run — node_ctx keyed by node_id, rolling upgrade per host + # ------------------------------------------------------------------ + + def run(self): + # Resolve paths now that setup() has populated docker_logs_path + self.base_log_root = f"{self.docker_logs_path}/upgrade_fio_logs" + + # ---------------------------------------------------------------- + # Step 1: Verify base version + # ---------------------------------------------------------------- + self.logger.info("Step 1: Verify base version on all nodes") + prev_versions = self.common_utils.get_all_node_versions() + for node_ip, version in prev_versions.items(): + assert self.base_version in version, ( + f"Base version mismatch on {node_ip}: {version}" + ) + + self.logger.info("Collect containers/images on all nodes (pre-upgrade)") + pre_upgrade_containers = {} + mgmt, storage = self.sbcli_utils.get_all_nodes_ip() + all_nodes = mgmt + storage + for node in all_nodes: + pre_upgrade_containers[node] = self.ssh_obj.get_image_dict(node=node) + + # ---------------------------------------------------------------- + # Step 2: Create pool + # ---------------------------------------------------------------- + self.logger.info("Step 2: Create storage pool") + self.sbcli_utils.add_storage_pool(pool_name=self.pool_name) + sleep_n_sec(5) + + # ---------------------------------------------------------------- + # Build IP → [node_id, ...] mapping + # ---------------------------------------------------------------- + ip_to_node_ids = self._build_ip_to_node_ids() + unique_storage_ips = list(dict.fromkeys(storage)) # de-dup, preserve order + + # Flatten all node_ids in host order + all_node_ids = [] + for ip in unique_storage_ips: + nids = ip_to_node_ids.get(ip, []) + assert len(nids) >= self.nodes_per_socket, ( + f"Expected at least {self.nodes_per_socket} nodes on {ip}, " + f"got {len(nids)}: {nids}" + ) + all_node_ids.extend(nids) + + self.logger.info( + f"Dual-node mapping: {len(unique_storage_ips)} hosts, " + f"{len(all_node_ids)} logical nodes: {ip_to_node_ids}" + ) + + # ---------------------------------------------------------------- + # Steps 3-4: Create VERIFY + FIO lvols per logical node + # ---------------------------------------------------------------- + node_ctx = {} # keyed by node_id + + self.logger.info( + f"Step 3-4: Pre-upgrade: {VERIFY_LVOLS_PER_NODE} verify lvol(s) + " + f"{FIO_LVOLS_PER_NODE} fio lvol(s) per logical node " + f"({len(all_node_ids)} nodes total)" + ) + + for nid_idx, node_id in enumerate(all_node_ids): + # Find host IP for this node_id + host_ip = None + for ip, nids in ip_to_node_ids.items(): + if node_id in nids: + host_ip = ip + break + + verify_lvols = [] + fio_lvols = [] + + # --- Verify lvols --- + for lvol_idx in range(VERIFY_LVOLS_PER_NODE): + tag = f"vfy_{nid_idx}_{lvol_idx}" + lvol_name = f"{self.lvol_name}_{tag}" + snap_name = f"{self.snapshot_name}_{tag}" + clone_name = f"{self.clone_name}_{tag}" + mount_path = f"{self.base_mount_root}_{tag}" + clone_mount = f"{self.base_mount_root}_{tag}_clone" + pre_log = f"{self.base_log_root}/fio_pre_{tag}.log" + client_node = random.choice(self.fio_node) + + self.logger.info( + f"[{node_id}@{host_ip}] Creating verify LVOL " + f"{lvol_idx+1}/{VERIFY_LVOLS_PER_NODE}: {lvol_name}" + ) + self.sbcli_utils.add_lvol( + lvol_name=lvol_name, pool_name=self.pool_name, size="5G" + ) + sleep_n_sec(3) + + before = self.ssh_obj.get_devices(client_node) + for cmd in self.sbcli_utils.get_lvol_connect_str(lvol_name): + self.ssh_obj.exec_command(client_node, cmd) + sleep_n_sec(3) + after = self.ssh_obj.get_devices(client_node) + disk = self._detect_new_device(client_node, before, after) + self.ssh_obj.format_disk(client_node, disk) + self.ssh_obj.mount_path(client_node, disk, mount_path) + + verify_lvols.append({ + "tag": tag, + "client_node": client_node, + "lvol_name": lvol_name, + "mount_path": mount_path, + "pre_log": pre_log, + "snapshot_name": snap_name, + "snapshot_id": None, + "clone_name": clone_name, + "clone_mount": clone_mount, + "base_md5": None, + "clone_md5": None, + }) + + # --- FIO lvols --- + for lvol_idx in range(FIO_LVOLS_PER_NODE): + tag = f"fio_{nid_idx}_{lvol_idx}" + lvol_name = f"{self.lvol_name}_{tag}" + snap_name = f"{self.snapshot_name}_{tag}" + clone_name = f"{self.clone_name}_{tag}" + mount_path = f"{self.base_mount_root}_{tag}" + clone_mount = f"{self.base_mount_root}_{tag}_clone" + client_node = random.choice(self.fio_node) + + self.logger.info( + f"[{node_id}@{host_ip}] Creating fio LVOL " + f"{lvol_idx+1}/{FIO_LVOLS_PER_NODE}: {lvol_name}" + ) + self.sbcli_utils.add_lvol( + lvol_name=lvol_name, pool_name=self.pool_name, size="5G" + ) + sleep_n_sec(3) + + before = self.ssh_obj.get_devices(client_node) + for cmd in self.sbcli_utils.get_lvol_connect_str(lvol_name): + self.ssh_obj.exec_command(client_node, cmd) + sleep_n_sec(3) + after = self.ssh_obj.get_devices(client_node) + disk = self._detect_new_device(client_node, before, after) + self.ssh_obj.format_disk(client_node, disk) + self.ssh_obj.mount_path(client_node, disk, mount_path) + + # Create snapshot + clone + lvol_id = self.sbcli_utils.get_lvol_id(lvol_name) + self.ssh_obj.add_snapshot(self.mgmt_nodes[0], lvol_id, snap_name) + snap_id = self.ssh_obj.get_snapshot_id(self.mgmt_nodes[0], snap_name) + self.ssh_obj.add_clone(self.mgmt_nodes[0], snap_id, clone_name) + sleep_n_sec(3) + + before2 = self.ssh_obj.get_devices(client_node) + for cmd in self.sbcli_utils.get_lvol_connect_str(clone_name): + self.ssh_obj.exec_command(client_node, cmd) + sleep_n_sec(3) + after2 = self.ssh_obj.get_devices(client_node) + clone_disk = self._detect_new_device(client_node, before2, after2) + self.ssh_obj.mount_path(client_node, clone_disk, clone_mount) + + fio_lvols.append({ + "tag": tag, + "client_node": client_node, + "lvol_name": lvol_name, + "mount_path": mount_path, + "snapshot_name": snap_name, + "snapshot_id": snap_id, + "clone_name": clone_name, + "clone_mount": clone_mount, + "lvol_fio_session": None, + "lvol_fio_log": None, + "clone_fio_session": None, + "clone_fio_log": None, + }) + + node_ctx[node_id] = { + "host_ip": host_ip, + "verify_lvols": verify_lvols, + "fio_lvols": fio_lvols, + } + + # ---------------------------------------------------------------- + # Step 5: Short FIO on all verify lvols + # ---------------------------------------------------------------- + self.logger.info("Step 5: Start short pre-upgrade fio on all verify lvols (runtime=120s)") + pre_fio_threads = [] + pre_fio_results = {} + for node_id in all_node_ids: + for lvol_ctx in node_ctx[node_id]["verify_lvols"]: + tag = lvol_ctx["tag"] + t = threading.Thread( + target=self._start_fio_tmux_thread, + args=(lvol_ctx["client_node"], lvol_ctx["mount_path"], + lvol_ctx["pre_log"], f"fio_pre_{tag}", 120, + pre_fio_results, tag), + daemon=True, + ) + t.start() + pre_fio_threads.append(t) + sleep_n_sec(1) + + for t in pre_fio_threads: + t.join(timeout=30) + + self.logger.info("Step 5: Waiting for all verify fio sessions to complete") + for node_id in all_node_ids: + for lvol_ctx in node_ctx[node_id]["verify_lvols"]: + tag = lvol_ctx["tag"] + session = pre_fio_results.get(tag, f"fio_fio_pre_{tag}") + self._wait_tmux_gone(lvol_ctx["client_node"], session, timeout=600) + self._assert_fio_log_clean(lvol_ctx["client_node"], lvol_ctx["pre_log"]) + + # ---------------------------------------------------------------- + # Step 6: Snap + clone + md5 verify on all verify lvols + # ---------------------------------------------------------------- + self.logger.info("Step 6: Snapshot + clone + md5 verify on all verify lvols") + for node_id in all_node_ids: + for lvol_ctx in node_ctx[node_id]["verify_lvols"]: + lvol_name = lvol_ctx["lvol_name"] + snap_name = lvol_ctx["snapshot_name"] + clone_name = lvol_ctx["clone_name"] + client_node = lvol_ctx["client_node"] + mount_path = lvol_ctx["mount_path"] + clone_mount = lvol_ctx["clone_mount"] + + lvol_id = self.sbcli_utils.get_lvol_id(lvol_name) + self.ssh_obj.add_snapshot(self.mgmt_nodes[0], lvol_id, snap_name) + snap_id = self.ssh_obj.get_snapshot_id(self.mgmt_nodes[0], snap_name) + self.ssh_obj.add_clone(self.mgmt_nodes[0], snap_id, clone_name) + sleep_n_sec(3) + + base_files = self.ssh_obj.find_files(client_node, mount_path) + base_md5 = self.ssh_obj.generate_checksums(client_node, base_files) + + before2 = self.ssh_obj.get_devices(client_node) + for cmd in self.sbcli_utils.get_lvol_connect_str(clone_name): + self.ssh_obj.exec_command(client_node, cmd) + sleep_n_sec(3) + after2 = self.ssh_obj.get_devices(client_node) + clone_disk = self._detect_new_device(client_node, before2, after2) + self.ssh_obj.mount_path(client_node, clone_disk, clone_mount) + + clone_files = self.ssh_obj.find_files(client_node, clone_mount) + clone_md5 = self.ssh_obj.generate_checksums(client_node, clone_files) + + assert set(base_md5.values()) == set(clone_md5.values()), ( + f"[{client_node}] Pre-upgrade md5 mismatch (lvol vs clone) " + f"for {lvol_name}" + ) + + lvol_ctx["snapshot_id"] = snap_id + lvol_ctx["base_md5"] = base_md5 + lvol_ctx["clone_md5"] = clone_md5 + + # ---------------------------------------------------------------- + # Step 7: Start long fio on all fio lvols + clones + # ---------------------------------------------------------------- + if self.fio_during_upgrade: + self.logger.info( + "Step 7: Start long fio (3600s) on all fio lvols + clones " + f"({FIO_LVOLS_PER_NODE * 2} sessions per node)" + ) + upgrade_fio_threads = [] + upgrade_fio_results = {} + + for node_id in all_node_ids: + for lvol_ctx in node_ctx[node_id]["fio_lvols"]: + tag = lvol_ctx["tag"] + client_node = lvol_ctx["client_node"] + + # FIO on the lvol + lvol_log = f"{self.base_log_root}/fio_upgrade_{tag}_lvol.log" + lvol_ctx["lvol_fio_log"] = lvol_log + t = threading.Thread( + target=self._start_fio_tmux_thread, + args=(client_node, lvol_ctx["mount_path"], + lvol_log, f"fio_upg_{tag}_lvol", 3600, + upgrade_fio_results, f"{tag}_lvol"), + daemon=True, + ) + t.start() + upgrade_fio_threads.append(t) + sleep_n_sec(1) + + # FIO on the clone + clone_log = f"{self.base_log_root}/fio_upgrade_{tag}_clone.log" + lvol_ctx["clone_fio_log"] = clone_log + t = threading.Thread( + target=self._start_fio_tmux_thread, + args=(client_node, lvol_ctx["clone_mount"], + clone_log, f"fio_upg_{tag}_clone", 3600, + upgrade_fio_results, f"{tag}_clone"), + daemon=True, + ) + t.start() + upgrade_fio_threads.append(t) + sleep_n_sec(1) + + for t in upgrade_fio_threads: + t.join(timeout=30) + + for node_id in all_node_ids: + for lvol_ctx in node_ctx[node_id]["fio_lvols"]: + tag = lvol_ctx["tag"] + lvol_ctx["lvol_fio_session"] = upgrade_fio_results.get( + f"{tag}_lvol", f"fio_fio_upg_{tag}_lvol" + ) + lvol_ctx["clone_fio_session"] = upgrade_fio_results.get( + f"{tag}_clone", f"fio_fio_upg_{tag}_clone" + ) + self.logger.info( + f" [{lvol_ctx['client_node']}] fio sessions: " + f"lvol={lvol_ctx['lvol_fio_session']} " + f"clone={lvol_ctx['clone_fio_session']}" + ) + + sleep_n_sec(10) + else: + self.logger.info("Step 7: Skipping FIO during upgrade (non-HA mode)") + + # ---------------------------------------------------------------- + # Step 8: pip install target sbcli on ALL nodes + # ---------------------------------------------------------------- + self.logger.info("Step 8: pip install target sbcli on ALL nodes") + unique_all_nodes = list(dict.fromkeys(all_nodes)) + for node in unique_all_nodes: + self._pip_install_target(node) + sleep_n_sec(5) + + # ---------------------------------------------------------------- + # Step 8b: Update env_var on ALL mgmt nodes + # ---------------------------------------------------------------- + self.logger.info("Step 8b: Update simplyblock_core/env_var on all mgmt nodes") + for node in mgmt: + self._update_node_env(node) + + # ---------------------------------------------------------------- + # Step 9: Cluster update cp-only + # ---------------------------------------------------------------- + self.logger.info("Step 9: sbctl -d cluster update --cp-only true") + self.ssh_obj.exec_command( + self.mgmt_nodes[0], + f"{self.sbctl_cmd} -d cluster update {self.cluster_id} --cp-only true", + raise_on_error=True, + ) + sleep_n_sec(60) + + # ---------------------------------------------------------------- + # Step 9b: DB migration + # ---------------------------------------------------------------- + if self._needs_db_migration(): + self.logger.info("Step 9b: Running DB migration script on mgmt node") + self._run_r25_to_r26_migration(self.mgmt_nodes[0]) + sleep_n_sec(self.step_sleep) + else: + self.logger.info( + f"Step 9b: Skipping DB migration " + f"(base={self.base_version}, target={self.target_version})" + ) + + # ---------------------------------------------------------------- + # Step 10: Rolling upgrade — per physical host + # ---------------------------------------------------------------- + self.logger.info( + f"Step 10: Rolling upgrade of storage nodes " + f"({len(unique_storage_ips)} hosts, {len(all_node_ids)} logical nodes)" + ) + + for host_ip in unique_storage_ips: + node_ids_on_host = ip_to_node_ids.get(host_ip, []) + self.logger.info( + f"[HOST {host_ip}] Upgrading {len(node_ids_on_host)} nodes: " + f"{node_ids_on_host}" + ) + + # Verify FIO sessions for all nodes on this host + if self.fio_during_upgrade: + self.logger.info(f"[HOST {host_ip}] Verifying fio sessions") + for nid in node_ids_on_host: + for lvol_ctx in node_ctx[nid]["fio_lvols"]: + cn = lvol_ctx["client_node"] + for sess_key in ("lvol_fio_session", "clone_fio_session"): + session = lvol_ctx[sess_key] + assert self._is_tmux_running(cn, session), ( + f"FIO session {session} on {cn} is not running " + f"before upgrade of {nid}@{host_ip}!" + ) + + # Suspend ALL nodes on this host + for nid in node_ids_on_host: + self.logger.info(f"[{nid}@{host_ip}] Suspending") + self.ssh_obj.exec_command( + self.mgmt_nodes[0], + f"{self.sbctl_cmd} -d sn suspend {nid}", + raise_on_error=True, + ) + for nid in node_ids_on_host: + self.sbcli_utils.wait_for_storage_node_status( + nid, "suspended", timeout=1000 + ) + sleep_n_sec(self.step_sleep) + + # Shutdown ALL nodes on this host + for nid in node_ids_on_host: + self.logger.info(f"[{nid}@{host_ip}] Shutting down") + self.ssh_obj.exec_command( + self.mgmt_nodes[0], + f"{self.sbctl_cmd} -d sn shutdown {nid}", + raise_on_error=True, + ) + for nid in node_ids_on_host: + self.sbcli_utils.wait_for_storage_node_status( + nid, "offline", timeout=1000 + ) + sleep_n_sec(self.step_sleep) + + # Physical host ops — once per IP + self.logger.info(f"[HOST {host_ip}] Updating env_var with target images") + self._update_node_env(host_ip) + sleep_n_sec(self.step_sleep) + + self.logger.info(f"[HOST {host_ip}] Running sn deploy") + self.ssh_obj.exec_command( + host_ip, + f"{self.sbctl_cmd} -d sn deploy --ifname {self.ifname}", + raise_on_error=True, + ) + sleep_n_sec(self.step_sleep) + + # Restart ALL nodes on this host + for nid in node_ids_on_host: + proxy_flag = ( + f" --spdk-proxy-image {self.target_docker_image}" + if self.target_docker_image else "" + ) + self.logger.info( + f"[{nid}@{host_ip}] Restarting with " + f"spdk-image={self.spdk_image}, " + f"spdk-proxy-image={self.target_docker_image or '(default)'}" + ) + self.ssh_obj.exec_command( + self.mgmt_nodes[0], + f"{self.sbctl_cmd} --dev -d sn restart {nid} " + f"--spdk-image {self.spdk_image}{proxy_flag}", + raise_on_error=True, + ) + + # Wait for ALL nodes online + for nid in node_ids_on_host: + try: + self.sbcli_utils.wait_for_storage_node_status( + nid, "online", timeout=1000 + ) + except Exception: + self.logger.warning( + f"[{nid}@{host_ip}] Restart status check failed — continuing" + ) + # Restart Docker logging — once per IP + if not self.k8s_test: + for node in unique_storage_ips: + if node == host_ip: + self.ssh_obj.restart_docker_logging( + node_ip=host_ip, + containers=self.container_nodes.get(host_ip, []), + log_dir=os.path.join(self.docker_logs_path, host_ip), + test_name=self.test_name, + ) + else: + self.runner_k8s_log.restart_logging() + sleep_n_sec(self.step_sleep) + + # Wait for migration for ALL nodes on this host + for nid in node_ids_on_host: + self.logger.info(f"[{nid}@{host_ip}] Waiting for migration tasks") + migration_ts = int(time.time()) - 120 + self.validate_migration_for_node( + timestamp=migration_ts, + timeout=1800, + node_id=nid, + check_interval=30, + no_task_ok=(not self.fio_during_upgrade), + ) + sleep_n_sec(self.step_sleep) + + # ---------------------------------------------------------------- + # Step 11: Validate docker images upgraded + # ---------------------------------------------------------------- + self.logger.info("Step 11: Validate upgraded docker images/containers") + post_upgrade_containers = {} + for node in unique_all_nodes: + post_upgrade_containers[node] = self.ssh_obj.get_image_dict(node=node) + self.common_utils.assert_upgrade_docker_image( + pre_upgrade_containers, post_upgrade_containers + ) + sleep_n_sec(self.step_sleep) + + # ---------------------------------------------------------------- + # Step 12: Verify fio still running, wait for finish + # ---------------------------------------------------------------- + if self.fio_during_upgrade: + self.logger.info("Step 12: Verify fio still running post-upgrade") + for node_id in all_node_ids: + for lvol_ctx in node_ctx[node_id]["fio_lvols"]: + cn = lvol_ctx["client_node"] + for sess_key, log_key in ( + ("lvol_fio_session", "lvol_fio_log"), + ("clone_fio_session", "clone_fio_log"), + ): + session = lvol_ctx[sess_key] + if self._is_tmux_running(cn, session): + self.logger.info(f" [{cn}] {session}: still running") + else: + self.logger.warning( + f" [{cn}] {session}: already finished — will check log" + ) + + self.logger.info("Step 12: Waiting for all fio sessions to complete") + for node_id in all_node_ids: + for lvol_ctx in node_ctx[node_id]["fio_lvols"]: + cn = lvol_ctx["client_node"] + for sess_key, log_key in ( + ("lvol_fio_session", "lvol_fio_log"), + ("clone_fio_session", "clone_fio_log"), + ): + self._wait_tmux_gone(cn, lvol_ctx[sess_key], timeout=3600) + self._assert_fio_log_clean(cn, lvol_ctx[log_key]) + else: + self.logger.info("Step 12: Skipping FIO wait (non-HA mode)") + + # ---------------------------------------------------------------- + # Step 13: Post-upgrade md5 check on verify clone mounts + # ---------------------------------------------------------------- + self.logger.info("Step 13: Post-upgrade md5 check on verify clones") + for node_id in all_node_ids: + host_ip = node_ctx[node_id]["host_ip"] + for lvol_ctx in node_ctx[node_id]["verify_lvols"]: + clone_mount = lvol_ctx["clone_mount"] + pre_clone_md5 = lvol_ctx["clone_md5"] + client_node = lvol_ctx["client_node"] + + files = self.ssh_obj.find_files(client_node, clone_mount) + post_md5 = self.ssh_obj.generate_checksums(client_node, files) + + assert set(pre_clone_md5.values()) == set(post_md5.values()), ( + f"[{node_id}@{host_ip}/{lvol_ctx['lvol_name']}] " + "Post-upgrade verify clone md5 mismatch!" + ) + + self.logger.info("TEST CASE PASSED !!!") diff --git a/e2e/scripts/cleanup_upgrade_test.sh b/e2e/scripts/cleanup_upgrade_test.sh new file mode 100644 index 0000000000..94e6f97072 --- /dev/null +++ b/e2e/scripts/cleanup_upgrade_test.sh @@ -0,0 +1,459 @@ +#!/usr/bin/env bash +# cleanup_upgrade_test.sh — Full cleanup of both R25 and R26 simplyblock setups +# +# Cleans up everything so the next upgrade test run starts completely fresh. +# Handles: Helm releases (R25 sbcli/spdk-csi + R26 operator), CRs, CRDs, +# kube-system leftovers, cert-manager, PVCs, PVs, snapshots, NVMe, hugepages, +# node labels, CSI hostpath data, and namespaces. +# +# Usage: +# ./cleanup_upgrade_test.sh [OPTIONS] +# +# Options: +# -n NAMESPACE Namespace (default: simplyblock) +# -e ENVIRONMENT Cluster environment: local|openshift-baremetal|openshift-local|aws-openshift|gcp +# (default: openshift-baremetal) +# -w WORKER_NODES Comma-separated worker node names +# (default: worker-0.ocp.simplyblock.ai,...,worker-5.ocp.simplyblock.ai) +# -c CRD_DIR Path to operator CRDs directory for deletion +# (default: searches common locations) +# -h Show this help message +# +# Examples: +# # Default (openshift-baremetal, 6 workers) +# ./cleanup_upgrade_test.sh +# +# # Custom namespace and environment +# ./cleanup_upgrade_test.sh -n simplyblock -e local -w "node1,node2,node3" + +set +e # Don't exit on errors — cleanup must be best-effort + +# ── Parse arguments ── +NAMESPACE="simplyblock" +CLUSTER_ENV="openshift-baremetal" +WORKER_NODES="worker-0.ocp.simplyblock.ai,worker-1.ocp.simplyblock.ai,worker-2.ocp.simplyblock.ai,worker-3.ocp.simplyblock.ai,worker-4.ocp.simplyblock.ai,worker-5.ocp.simplyblock.ai" +CRD_DIR="" + +while getopts "n:e:w:c:h" opt; do + case $opt in + n) NAMESPACE="$OPTARG" ;; + e) CLUSTER_ENV="$OPTARG" ;; + w) WORKER_NODES="$OPTARG" ;; + c) CRD_DIR="$OPTARG" ;; + h) + head -30 "$0" | grep '^#' | sed 's/^# \?//' + exit 0 + ;; + *) echo "Unknown option: -$opt" >&2; exit 1 ;; + esac +done + +KUBECTL_TIMEOUT="--request-timeout=120s" + +echo "============================================================" +echo " SimplyBlock Upgrade Test Cleanup" +echo " Namespace: $NAMESPACE" +echo " Environment: $CLUSTER_ENV" +echo " Workers: $WORKER_NODES" +echo "============================================================" +echo "" + +# Helper: retry a command up to N times with backoff +retry_cmd() { + local max_attempts=$1 + shift + local attempt=1 + while [ $attempt -le $max_attempts ]; do + if "$@" 2>/dev/null; then + return 0 + fi + echo " Attempt $attempt/$max_attempts failed, retrying in 5s..." + sleep 5 + attempt=$((attempt + 1)) + done + echo " All $max_attempts attempts failed for: $*" + return 1 +} + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Helm uninstall (R25 + R26 releases) +# ══════════════════════════════════════════════════════════════════ +echo "=== Phase 1: Uninstall all Helm releases (R25 + R26) ===" + +# R25 charts (explicit names) +echo " Uninstalling R25 charts (spdk-csi, sbcli)..." +helm uninstall spdk-csi -n $NAMESPACE --no-hooks --timeout 60s 2>/dev/null || true +helm uninstall sbcli -n $NAMESPACE --no-hooks --timeout 60s 2>/dev/null || true + +# R26 charts / any remaining releases +echo " Uninstalling any remaining Helm releases in $NAMESPACE..." +for rel in $(helm list -n $NAMESPACE -q 2>/dev/null); do + echo " Uninstalling: $rel" + helm uninstall "$rel" -n $NAMESPACE --no-hooks --timeout 60s 2>/dev/null || true +done + +# cert-manager +echo " Uninstalling cert-manager..." +helm uninstall cert-manager -n cert-manager --no-hooks --timeout 60s 2>/dev/null || true + +echo " Phase 1 complete." +echo "" + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Delete kube-system resources +# ══════════════════════════════════════════════════════════════════ +echo "=== Phase 2: Delete simplyblock resources in kube-system ===" + +# snapshot-controller and other simplyblock resources survive helm uninstall +# due to helm.sh/resource-policy: keep +for RTYPE in deployment daemonset service sa configmap; do + for NAME in $(kubectl -n kube-system $KUBECTL_TIMEOUT get $RTYPE --no-headers -o custom-columns=:metadata.name 2>/dev/null | grep -i simplyblock 2>/dev/null); do + echo " Deleting kube-system $RTYPE/$NAME" + kubectl -n kube-system $KUBECTL_TIMEOUT delete $RTYPE "$NAME" --ignore-not-found 2>/dev/null || true + done +done + +# Specifically clean numa-resource-plugin resources +kubectl -n kube-system $KUBECTL_TIMEOUT delete ds simplyblock-numa-resource-plugin --ignore-not-found 2>/dev/null || true +kubectl -n kube-system $KUBECTL_TIMEOUT delete sa simplyblock-numa-resource-plugin --ignore-not-found 2>/dev/null || true +kubectl -n kube-system $KUBECTL_TIMEOUT delete cm simplyblock-numa-resource-plugin-config --ignore-not-found 2>/dev/null || true +kubectl $KUBECTL_TIMEOUT delete clusterrole simplyblock-numa-resource-plugin --ignore-not-found 2>/dev/null || true +kubectl $KUBECTL_TIMEOUT delete clusterrolebinding simplyblock-numa-resource-plugin --ignore-not-found 2>/dev/null || true + +echo " Phase 2 complete." +echo "" + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Delete CRs (patch finalizers first) +# ══════════════════════════════════════════════════════════════════ +echo "=== Phase 3: Delete all simplyblock CRs ===" + +# All known CR types (both old and new API group names) +CR_TYPES=( + "simplyblockpool.storage.simplyblock.io" + "simplyblocklvol.storage.simplyblock.io" + "simplyblocktask.storage.simplyblock.io" + "simplyblockdevices.storage.simplyblock.io" + "simplyblockstoragenodes.storage.simplyblock.io" + "simplyblockstoragenodesets.storage.simplyblock.io" + "simplyblockstoragenodeops.storage.simplyblock.io" + "simplyblockstorageclusters.storage.simplyblock.io" + "simplyblocksnapshotreplications.storage.simplyblock.io" + "pool.storage.simplyblock.io" + "lvol.storage.simplyblock.io" + "task.storage.simplyblock.io" + "devices.storage.simplyblock.io" + "storagenodes.storage.simplyblock.io" + "storagenodesets.storage.simplyblock.io" + "storagenodeops.storage.simplyblock.io" + "storageclusters.storage.simplyblock.io" + "snapshotreplications.storage.simplyblock.io" + "storagebackups.storage.simplyblock.io" + "backuprestores.storage.simplyblock.io" + "backuppolicies.storage.simplyblock.io" + "backupimports.storage.simplyblock.io" +) + +for CR_TYPE in "${CR_TYPES[@]}"; do + for CR_NAME in $(kubectl -n $NAMESPACE $KUBECTL_TIMEOUT get "$CR_TYPE" --no-headers -o custom-columns=:metadata.name 2>/dev/null); do + kubectl -n $NAMESPACE $KUBECTL_TIMEOUT patch "$CR_TYPE" "$CR_NAME" \ + --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + kubectl -n $NAMESPACE $KUBECTL_TIMEOUT delete "$CR_TYPE" "$CR_NAME" \ + --ignore-not-found --wait=false 2>/dev/null || true + done +done + +# Also handle FDB CRs (from R25 sbcli chart) +for FDB_TYPE in foundationdbcluster foundationdbbackup foundationdbrestore; do + for CR_NAME in $(kubectl -n $NAMESPACE $KUBECTL_TIMEOUT get "$FDB_TYPE" --no-headers -o custom-columns=:metadata.name 2>/dev/null); do + kubectl -n $NAMESPACE $KUBECTL_TIMEOUT patch "$FDB_TYPE" "$CR_NAME" \ + --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + kubectl -n $NAMESPACE $KUBECTL_TIMEOUT delete "$FDB_TYPE" "$CR_NAME" \ + --ignore-not-found --wait=false 2>/dev/null || true + done +done + +echo " Phase 3 complete." +echo "" + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Delete VolumeSnapshots, PVCs, PVs +# ══════════════════════════════════════════════════════════════════ +echo "=== Phase 4: Delete VolumeSnapshots, PVCs, PVs ===" + +# VolumeSnapshots +echo " Bulk deleting VolumeSnapshots..." +retry_cmd 3 kubectl -n $NAMESPACE $KUBECTL_TIMEOUT delete volumesnapshot --all --wait=false + +VS_REMAINING=$(kubectl -n $NAMESPACE $KUBECTL_TIMEOUT get volumesnapshot --no-headers 2>/dev/null | wc -l) +if [ "${VS_REMAINING:-0}" -gt 0 ]; then + echo " $VS_REMAINING VolumeSnapshots stuck, patching finalizers..." + kubectl -n $NAMESPACE $KUBECTL_TIMEOUT get volumesnapshot --no-headers \ + -o custom-columns=:metadata.name 2>/dev/null | \ + xargs -P 20 -I {} kubectl -n $NAMESPACE $KUBECTL_TIMEOUT patch volumesnapshot {} \ + --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null + retry_cmd 3 kubectl -n $NAMESPACE $KUBECTL_TIMEOUT delete volumesnapshot --all \ + --force --grace-period=0 --wait=false +fi + +# VolumeSnapshotContents (cluster-scoped) +echo " Bulk deleting VolumeSnapshotContents..." +retry_cmd 3 kubectl $KUBECTL_TIMEOUT delete volumesnapshotcontent --all --wait=false + +VSC_REMAINING=$(kubectl $KUBECTL_TIMEOUT get volumesnapshotcontent --no-headers 2>/dev/null | wc -l) +if [ "${VSC_REMAINING:-0}" -gt 0 ]; then + echo " $VSC_REMAINING VolumeSnapshotContents stuck, patching finalizers..." + kubectl $KUBECTL_TIMEOUT get volumesnapshotcontent --no-headers \ + -o custom-columns=:metadata.name 2>/dev/null | \ + xargs -P 20 -I {} kubectl $KUBECTL_TIMEOUT patch volumesnapshotcontent {} \ + --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null + retry_cmd 3 kubectl $KUBECTL_TIMEOUT delete volumesnapshotcontent --all \ + --force --grace-period=0 --wait=false +fi + +# VolumeSnapshotClasses +for VSCLASS in $(kubectl $KUBECTL_TIMEOUT get volumesnapshotclass --no-headers -o custom-columns=:metadata.name 2>/dev/null); do + kubectl $KUBECTL_TIMEOUT delete volumesnapshotclass "$VSCLASS" --ignore-not-found 2>/dev/null || true +done + +# PVCs +echo " Deleting PVCs..." +retry_cmd 3 kubectl -n $NAMESPACE $KUBECTL_TIMEOUT delete pvc --all --wait=false +sleep 10 + +STUCK_PVCS=$(kubectl -n $NAMESPACE $KUBECTL_TIMEOUT get pvc --no-headers -o custom-columns=:metadata.name 2>/dev/null) +if [ -n "$STUCK_PVCS" ]; then + echo " Patching finalizers on stuck PVCs..." + echo "$STUCK_PVCS" | xargs -P 20 -I {} \ + kubectl -n $NAMESPACE $KUBECTL_TIMEOUT patch pvc {} \ + --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null + retry_cmd 3 kubectl -n $NAMESPACE $KUBECTL_TIMEOUT delete pvc --all \ + --force --grace-period=0 --wait=false +fi + +# PVs (skip vault PVs) +echo " Deleting PVs..." +PV_LIST=$(kubectl $KUBECTL_TIMEOUT get pv --no-headers -o custom-columns=NAME:.metadata.name,CLAIM:.spec.claimRef.namespace 2>/dev/null \ + | grep -v -E '\bvault\b' 2>/dev/null | awk '{print $1}') +if [ -n "$PV_LIST" ]; then + echo "$PV_LIST" | xargs -P 20 -I {} \ + kubectl $KUBECTL_TIMEOUT patch pv {} \ + --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null + echo "$PV_LIST" | xargs -P 20 -I {} \ + kubectl $KUBECTL_TIMEOUT delete pv {} --force --grace-period=0 --wait=false 2>/dev/null +fi + +# Also delete simplyblock-provisioned PVs +for pv in $(kubectl get pv --no-headers 2>/dev/null | grep 'simplyblock/' | awk '{print $1}'); do + kubectl delete pv "$pv" --ignore-not-found 2>/dev/null || true +done + +echo " Phase 4 complete." +echo "" + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: Force delete remaining namespaced resources +# ══════════════════════════════════════════════════════════════════ +echo "=== Phase 5: Force delete remaining namespaced resources ===" + +for RTYPE in pod jobs service ds statefulset deployment replicaset secret sa configmap; do + kubectl -n $NAMESPACE $KUBECTL_TIMEOUT delete $RTYPE --all --force --grace-period=0 2>/dev/null || true +done + +echo " Phase 5 complete." +echo "" + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Cleanup cluster-scoped resources +# ══════════════════════════════════════════════════════════════════ +echo "=== Phase 6: Delete cluster-scoped resources ===" + +# StorageClasses +for SC in $(kubectl $KUBECTL_TIMEOUT get sc --no-headers -o custom-columns=:metadata.name 2>/dev/null | grep -i simplyblock 2>/dev/null); do + kubectl $KUBECTL_TIMEOUT delete sc "$SC" --ignore-not-found 2>/dev/null || true +done + +# ClusterRoles and ClusterRoleBindings +kubectl $KUBECTL_TIMEOUT delete clusterrole simplyblock-storage-node-role --ignore-not-found 2>/dev/null || true +kubectl $KUBECTL_TIMEOUT delete clusterrolebinding simplyblock-storage-node-binding --ignore-not-found 2>/dev/null || true + +for RES in clusterrole clusterrolebinding; do + for NAME in $(kubectl $KUBECTL_TIMEOUT get $RES --no-headers -o custom-columns=:metadata.name 2>/dev/null | grep -i simplyblock 2>/dev/null); do + kubectl $KUBECTL_TIMEOUT delete "$RES" "$NAME" --ignore-not-found 2>/dev/null || true + done + # Also catch FDB-related cluster roles + for NAME in $(kubectl $KUBECTL_TIMEOUT get $RES --no-headers -o custom-columns=:metadata.name 2>/dev/null | grep -i "fdb-manager" 2>/dev/null); do + kubectl $KUBECTL_TIMEOUT delete "$RES" "$NAME" --ignore-not-found 2>/dev/null || true + done +done + +# Webhook configurations +for WH in $(kubectl $KUBECTL_TIMEOUT get mutatingwebhookconfiguration --no-headers -o custom-columns=:metadata.name 2>/dev/null | grep -i simplyblock 2>/dev/null); do + kubectl $KUBECTL_TIMEOUT delete mutatingwebhookconfiguration "$WH" --ignore-not-found 2>/dev/null || true +done +for WH in $(kubectl $KUBECTL_TIMEOUT get validatingwebhookconfiguration --no-headers -o custom-columns=:metadata.name 2>/dev/null | grep -i simplyblock 2>/dev/null); do + kubectl $KUBECTL_TIMEOUT delete validatingwebhookconfiguration "$WH" --ignore-not-found 2>/dev/null || true +done + +echo " Phase 6 complete." +echo "" + +# ══════════════════════════════════════════════════════════════════ +# Phase 7: Delete CRDs +# ══════════════════════════════════════════════════════════════════ +echo "=== Phase 7: Delete CRDs ===" + +# Clear finalizers on all CR instances first (so CRD deletion doesn't hang) +for crd in $(kubectl get crd -o name 2>/dev/null | grep simplyblock); do + crd_name="${crd#customresourcedefinition.apiextensions.k8s.io/}" + for cr_info in $(kubectl get "$crd_name" -A --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null); do + IFS=' ' read -r ns name <<< "$cr_info" + kubectl patch "$crd_name" "$name" -n "$ns" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + done + kubectl patch crd "$crd_name" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true +done + +# Delete CRDs from operator directory if available +if [ -n "$CRD_DIR" ] && [ -d "$CRD_DIR" ]; then + kubectl delete -f "$CRD_DIR" --ignore-not-found --timeout=60s 2>/dev/null || true +fi + +# Force-delete any remaining simplyblock CRDs +for crd in $(kubectl get crd -o name 2>/dev/null | grep simplyblock); do + crd_name="${crd#customresourcedefinition.apiextensions.k8s.io/}" + kubectl patch crd "$crd_name" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + kubectl delete crd "$crd_name" --ignore-not-found --timeout=30s 2>/dev/null || true +done + +# Delete FDB CRDs +for crd in $(kubectl get crd -o name 2>/dev/null | grep foundationdb); do + crd_name="${crd#customresourcedefinition.apiextensions.k8s.io/}" + kubectl patch crd "$crd_name" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + kubectl delete crd "$crd_name" --ignore-not-found --timeout=30s 2>/dev/null || true +done + +# Delete cert-manager CRDs +kubectl delete crd certificaterequests.cert-manager.io certificates.cert-manager.io \ + challenges.acme.cert-manager.io clusterissuers.cert-manager.io \ + issuers.cert-manager.io orders.acme.cert-manager.io \ + --ignore-not-found --force --grace-period=0 2>/dev/null || true + +echo " Phase 7 complete." +echo "" + +# ══════════════════════════════════════════════════════════════════ +# Phase 8: Delete + finalize namespaces +# ══════════════════════════════════════════════════════════════════ +echo "=== Phase 8: Delete namespaces ===" + +kubectl delete namespace $NAMESPACE --force --grace-period=0 --wait=false 2>/dev/null || true +kubectl delete namespace cert-manager --force --grace-period=0 --wait=false 2>/dev/null || true + +# Wait for namespace deletion with force-finalize +for i in $(seq 1 12); do + REMAINING=0 + for ns in $NAMESPACE cert-manager; do + if kubectl get namespace $ns &>/dev/null; then + REMAINING=$((REMAINING + 1)) + echo " Namespace $ns still terminating ($i/12), force-finalizing..." + kubectl get namespace $ns -o json 2>/dev/null \ + | jq '.spec.finalizers = []' \ + | kubectl replace --raw "/api/v1/namespaces/$ns/finalize" -f - 2>/dev/null || true + fi + done + [ "$REMAINING" -eq 0 ] && echo " All namespaces deleted" && break + sleep 5 +done + +echo " Phase 8 complete." +echo "" + +# ══════════════════════════════════════════════════════════════════ +# Phase 9: Node-level cleanup (NVMe, hugepages, kubelet) +# ══════════════════════════════════════════════════════════════════ +echo "=== Phase 9: NVMe disconnect, hugepages reset, kubelet restart ===" + +IFS=',' read -ra NODES <<< "$WORKER_NODES" +for NODE in "${NODES[@]}"; do + ( + echo " Cleaning node: $NODE" + if [[ "$CLUSTER_ENV" == *"openshift"* ]]; then + timeout 90 oc debug node/"$NODE" -- chroot /host bash -c \ + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages && systemctl restart kubelet" 2>/dev/null || true + else + timeout 90 kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ + "nvme disconnect-all 2>/dev/null; echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages && systemctl restart kubelet" 2>/dev/null || true + fi + echo " Done: $NODE" + ) & +done +wait + +echo " Phase 9 complete." +echo "" + +# ══════════════════════════════════════════════════════════════════ +# Phase 10: Remove stale node labels +# ══════════════════════════════════════════════════════════════════ +echo "=== Phase 10: Remove stale node labels ===" + +for NODE in "${NODES[@]}"; do + kubectl label node "$NODE" io.simplyblock.storagenodeset- 2>/dev/null || true + kubectl label node "$NODE" io.simplyblock.node-type- 2>/dev/null || true + echo " Removed labels from $NODE" +done + +echo " Phase 10 complete." +echo "" + +# ══════════════════════════════════════════════════════════════════ +# Phase 11: Cleanup stale CSI hostpath data +# ══════════════════════════════════════════════════════════════════ +echo "=== Phase 11: Cleanup stale CSI hostpath data ===" + +for NODE in "${NODES[@]}"; do + echo " Cleaning CSI hostpath data on $NODE..." + if [[ "$CLUSTER_ENV" == *"openshift"* ]]; then + oc debug node/"$NODE" -- chroot /host bash -c \ + "find /var/lib/csi-hostpath-data -mindepth 1 -maxdepth 1 -type d -mtime +2 -exec rm -rf {} \;" 2>/dev/null || true + else + kubectl debug node/"$NODE" -q --image=busybox:latest -- chroot /host sh -c \ + "find /var/lib/csi-hostpath-data -mindepth 1 -maxdepth 1 -type d -mtime +2 -exec rm -rf {} \;" 2>/dev/null || true + fi +done + +echo " Phase 11 complete." +echo "" + +# ══════════════════════════════════════════════════════════════════ +# Phase 12: Final verification +# ══════════════════════════════════════════════════════════════════ +echo "=== Phase 12: Final verification ===" + +echo " Namespaces:" +kubectl get namespace $NAMESPACE 2>/dev/null && echo " WARNING: $NAMESPACE still exists!" || echo " $NAMESPACE: gone" +kubectl get namespace cert-manager 2>/dev/null && echo " WARNING: cert-manager still exists!" || echo " cert-manager: gone" + +echo "" +echo " Helm releases:" +helm list -n $NAMESPACE 2>/dev/null || echo " None" + +echo "" +echo " SimplyBlock CRDs:" +kubectl get crd 2>/dev/null | grep -i simplyblock || echo " None" + +echo "" +echo " FDB CRDs:" +kubectl get crd 2>/dev/null | grep -i foundationdb || echo " None" + +echo "" +echo " kube-system simplyblock resources:" +for RTYPE in deployment daemonset service sa configmap; do + kubectl -n kube-system get $RTYPE --no-headers 2>/dev/null | grep -i simplyblock || true +done +echo " (empty = clean)" + +echo "" +echo "============================================================" +echo " Cleanup complete!" +echo "============================================================" diff --git a/e2e/stress_test/continuous_k8s_native_failover.py b/e2e/stress_test/continuous_k8s_native_failover.py index 8333926ba1..6f684535d6 100644 --- a/e2e/stress_test/continuous_k8s_native_failover.py +++ b/e2e/stress_test/continuous_k8s_native_failover.py @@ -2973,6 +2973,37 @@ def wait_for_fio_complete(self, timeout: int = None) -> set[str]: f"as failed (likely volume mount " f"or image pull failure)" ) + # Capture kubectl describe pod for + # debugging why the pod is stuck + try: + ns = self.namespace + desc_out, _ = self.k8s_utils._exec_kubectl( + f"kubectl describe pod {pod_name}" + f" -n {ns}", + timeout=30, + ) + if desc_out: + desc_dir = os.path.join( + self.docker_logs_path, + "stuck_pod_describes", + ) + os.makedirs(desc_dir, exist_ok=True) + desc_file = os.path.join( + desc_dir, + f"{pod_name}_describe.txt", + ) + with open(desc_file, "w") as f: + f.write(desc_out) + self.logger.info( + f"[wait_fio] Saved describe" + f" for stuck pod {pod_name}" + f" → {desc_file}" + ) + except Exception as e: + self.logger.warning( + f"[wait_fio] Failed to describe" + f" stuck pod {pod_name}: {e}" + ) still_running.discard(job_name) stuck_init_since.pop(job_name, None) failed_jobs.add(job_name) @@ -2995,6 +3026,38 @@ def wait_for_fio_complete(self, timeout: int = None) -> set[str]: f"[wait_fio] {len(still_running)} jobs did not complete " f"within {timeout}s: {sorted(still_running)}" ) + # Capture kubectl describe for timed-out pods + for job_name in still_running: + try: + pod_name = self.k8s_utils.get_job_pod_name(job_name) + if not pod_name: + continue + ns = self.namespace + desc_out, _ = self.k8s_utils._exec_kubectl( + f"kubectl describe pod {pod_name} -n {ns}", + timeout=30, + ) + if desc_out: + desc_dir = os.path.join( + self.docker_logs_path, + "stuck_pod_describes", + ) + os.makedirs(desc_dir, exist_ok=True) + desc_file = os.path.join( + desc_dir, + f"{pod_name}_describe.txt", + ) + with open(desc_file, "w") as f: + f.write(desc_out) + self.logger.info( + f"[wait_fio] Saved describe for " + f"timed-out pod {pod_name} → {desc_file}" + ) + except Exception as e: + self.logger.warning( + f"[wait_fio] Failed to describe " + f"timed-out pod for {job_name}: {e}" + ) failed_jobs.update(still_running) if failed_jobs: self.logger.error( diff --git a/e2e/stress_test/mass_create_delete_stress.py b/e2e/stress_test/mass_create_delete_stress.py index 4598b19123..572cd3c080 100644 --- a/e2e/stress_test/mass_create_delete_stress.py +++ b/e2e/stress_test/mass_create_delete_stress.py @@ -124,6 +124,9 @@ class _MassCreateDeleteMixin: # max_lvols = MAX_ENTITY_COUNT // (1 + SNAPSHOTS_PER_LVOL) MAX_ENTITY_COUNT = 0 + # ── API hard limit for namespaces per subsystem ────────────────────── + MAX_NS_PER_SUBSYSTEM = 50 + # ── Persistent retry mode ───────────────────────────────────────────── # Subclasses set PERSISTENT_RETRY = True to retry failed items until # all expected entities are created or a terminal error is hit. @@ -565,6 +568,20 @@ def _run_mass_create_delete_test(self): self.NS_PER_SUBSYSTEM = effective_per_sub self.NUM_SUBSYSTEMS = effective_num_sub + # Enforce API hard limit: max_namespace_per_subsys <= MAX_NS_PER_SUBSYSTEM + if self.NS_PER_SUBSYSTEM > self.MAX_NS_PER_SUBSYSTEM: + total_lvols = self.NUM_SUBSYSTEMS * self.NS_PER_SUBSYSTEM + new_num_sub = math.ceil(total_lvols / self.MAX_NS_PER_SUBSYSTEM) + new_per_sub = total_lvols // new_num_sub + self.logger.info( + f"[NS limit] NS_PER_SUBSYSTEM={self.NS_PER_SUBSYSTEM} exceeds " + f"hard limit of {self.MAX_NS_PER_SUBSYSTEM}. Redistributing " + f"{total_lvols} lvols: {self.NUM_SUBSYSTEMS}x{self.NS_PER_SUBSYSTEM} " + f"→ {new_num_sub}x{new_per_sub}" + ) + self.NS_PER_SUBSYSTEM = new_per_sub + self.NUM_SUBSYSTEMS = new_num_sub + total = self.NUM_SUBSYSTEMS * self.NS_PER_SUBSYSTEM max_dur = getattr(self, 'MAX_TEST_DURATION', 6 * 3600) max_dur_h = round(max_dur / 3600, 1) @@ -963,6 +980,452 @@ def _phase_node_outage(self, label: str, exclude_host_ip: str | None = None): self._phase_durations[label] = total_dur return node_ip + # ── Rapid restart cycle (30x stop/restart, no migration wait) ──────── + + RAPID_RESTART_ITERATIONS = 30 + RAPID_RESTART_COOLDOWN = 60 # seconds to wait after node is online + + def _pick_node_with_entries(self): + """Pick a storage node that hosts lvols/snapshots (has entries). + + Queries storage nodes and returns the first non-secondary, online + node. With thousands of entities spread across nodes, all nodes + should have entries. + """ + nodes = self.sbcli_utils.get_storage_nodes().get("results", []) + for n in nodes: + if n.get("is_secondary_node"): + continue + if n.get("status") == "online": + return n + # fallback: any non-secondary node + for n in nodes: + if not n.get("is_secondary_node"): + return n + return nodes[0] if nodes else None + + def _phase_rapid_restart_cycles(self, label: str, + iterations: int | None = None): + """Stop and restart a storage node *iterations* times in a row. + + Does NOT wait for migration/rebalancing to complete — only waits + for the node status to return to ``online``, then sleeps + ``RAPID_RESTART_COOLDOWN`` seconds before the next iteration. + + Returns a list of dicts, one per iteration: + [{iteration, stop_time_iso, online_time_iso, + stop_to_online_sec, cooldown_sec}, ...] + """ + if iterations is None: + iterations = self.RAPID_RESTART_ITERATIONS + cooldown = self.RAPID_RESTART_COOLDOWN + + target = self._pick_node_with_entries() + if not target: + self.logger.warning( + f"[{label}] No storage nodes found — skipping rapid restart" + ) + return [] + + node_uuid = target["id"] + node_ip = target["mgmt_ip"] + rpc_port = target.get("rpc_port", 0) + + self.logger.info( + f"[{label}] Starting {iterations} rapid restart cycles on " + f"node {node_uuid} ({node_ip}), " + f"cooldown={cooldown}s between iterations" + ) + + results = [] + for i in range(1, iterations + 1): + self.logger.info( + f"[{label}] --- Iteration {i}/{iterations} ---" + ) + + # Stop container / delete pod + stop_ts = datetime.now(timezone.utc) + t_stop = time.time() + if getattr(self, "k8s_test", False): + self.k8s_utils.restart_spdk_pod(node_ip) + else: + self.ssh_obj.stop_spdk_process( + node_ip, rpc_port, self.cluster_id + ) + kill_dur = round(time.time() - t_stop, 1) + self.logger.info( + f"[{label}][{i}] Kill command took {kill_dur}s" + ) + + # Wait for node to go offline then back online + try: + self.sbcli_utils.wait_for_storage_node_status( + node_uuid, ["offline", "unreachable"], timeout=600, + ) + except Exception as exc: + self.logger.warning( + f"[{label}][{i}] Timed out waiting for " + f"offline/unreachable: {exc}" + ) + + self.sbcli_utils.wait_for_storage_node_status( + node_uuid, "online", timeout=900, + ) + online_ts = datetime.now(timezone.utc) + stop_to_online = round(time.time() - t_stop, 1) + + self.logger.info( + f"[{label}][{i}] Node online after {stop_to_online}s " + f"— cooling down {cooldown}s" + ) + + results.append({ + "iteration": i, + "stop_time_iso": stop_ts.isoformat(), + "online_time_iso": online_ts.isoformat(), + "stop_to_online_sec": stop_to_online, + "cooldown_sec": cooldown, + }) + + # Cooldown — no waiting for migration, just a short pause + time.sleep(cooldown) + + total_dur = sum(r["stop_to_online_sec"] for r in results) + self._phase_durations[label] = round(total_dur, 1) + self.logger.info( + f"[{label}] Completed {iterations} restart cycles, " + f"cumulative stop-to-online: {total_dur}s" + ) + return results + + # ── Rapid restart test orchestrator ────────────────────────────────── + + def _run_mass_create_rapid_restart_test(self): + """Create lvols+snapshots, run 30 rapid restarts, then + delete lvols + create clones, run 30 more rapid restarts. + + The final summary prints per-iteration stop-to-online times + for both phases (60 entries total). + """ + self._init_mixin_state() + self._rapid_restart_results = { + "lvol_snapshot": [], + "snapshot_clone": [], + } + + original_total = self.NUM_SUBSYSTEMS * self.NS_PER_SUBSYSTEM + if self.MAX_ENTITY_COUNT > 0: + max_lvols = self.MAX_ENTITY_COUNT // (1 + self.SNAPSHOTS_PER_LVOL) + if max_lvols < original_total: + self.logger.info( + f"[Entity cap] Reducing lvol target from " + f"{original_total} to {max_lvols} " + f"(MAX_ENTITY_COUNT={self.MAX_ENTITY_COUNT}, " + f"SNAPSHOTS_PER_LVOL={self.SNAPSHOTS_PER_LVOL})" + ) + effective_per_sub = max(1, max_lvols // self.NUM_SUBSYSTEMS) + effective_num_sub = self.NUM_SUBSYSTEMS + if effective_per_sub == 1 and max_lvols < self.NUM_SUBSYSTEMS: + effective_num_sub = max_lvols + self.NS_PER_SUBSYSTEM = effective_per_sub + self.NUM_SUBSYSTEMS = effective_num_sub + + # Enforce API hard limit: max_namespace_per_subsys <= MAX_NS_PER_SUBSYSTEM + if self.NS_PER_SUBSYSTEM > self.MAX_NS_PER_SUBSYSTEM: + total_lvols = self.NUM_SUBSYSTEMS * self.NS_PER_SUBSYSTEM + new_num_sub = math.ceil(total_lvols / self.MAX_NS_PER_SUBSYSTEM) + new_per_sub = total_lvols // new_num_sub + self.logger.info( + f"[NS limit] NS_PER_SUBSYSTEM={self.NS_PER_SUBSYSTEM} exceeds " + f"hard limit of {self.MAX_NS_PER_SUBSYSTEM}. Redistributing " + f"{total_lvols} lvols: {self.NUM_SUBSYSTEMS}x{self.NS_PER_SUBSYSTEM} " + f"→ {new_num_sub}x{new_per_sub}" + ) + self.NS_PER_SUBSYSTEM = new_per_sub + self.NUM_SUBSYSTEMS = new_num_sub + + total = self.NUM_SUBSYSTEMS * self.NS_PER_SUBSYSTEM + self.logger.info( + f"=== Starting {self.__class__.__name__}: " + f"{total} lvols, {self.SNAPSHOTS_PER_LVOL} snaps/lvol, " + f"{self.RAPID_RESTART_ITERATIONS} restart cycles per phase ===" + ) + + # Start periodic kubectl resource collection + periodic_stop = self.start_periodic_resource_collection(interval=1800) + + try: + # Phase 1: Create lvols + t0 = time.time() + self._phase_1_create_lvols() + self._phase_durations["1_create_lvols"] = round( + time.time() - t0, 1 + ) + self._metrics["lvols_created"] = len(self._lvol_registry) + self.logger.info( + f"[Phase 1] {len(self._lvol_registry)} lvols created " + f"in {self._phase_durations['1_create_lvols']}s" + ) + if not self._lvol_registry: + raise RuntimeError("No lvols created — cannot proceed") + + # Phase 2: FIO on sampled lvols + t0 = time.time() + self._phase_2_fio_on_lvols() + self._phase_durations["2_fio_lvols"] = round( + time.time() - t0, 1 + ) + self.logger.info( + f"[Phase 2] FIO started on " + f"{self._metrics['fio_lvol_started']} lvols" + ) + + # Wait for FIO to finish before snapshots + if self._fio_lvol_threads: + fio_timeout = self.FIO_RUNTIME + 120 + for t in self._fio_lvol_threads: + t.join(timeout=fio_timeout) + + if hasattr(self, '_collect_fio_logs_to_nfs'): + self._collect_fio_logs_to_nfs( + label="Phase_2", threads=self._fio_lvol_threads, + ) + + if hasattr(self, '_catchup_newly_bound_pvcs'): + self._catchup_newly_bound_pvcs( + label="pre-Phase 3 catch-up" + ) + + # Phase 3: Create snapshots + t0 = time.time() + self._phase_3_create_snapshots() + self._phase_durations["3_create_snapshots"] = round( + time.time() - t0, 1 + ) + + if hasattr(self, '_verify_snapshots_ready'): + self._verify_snapshots_ready() + if hasattr(self, '_verify_backend_snapshot_count'): + self._verify_backend_snapshot_count( + len(self._snapshot_registry) + ) + + self._metrics["snapshots_created"] = len(self._snapshot_registry) + self.logger.info( + f"[Phase 3] {len(self._snapshot_registry)} snapshots " + f"created in {self._phase_durations['3_create_snapshots']}s" + ) + + # ── Phase 3r: 30x rapid restart (lvol + snapshot) ──────── + self.logger.info( + "=" * 60 + + "\n Phase 3r: RAPID RESTART — lvol + snapshot phase" + + "\n" + "=" * 60 + ) + self._rapid_restart_results["lvol_snapshot"] = ( + self._phase_rapid_restart_cycles( + "3r_rapid_restart_lvol_snap", + self.RAPID_RESTART_ITERATIONS, + ) + ) + + # Phase 4: Delete lvols (orphan snapshots for cloning) + t0 = time.time() + self._phase_4_delete_lvols() + self._phase_durations["4_delete_lvols"] = round( + time.time() - t0, 1 + ) + self.logger.info( + f"[Phase 4] Lvols deleted " + f"in {self._phase_durations['4_delete_lvols']}s" + ) + + # Phase 5: Create clones from orphaned snapshots + t0 = time.time() + self._phase_5_create_clones() + self._phase_durations["5_create_clones"] = round( + time.time() - t0, 1 + ) + self._metrics["clones_created"] = len(self._clone_registry) + self.logger.info( + f"[Phase 5] {len(self._clone_registry)} clones " + f"in {self._phase_durations['5_create_clones']}s" + ) + + # ── Phase 5r: 30x rapid restart (snapshot + clone) ────── + self.logger.info( + "=" * 60 + + "\n Phase 5r: RAPID RESTART — snapshot + clone phase" + + "\n" + "=" * 60 + ) + self._rapid_restart_results["snapshot_clone"] = ( + self._phase_rapid_restart_cycles( + "5r_rapid_restart_snap_clone", + self.RAPID_RESTART_ITERATIONS, + ) + ) + + # Phase 7: Delete clones + t0 = time.time() + self._phase_7_delete_clones() + self._phase_durations["7_delete_clones"] = round( + time.time() - t0, 1 + ) + + # Phase 8: Delete snapshots + t0 = time.time() + self._phase_8_delete_snapshots() + self._phase_durations["8_delete_snapshots"] = round( + time.time() - t0, 1 + ) + + finally: + periodic_stop.set() + try: + self.collect_management_details( + suffix="_pre_delete" + ) + except Exception as exc: + self.logger.warning( + f"Failed to collect management details: {exc}" + ) + _has_failure = ( + bool(self._soft_failures) + or sys.exc_info()[1] is not None + ) + if _has_failure and getattr( + self, 'preserve_resources_on_failure', False + ): + self.logger.info( + "[cleanup] Skipping cleanup — " + "preserve_resources_on_failure is set" + ) + else: + t0 = time.time() + self._phase_cleanup() + self._phase_durations["cleanup"] = round( + time.time() - t0, 1 + ) + self._print_rapid_restart_summary() + self._write_rapid_restart_json() + + if self._soft_failures: + for f in self._soft_failures: + self.logger.error(f"SOFT FAILURE: {f}") + raise AssertionError( + f"Test had {len(self._soft_failures)} soft failures: " + + "; ".join(self._soft_failures) + ) + + # ── Rapid restart summary ─────────────────────────────────────────── + + def _print_rapid_restart_summary(self): + total = self.NUM_SUBSYSTEMS * self.NS_PER_SUBSYSTEM + self.logger.info("=" * 75) + self.logger.info( + " MASS CREATE + RAPID RESTART STRESS TEST — SUMMARY" + ) + self.logger.info("=" * 75) + self.logger.info( + f" Config: {total} lvols, " + f"{self.SNAPSHOTS_PER_LVOL} snaps/lvol, " + f"cooldown={self.RAPID_RESTART_COOLDOWN}s" + ) + self.logger.info( + f" Lvols created: {self._metrics['lvols_created']}" + ) + self.logger.info( + f" Snaps created: {self._metrics['snapshots_created']}" + ) + self.logger.info( + f" Clones created: {self._metrics['clones_created']}" + ) + + for phase, dur in self._phase_durations.items(): + if "rapid_restart" not in phase: + self.logger.info(f" Phase {phase:30s}: {dur}s") + + # Per-iteration restart timing tables + for phase_label, display_name in [ + ("lvol_snapshot", "LVOL + SNAPSHOT"), + ("snapshot_clone", "SNAPSHOT + CLONE"), + ]: + entries = self._rapid_restart_results.get(phase_label, []) + self.logger.info("") + self.logger.info("-" * 75) + self.logger.info( + f" RESTART TIMINGS — {display_name} " + f"({len(entries)} iterations)" + ) + self.logger.info("-" * 75) + self.logger.info( + f" {'#':>3s} {'Stop Time (UTC)':>25s} " + f"{'Online Time (UTC)':>25s} " + f"{'Stop→Online (s)':>16s}" + ) + self.logger.info( + f" {'---':>3s} {'-' * 25:>25s} " + f"{'-' * 25:>25s} " + f"{'-' * 16:>16s}" + ) + for e in entries: + self.logger.info( + f" {e['iteration']:3d} " + f"{e['stop_time_iso'][:25]:>25s} " + f"{e['online_time_iso'][:25]:>25s} " + f"{e['stop_to_online_sec']:16.1f}" + ) + if entries: + times = [e["stop_to_online_sec"] for e in entries] + self.logger.info( + f" MIN: {min(times):.1f}s " + f"MAX: {max(times):.1f}s " + f"SUM: {sum(times):.1f}s" + ) + + total_dur = sum(self._phase_durations.values()) + self.logger.info("") + self.logger.info(f" Total test duration: {total_dur:.1f}s") + self.logger.info("=" * 75) + + def _write_rapid_restart_json(self): + total_dur = sum(self._phase_durations.values()) + report = { + "test_class": self.__class__.__name__, + "timestamp": datetime.now(timezone.utc).isoformat(), + "status": "passed", + "config": { + "num_subsystems": self.NUM_SUBSYSTEMS, + "ns_per_subsystem": self.NS_PER_SUBSYSTEM, + "snapshots_per_lvol": self.SNAPSHOTS_PER_LVOL, + "rapid_restart_iterations": + self.RAPID_RESTART_ITERATIONS, + "rapid_restart_cooldown_sec": + self.RAPID_RESTART_COOLDOWN, + "max_entity_count": self.MAX_ENTITY_COUNT, + }, + "phases": { + name: dur + for name, dur in self._phase_durations.items() + }, + "metrics": self._metrics, + "rapid_restart_lvol_snapshot": + self._rapid_restart_results.get("lvol_snapshot", []), + "rapid_restart_snapshot_clone": + self._rapid_restart_results.get("snapshot_clone", []), + "summary": { + "total_duration_sec": round(total_dur, 2), + }, + } + out_dir = Path("logs") + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / "mass_create_rapid_restart_timing.json" + with open(out_path, "w") as f: + _json.dump(report, f, indent=2) + self.logger.info( + f"Rapid restart timing JSON written to {out_path}" + ) + # ── Summary ──────────────────────────────────────────────────────────── def _print_summary(self): @@ -4597,3 +5060,88 @@ class MassCreateDeleteRestart_300x10_10Snap_K8s(_MassCreateDeleteK8s): NUM_SUBSYSTEMS = 10 NS_PER_SUBSYSTEM = 300 SNAPSHOTS_PER_LVOL = 10 + + +# ───────────────────────────────────────────────────────────────────────────── +# Rapid-restart variants: 6000 entity cap, 3 snaps/lvol (1:3 ratio), +# 30 container stop/restart cycles after lvol+snapshot creation, then 30 +# more after clone creation. +# +# Entity cap 6000 with SNAPSHOTS_PER_LVOL=3 → max_lvols = 6000 // 4 = 1500 +# Peak: 1500 lvols + 4500 snapshots = 6000 entities. +# +# Each restart cycle: +# 1. Stop container / delete pod (target node with entries) +# 2. Wait for node → online (no wait for migration/rebalancing) +# 3. Wait 60s cooldown +# +# Summary output: per-iteration stop→online time for both phases +# (30 entries for lvol+snapshot, 30 entries for snapshot+clone = 60 total). +# ───────────────────────────────────────────────────────────────────────────── + +# Docker rapid-restart variants + + +class MassCreateRapidRestart_6k_3Snap_Docker(_MassCreateDeleteDocker): + """6000 entity cap, 1:3 ratio (3 snaps/lvol → 1500 lvols + 4500 snaps), + 30 rapid container stop/restart cycles per phase.""" + PERSISTENT_RETRY = True + MAX_ENTITY_COUNT = 6000 + NUM_SUBSYSTEMS = 30 + NS_PER_SUBSYSTEM = 50 + SNAPSHOTS_PER_LVOL = 3 + RAPID_RESTART_ITERATIONS = 30 + RAPID_RESTART_COOLDOWN = 60 + + def run(self): + actual_pool = self.sbcli_utils.add_storage_pool( + pool_name=self.pool_name + ) + if actual_pool and actual_pool != self.pool_name: + self.pool_name = actual_pool + storage_nodes = self.sbcli_utils.get_storage_nodes() + for result in storage_nodes["results"]: + self.sn_nodes.append(result["uuid"]) + self._run_mass_create_rapid_restart_test() + + +# K8s rapid-restart variants + + +class MassCreateRapidRestart_6k_3Snap_K8s(_MassCreateDeleteK8s): + """6000 entity cap, 1:3 ratio (3 snaps/lvol → 1500 PVCs + 4500 snaps), + 30 rapid pod delete/restart cycles per phase.""" + PERSISTENT_RETRY = True + MAX_ENTITY_COUNT = 6000 + NUM_SUBSYSTEMS = 30 + NS_PER_SUBSYSTEM = 50 + SNAPSHOTS_PER_LVOL = 3 + RAPID_RESTART_ITERATIONS = 30 + RAPID_RESTART_COOLDOWN = 60 + + def run(self): + storage_nodes = self.sbcli_utils.get_storage_nodes() + for result in storage_nodes["results"]: + self.sn_nodes.append(result["uuid"]) + self.node_vs_pvc[result["uuid"]] = [] + + actual_pool = self.sbcli_utils.add_storage_pool( + pool_name=self.pool_name + ) + if actual_pool and actual_pool != self.pool_name: + self.pool_name = actual_pool + + cluster_id = self.cluster_id or os.environ.get("CLUSTER_ID", "") + self.k8s_utils.create_storage_class( + name=self.STORAGE_CLASS_NAME, + cluster_id=cluster_id, + pool_name=self.pool_name, + ndcs=self.ndcs, + npcs=self.npcs, + max_namespace_per_subsys=self.NS_PER_SUBSYSTEM, + ) + self.k8s_utils.create_volume_snapshot_class( + name=self.SNAPSHOT_CLASS_NAME, + ) + + self._run_mass_create_rapid_restart_test() diff --git a/e2e/upgrade_e2e.py b/e2e/upgrade_e2e.py index 3bba1b536e..bf90b2226c 100644 --- a/e2e/upgrade_e2e.py +++ b/e2e/upgrade_e2e.py @@ -20,6 +20,9 @@ def main(): parser.add_argument('--run_k8s', type=bool, help="Run K8s tests", default=False) parser.add_argument('--send_debug_notification', type=bool, help="Send notification for debug", default=False) parser.add_argument('--testname', type=str, help="The name of the test to run", default=None) + parser.add_argument('--preserve_resources_on_failure', type=bool, + help="Skip K8s resource cleanup when test fails (preserve PVCs/pods for debugging)", + default=True) args = parser.parse_args() @@ -45,7 +48,8 @@ def main(): target_spdk_image=args.target_spdk_image, target_docker_image=args.target_docker_image, fio_debug=args.fio_debug, - k8s_run=args.run_k8s) + k8s_run=args.run_k8s, + preserve_resources_on_failure=args.preserve_resources_on_failure) try: test_obj.setup() if i == 0: @@ -56,25 +60,37 @@ def main(): except Exception as exp: logger.error(traceback.format_exc()) errors[f"{test.__name__}"] = [exp] + stop_after_teardown = True + else: + stop_after_teardown = False try: if not args.run_k8s: test_obj.stop_docker_logs_collect() else: test_obj.stop_k8s_log_collect() test_obj.fetch_all_nodes_distrib_log() - if i == (len(test_class_run) - 1) or check_for_dumps(): + if i == (len(test_class_run) - 1) or stop_after_teardown or check_for_dumps(): test_obj.collect_management_details() if not args.run_k8s: all_nodes = test_obj._get_all_nodes() test_obj.ssh_obj.collect_final_docker_logs_simple(all_nodes, test_obj.docker_logs_path) test_obj.export_graylog_logs() test_obj.extract_delay_qpair_logs() - test_obj.teardown() - # pass + _skip_k8s = stop_after_teardown and test_obj.preserve_resources_on_failure + if _skip_k8s: + logger.info( + f"[cleanup] Test {test.__name__} failed — preserving K8s " + "resources for debugging (--preserve_resources_on_failure)" + ) + test_obj.teardown(skip_k8s_cleanup=_skip_k8s) except Exception as _: logger.error(f"Error During Teardown for test: {test.__name__}") logger.error(traceback.format_exc()) finally: + if stop_after_teardown: + logger.info("Previous test failed. " + "Cannot execute more upgrade tests as cluster state is unknown. Exiting") + break if check_for_dumps(): logger.info("Found a core dump during test execution. " "Cannot execute more tests as cluster is not stable. Exiting") diff --git a/e2e/utils/k8s_utils.py b/e2e/utils/k8s_utils.py index a120c31110..23aee36f35 100644 --- a/e2e/utils/k8s_utils.py +++ b/e2e/utils/k8s_utils.py @@ -140,10 +140,22 @@ def exec_sbcli(self, command: str, supress_logs: bool = False): ) stdout, stderr = self._exec_kubectl(kubectl_cmd, supress_logs=supress_logs) - # If the admin pod was recreated (e.g. during outage), retry with fresh pod - if "NotFound" in (stderr or ""): + # If the admin pod was recreated (e.g. during upgrade), retry with + # a freshly-resolved pod. kubectl may report different error strings + # depending on the phase of termination: + # - "NotFound" (pod fully deleted) + # - "unable to upgrade connection: pod does not exist" + # - "pod … not found" + _err = stderr or "" + _pod_gone = ( + "NotFound" in _err + or "pod does not exist" in _err + or "pod not found" in _err.lower() + ) + if _pod_gone: self.logger.warning( - f"[K8sUtils] Admin pod '{admin_pod}' not found, re-resolving..." + f"[K8sUtils] Admin pod '{admin_pod}' gone ({_err.strip()[:80]}), " + "re-resolving..." ) admin_pod = self.get_admin_pod(refresh=True) kubectl_cmd = ( @@ -1543,8 +1555,28 @@ def log_fio_pvc_mapping(self, pvc_details: dict, clone_details: dict = None, def create_volume_snapshot(self, name: str, pvc_name: str, snapshot_class: str = "simplyblock-csi-snapshotclass", namespace: str = None): - """Create a VolumeSnapshot from a PVC.""" + """Create a VolumeSnapshot from a PVC. + + If a stale VolumeSnapshot with the same name already exists (e.g. + from a previous test run that did not clean up), it is deleted + first to avoid ``persistentVolumeClaimName is immutable`` errors + from ``kubectl apply``. + """ ns = namespace or self.namespace + # Remove any stale VolumeSnapshot with the same name to avoid + # immutable-field collisions from a prior test. + existing = self.get_resource_json("volumesnapshot", name, namespace=ns) + if existing: + existing_pvc = (existing.get("spec", {}) + .get("source", {}) + .get("persistentVolumeClaimName", "")) + if existing_pvc != pvc_name: + self.logger.warning( + f"[K8sUtils] Stale VolumeSnapshot '{name}' found " + f"(source PVC '{existing_pvc}' != '{pvc_name}'), " + f"deleting before re-creating" + ) + self.delete_volume_snapshot(name, namespace=ns, wait=True) yaml_content = ( f"apiVersion: snapshot.storage.k8s.io/v1\n" f"kind: VolumeSnapshot\n" @@ -1584,11 +1616,23 @@ def wait_volume_snapshot_ready(self, name: str, timeout: int = 300, f"[K8sUtils] VolumeSnapshot '{name}' not ready within {timeout}s" ) - def delete_volume_snapshot(self, name: str, namespace: str = None): - """Delete a VolumeSnapshot.""" + def delete_volume_snapshot(self, name: str, namespace: str = None, + wait: bool = False): + """Delete a VolumeSnapshot. + + When *wait* is True the call blocks until the VolumeSnapshot is fully + removed, preventing stale-object collisions in subsequent tests. + """ ns = namespace or self.namespace - self.logger.info(f"[K8sUtils] Deleting VolumeSnapshot '{name}'") - self.delete_resource("volumesnapshot", name, namespace=ns) + self.logger.info(f"[K8sUtils] Deleting VolumeSnapshot '{name}'" + f"{' (waiting)' if wait else ''}") + if wait: + self._exec_kubectl( + f"kubectl delete volumesnapshot {name} -n {ns} " + f"--ignore-not-found --wait=true --timeout=120s" + ) + else: + self.delete_resource("volumesnapshot", name, namespace=ns) def has_client_nodes(self) -> bool: """Return True if any K8s node has the 'client' role label.""" @@ -1900,9 +1944,9 @@ def cleanup_stale_fio_resources(self, namespace: str = None): # Delete clone PVCs (prefixed clone-) f"kubectl get pvc -n {ns} --no-headers -o custom-columns=NAME:.metadata.name " f"2>/dev/null | grep '^clone-' | xargs -r kubectl delete pvc -n {ns} --ignore-not-found", - # Delete VolumeSnapshots (prefixed snap-) + # Delete VolumeSnapshots (prefixed snap- or snapshot-) f"kubectl get volumesnapshot -n {ns} --no-headers -o custom-columns=NAME:.metadata.name " - f"2>/dev/null | grep '^snap-' | xargs -r kubectl delete volumesnapshot -n {ns} --ignore-not-found", + f"2>/dev/null | grep -E '^(snap-|snapshot-)' | xargs -r kubectl delete volumesnapshot -n {ns} --ignore-not-found --wait=true", # Delete test PVCs (various prefixes) f"kubectl get pvc -n {ns} --no-headers -o custom-columns=NAME:.metadata.name " f"2>/dev/null | grep -E '^(pvc-|mig-pvc-|add-pvc-)' | xargs -r kubectl delete pvc -n {ns} --ignore-not-found", @@ -2156,6 +2200,11 @@ def patch_storage_node_add_workers(self, new_workers: list, new CR and handles provisioning automatically — no separate ``StorageCluster`` expand patch is needed. + Device configuration (``driveSizeRange``, ``pcieModel``) is + read from the parent StorageNodeSet and included in the + ``overrides`` block so the init container can find the correct + SSD devices on the new worker. + Parameters ---------- new_workers : list[str] @@ -2167,8 +2216,31 @@ def patch_storage_node_add_workers(self, new_workers: list, Override namespace (default ``self.namespace``). """ ns = namespace or self.namespace + + # Read device config from parent StorageNodeSet + sns_json = self.get_resource_json( + "storagenodeset.storage.simplyblock.io", + storage_node_set_ref, + namespace=ns, + ) + sns_spec = sns_json.get("spec", {}) + drive_size_range = sns_spec.get("driveSizeRange", "") + pcie_model = sns_spec.get("pcieModel", "") + if drive_size_range or pcie_model: + self.logger.info( + f"[K8sUtils] Read device config from StorageNodeSet " + f"'{storage_node_set_ref}': driveSizeRange={drive_size_range!r}, " + f"pcieModel={pcie_model!r}" + ) + for worker in new_workers: cr_name = f"{storage_node_set_ref}-expand-{worker}" + overrides = " expand: true\n" + if drive_size_range: + overrides += f' driveSizeRange: "{drive_size_range}"\n' + if pcie_model: + overrides += f' pcieModel: "{pcie_model}"\n' + yaml_content = ( "apiVersion: storage.simplyblock.io/v1alpha1\n" "kind: StorageNode\n" @@ -2180,7 +2252,7 @@ def patch_storage_node_add_workers(self, new_workers: list, f" workerNode: {worker}\n" " socketIndex: 0\n" " overrides:\n" - " expand: true\n" + f"{overrides}" ) self.logger.info( f"[K8sUtils] Creating StorageNode CR '{cr_name}' " @@ -2688,6 +2760,77 @@ def delete_backup_restore(self, name: str, namespace: str = None): self.logger.info(f"[K8sUtils] Deleting BackupRestore '{name}'") self.delete_resource("backuprestore", name, namespace=ns) + # ── BackupImport CRD operations ────────────────────────────────────────── + + def create_backup_import(self, name: str, + source_cluster_name: str, + source_backup_id: str, + target_cluster_name: str, + namespace: str = None): + """Create a BackupImport CRD to import a backup from another cluster. + + The operator will create a corresponding StorageBackup on the target + cluster and handle source-switching automatically. + """ + ns = namespace or self.namespace + yaml_content = ( + f"apiVersion: storage.simplyblock.io/v1alpha1\n" + f"kind: BackupImport\n" + f"metadata:\n" + f" name: {name}\n" + f" namespace: {ns}\n" + f"spec:\n" + f" sourceClusterName: {source_cluster_name}\n" + f" sourceBackupID: {source_backup_id}\n" + f" targetClusterName: {target_cluster_name}\n" + ) + self.logger.info( + f"[K8sUtils] Creating BackupImport '{name}' " + f"(source={source_cluster_name}/{source_backup_id} " + f"-> target={target_cluster_name})" + ) + self.apply_yaml(yaml_content, namespace=ns) + + def wait_backup_import_done(self, name: str, timeout: int = 300, + namespace: str = None) -> dict: + """Poll until BackupImport phase is ``Done``. Returns resource JSON. + + The status will contain ``storageBackupRef`` — the name of the + StorageBackup CRD created on the target cluster. + """ + ns = namespace or self.namespace + deadline = time.time() + timeout + while time.time() < deadline: + res = self.get_resource_json("backupimport", name, namespace=ns) + phase = (res.get("status", {}).get("phase") or "").lower() + if phase == "done": + self.logger.info(f"[K8sUtils] BackupImport '{name}' is Done") + return res + if phase == "failed": + raise AssertionError( + f"BackupImport '{name}' failed: {res.get('status')}") + self.logger.info( + f"[K8sUtils] Waiting for BackupImport '{name}' " + f"(phase={res.get('status', {}).get('phase', 'unknown')})" + ) + time.sleep(10) + raise TimeoutError( + f"BackupImport '{name}' not Done within {timeout}s" + ) + + def get_backup_import_storage_backup_ref(self, name: str, + namespace: str = None) -> str: + """Return the storageBackupRef from a BackupImport's status.""" + ns = namespace or self.namespace + res = self.get_resource_json("backupimport", name, namespace=ns) + return res.get("status", {}).get("storageBackupRef", "") + + def delete_backup_import(self, name: str, namespace: str = None): + """Delete a BackupImport CRD.""" + ns = namespace or self.namespace + self.logger.info(f"[K8sUtils] Deleting BackupImport '{name}'") + self.delete_resource("backupimport", name, namespace=ns) + # ── BackupPolicy CRD operations ────────────────────────────────────────── def create_backup_policy(self, name: str, @@ -2872,11 +3015,104 @@ def generate_checksums_in_pvc(self, pod_name: str, files: list, checksums[parts[1]] = parts[0] return checksums - def delete_pod(self, pod_name: str, namespace: str = None): - """Delete a pod.""" + def delete_pod(self, pod_name: str, namespace: str = None, + wait: bool = False): + """Delete a pod. + + When *wait* is True the call blocks until the pod is fully removed, + preventing name-collision races when a new pod with the same name is + created shortly after deletion. + """ + ns = namespace or self.namespace + self.logger.info(f"[K8sUtils] Deleting pod '{pod_name}'" + f"{' (waiting)' if wait else ''}") + if wait: + self._exec_kubectl( + f"kubectl delete pod {pod_name} -n {ns} " + f"--ignore-not-found --wait=true --timeout=60s" + ) + else: + self.delete_resource("pod", pod_name, namespace=ns) + + def wait_for_per_node_config(self, worker_node: str, + configmap_name: str = "simplyblock-node-per-node-config", + namespace: str = None, + timeout: int = 120): + """Wait until the per-node-config ConfigMap has an entry for *worker_node*. + + The operator updates this ConfigMap when it reconciles a StorageNode CR. + The DaemonSet pod's ``node-env-writer`` init container reads the entry + to set ``MAX_LVOL``, ``MAX_SIZE``, etc. If the pod starts before the + entry exists it gets ``MAX_LVOL=0`` and crashes. + + Args: + worker_node: The K8s node name (e.g. ``worker-4.ocp.simplyblock.ai``). + configmap_name: Name of the per-node-config ConfigMap. + namespace: K8s namespace (defaults to ``self.namespace``). + timeout: Max seconds to wait. + """ + import time ns = namespace or self.namespace - self.logger.info(f"[K8sUtils] Deleting pod '{pod_name}'") - self.delete_resource("pod", pod_name, namespace=ns) + deadline = time.time() + timeout + while time.time() < deadline: + cm = self.get_resource_json("configmap", configmap_name, namespace=ns) + data = cm.get("data", {}) + if worker_node in data: + self.logger.info( + f"[K8sUtils] per-node-config has entry for '{worker_node}': " + f"{data[worker_node][:120]}..." + ) + return + self.logger.info( + f"[K8sUtils] Waiting for per-node-config entry for " + f"'{worker_node}' (keys: {list(data.keys())})..." + ) + time.sleep(10) + self.logger.warning( + f"[K8sUtils] per-node-config entry for '{worker_node}' not found " + f"after {timeout}s — proceeding anyway (pod may still fail)" + ) + + def delete_storage_node_pods_on_worker(self, worker_node: str, + namespace: str = None): + """Delete storage-node DaemonSet pods running on a specific worker. + + Call this after the per-node-config ConfigMap has been updated for the + worker (use ``wait_for_per_node_config`` first) so the DaemonSet + recreates the pod with the correct configuration. + """ + ns = namespace or self.namespace + cmd = ( + f"kubectl get pods -n {ns} " + f"--field-selector spec.nodeName={worker_node} " + f"--no-headers -o custom-columns=NAME:.metadata.name" + ) + out, _ = self._exec_kubectl(cmd, supress_logs=True) + deleted = 0 + for line in (out or "").strip().splitlines(): + pod_name = line.strip() + if not pod_name: + continue + if "simplyblock-storage-node-ds" in pod_name: + self.logger.info( + f"[K8sUtils] Deleting stale storage-node pod " + f"'{pod_name}' on worker '{worker_node}'" + ) + self._exec_kubectl( + f"kubectl delete pod {pod_name} -n {ns} " + f"--force --grace-period=0 --ignore-not-found" + ) + deleted += 1 + if deleted: + self.logger.info( + f"[K8sUtils] Deleted {deleted} stale storage-node pod(s) " + f"on worker '{worker_node}'" + ) + else: + self.logger.info( + f"[K8sUtils] No stale storage-node pods found on " + f"worker '{worker_node}'" + ) def verify_pvc_mount(self, pvc_name: str, namespace: str = None, timeout: int = 120) -> tuple: @@ -3148,11 +3384,20 @@ def wait_for_storage_node_status(self, node_id, status, timeout=60): actual_status = None status_list = status if isinstance(status, list) else [status] while timeout > 0: - node_details = self.get_storage_node_details(node_id) - actual_status = node_details[0]["status"] - if actual_status in status_list: - return node_details[0] - self.logger.info(f"Expected Status: {status_list} / Actual Status: {actual_status}") + try: + node_details = self.get_storage_node_details(node_id) + actual_status = node_details[0]["status"] + if actual_status in status_list: + return node_details[0] + self.logger.info( + f"Expected Status: {status_list} / Actual Status: {actual_status}" + ) + except (json.JSONDecodeError, IndexError, KeyError) as exc: + # Transient failure — admin-control pod may be recycling. + self.logger.warning( + f"[wait_for_storage_node_status] Transient error for " + f"{node_id}: {exc!r}, retrying..." + ) sleep_n_sec(1) timeout -= 1 raise TimeoutError( @@ -3317,6 +3562,55 @@ def add_storage_pool(self, pool_name, cluster_id=None, max_rw_iops=0, max_rw_mby f"Operator may not have reconciled the pool." ) + def add_storage_pool_direct(self, pool_name, cluster_id=None, sbcli_cmd=None): + """Create a pool directly via ``sbcli pool add`` (kubectl exec). + + Unlike ``add_storage_pool()``, this does NOT create a Pool CRD — + it calls the CLI directly in the admin pod. Use this for R25 + clusters that have no operator to reconcile Pool CRDs. + + sbcli_cmd: override the CLI binary name (e.g. "sbcli-dev" for R25). + Defaults to self.sbcli_cmd. + + Returns the pool name on success. + """ + cli = sbcli_cmd or self.sbcli_cmd + + def _list_pools(): + items = self._run_json(f"{cli} pool list --json") + return {item["Name"]: item["UUID"] for item in items} + + # 1. Check if sbcli already sees a pool + existing = _list_pools() + if existing: + actual = next(iter(existing)) + self.logger.info(f"[pool] Using existing pool '{actual}'") + return actual + + # 2. Create via CLI + cid = cluster_id or self.cluster_id + cmd = f"{cli} pool add {pool_name} {cid}" + self.logger.info(f"[pool] Creating pool directly via CLI: {cmd}") + out = self._run(cmd) + self.logger.info(f"[pool] pool add output: {out}") + + # 3. Wait for pool to appear in pool list + for attempt in range(30): # up to 150s + pools = _list_pools() + if pools: + actual = next(iter(pools)) + self.logger.info( + f"[pool] Pool '{actual}' visible after CLI create " + f"(attempt {attempt})" + ) + return actual + sleep_n_sec(5) + + raise TimeoutError( + f"[pool] Pool '{pool_name}' not visible in sbcli after 150s " + f"following direct CLI creation." + ) + def pool_crd_exists(self, pool_name): """Check if a Pool CRD exists in K8s (with or without simplyblock- prefix). diff --git a/e2e/utils/parity_report.py b/e2e/utils/parity_report.py index 6d8196f4e9..10fe0cae4a 100644 --- a/e2e/utils/parity_report.py +++ b/e2e/utils/parity_report.py @@ -70,7 +70,7 @@ def generate_html_report(findings, output_dir, cluster_id="", extra_meta=None): _CSS = """ body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; margin: 0; padding: 20px; background: #f5f5f5; color: #333; } -.container { max-width: 1200px; margin: 0 auto; } +.container { max-width: 1400px; margin: 0 auto; } h1 { color: #1a1a2e; border-bottom: 3px solid #16213e; padding-bottom: 10px; } h2 { color: #16213e; margin-top: 30px; } .meta { color: #666; font-size: 0.9em; margin-bottom: 20px; } @@ -87,7 +87,8 @@ def generate_html_report(findings, output_dir, cluster_id="", extra_meta=None): box-shadow: 0 1px 3px rgba(0,0,0,0.1); } th { background: #16213e; color: #fff; padding: 10px 14px; text-align: left; font-weight: 600; font-size: 0.85em; text-transform: uppercase; } -td { padding: 8px 14px; border-bottom: 1px solid #eee; font-size: 0.9em; } +td { padding: 8px 14px; border-bottom: 1px solid #eee; font-size: 0.9em; + vertical-align: top; } tr:hover { background: #f8f9fa; } .sev-error { color: #e74c3c; font-weight: bold; } .sev-warning { color: #f39c12; font-weight: bold; } @@ -101,13 +102,23 @@ def generate_html_report(findings, output_dir, cluster_id="", extra_meta=None): .badge-info { background: #e8f4fd; color: #3498db; } .mono { font-family: 'SF Mono', 'Consolas', monospace; font-size: 0.85em; } .empty { color: #999; font-style: italic; } +.detail-box { background: #f8f9fa; border: 1px solid #e9ecef; border-radius: 4px; + padding: 6px 10px; font-family: 'SF Mono', 'Consolas', monospace; + font-size: 0.82em; white-space: pre-wrap; word-break: break-all; + max-height: 200px; overflow-y: auto; margin-top: 4px; } +.api-call { color: #0d6efd; font-family: 'SF Mono', 'Consolas', monospace; + font-size: 0.82em; } +.http-status { font-weight: bold; } +.http-ok { color: #27ae60; } +.http-err { color: #e74c3c; } +.id-list { font-size: 0.8em; color: #666; } """ def _esc(s): """HTML-escape a string.""" if s is None: - return '—' + return '\u2014' s = str(s) return s.replace("&", "&").replace("<", "<").replace(">", ">") @@ -129,6 +140,26 @@ def _presence(val): return _esc(val) +def _http_status_html(status): + """Render an HTTP status code with color coding.""" + if status is None or status == "": + return '\u2014' + status = int(status) if status else 0 + cls = "http-ok" if 200 <= status < 300 else "http-err" + return f'{status}' + + +def _id_list_html(ids): + """Render a list of IDs as a compact display.""" + if not ids: + return "" + if isinstance(ids, list): + items = [str(i)[:12] for i in ids[:5]] + suffix = f" +{len(ids) - 5} more" if len(ids) > 5 else "" + return f'[{", ".join(items)}{suffix}]' + return _esc(ids) + + def _build_html(findings, errors, warnings, infos, categories, ts, cluster_id, extra_meta): parts = [] @@ -162,13 +193,14 @@ def _build_html(findings, errors, warnings, infos, categories, ts, not_tested = [f for f in findings if f.get("category") == "not_tested"] if not_tested: parts.append("

Operations Not Tested

") - parts.append("") + parts.append( + "
OperationCLIv1v2Reason
" + "" + ) for f in not_tested: parts.append( - f'' - f"" - f"" - f"" + f"" + f'' f'' ) parts.append("
SeverityOperationReason
{_esc(f.get("operation"))}{_presence(f.get('cli'))}{_presence(f.get('v1'))}{_presence(f.get('v2'))}
{_badge(f['severity'])}{_esc(f.get("operation"))}{_esc(f.get("detail"))}
") @@ -211,14 +243,24 @@ def _build_html(findings, errors, warnings, infos, categories, ts, iface_errors = [f for f in findings if f.get("category") == "interface_error"] if iface_errors: parts.append("

Interface Errors (one interface returned an error)

") - parts.append("" - "") + parts.append( + "
SeverityOperationInterfaceDetail
" + "" + "" + ) for f in iface_errors: + resp_preview = f.get("response_preview", "") + resp_html = "" + if resp_preview: + resp_html = f'
{_esc(resp_preview)}
' parts.append( f"" f'' f'' - f'' + f'' + f'' + f'' + f"" ) parts.append("
SeverityOperationInterfaceDetailAPI CallHTTP StatusResponse
{_badge(f['severity'])}{_esc(f.get("operation"))}{_esc(f.get("interface"))}{_esc(f.get("detail"))}
{_esc(f.get("detail"))}{_esc(f.get("api_call", ""))}{_http_status_html(f.get("http_status"))}{resp_html}
") @@ -226,15 +268,22 @@ def _build_html(findings, errors, warnings, infos, categories, ts, counts = [f for f in findings if f.get("category") == "count_mismatch"] if counts: parts.append("

Count Mismatches (different number of items returned)

") - parts.append("" - "") + parts.append( + "
SeverityOperationCLI countv1 countv2 count
" + "" + "" + "" + ) for f in counts: parts.append( f"" f'' f'' f'' - f'' + f'' + f'' + f'' + f'' ) parts.append("
SeverityOperationCLI countv1 countv2 countCLI IDs (sample)v1 IDs (sample)v2 IDs (sample)
{_badge(f['severity'])}{_esc(f.get("operation"))}{_esc(f.get("cli"))}{_esc(f.get("v1"))}{_esc(f.get("v2"))}
{_esc(f.get("v2"))}{_id_list_html(f.get("cli_ids"))}{_id_list_html(f.get("v1_ids"))}{_id_list_html(f.get("v2_ids"))}
") @@ -258,7 +307,7 @@ def _build_html(findings, errors, warnings, infos, categories, ts, parts.append("") if not findings: - parts.append('

No findings — all interfaces are in parity.

') + parts.append('

No findings \u2014 all interfaces are in parity.

') parts.append("") return "\n".join(parts) diff --git a/e2e/utils/ssh_utils.py b/e2e/utils/ssh_utils.py index 3ec694784d..fed82725f6 100644 --- a/e2e/utils/ssh_utils.py +++ b/e2e/utils/ssh_utils.py @@ -1901,7 +1901,7 @@ def get_lvol_vs_device(self, node, lvol_id=None, nqn=None, ns_id=None): # filesystem.append(columns[0]) # return filesystem - def deploy_storage_node(self, node, max_lvol, max_prov_gb, ifname="eth0", branch='main'): + def deploy_storage_node(self, node, max_lvol, max_prov_gb, ifname="eth0", branch='main', nodes_per_socket=1): """ Runs 'sn configure' and 'sn deploy' on the node with provided configuration. @@ -1910,13 +1910,14 @@ def deploy_storage_node(self, node, max_lvol, max_prov_gb, ifname="eth0", branch max_lvol (int): Maximum number of lvols. max_prov_gb (int): Maximum provision size in GB. ifname (str): Mgmt Interface (Default: eth0) + nodes_per_socket (int): Number of nodes per socket (Default: 1). Set to 2 for dual-node-per-host. """ cmd = f"pip install --force-reinstall git+https://github.com/simplyblock-io/sbcli.git@{branch}" self.exec_command(node=node, command=cmd) time.sleep(10) - configure_cmd = f"{self.base_cmd} -d sn configure --max-subsys {max_lvol}" + configure_cmd = f"{self.base_cmd} -d sn configure --max-subsys {max_lvol} --nodes-per-socket {nodes_per_socket}" deploy_cmd = f"{self.base_cmd} sn deploy --ifname {ifname}" self.logger.info(f"Deploying storage node: {node}")