[Virt] Add SSH retry to run_ssh_commands calls

tests/virt/cluster/common_templates/rhel/test_rhel_tablet_device.py

@@ -18,6 +18,7 @@
 
 from tests.os_params import FEDORA_LATEST, RHEL_LATEST, RHEL_LATEST_LABELS
 from tests.virt.cluster.common_templates.utils import check_vm_xml_tablet_device, set_vm_tablet_device_dict
+from utilities.constants.timeouts import TIMEOUT_2MIN
 from utilities.constants.virt import VIRTIO
 from utilities.virt import VirtualMachineForTestsFromTemplate, migrate_vm_and_verify
 
@@ -32,6 +33,7 @@ def check_vm_system_tablet_device(vm, expected_device):
     output = run_ssh_commands(
         host=vm.ssh_exec,
         commands=shlex.split(r"grep -rs '^QEMU *.* Tablet' /sys/devices ||true"),
+        wait_timeout=TIMEOUT_2MIN,
     )[0]
 
     assert re.search(rf"/sys/devices/pci(.*)QEMU {expected_device} Tablet", output), (

tests/virt/cluster/common_templates/utils.py

@@ -24,6 +24,7 @@
 from utilities.constants.timeouts import (
     TCP_TIMEOUT_30SEC,
     TIMEOUT_1MIN,
+    TIMEOUT_2MIN,
     TIMEOUT_15SEC,
     TIMEOUT_90SEC,
 )
@@ -51,7 +52,7 @@ def vm_os_version(vm):
     grep_cmd = os_name.title() if "fedora" in vm.os_flavor else os.replace("-", ".")
     command = shlex.split(f"cat /etc/{os_name}-release | grep {grep_cmd}")
 
-    run_ssh_commands(host=vm.ssh_exec, commands=command)
+    run_ssh_commands(host=vm.ssh_exec, commands=command, wait_timeout=TIMEOUT_2MIN)
 
 
 # Guest agent data comparison functions.
@@ -169,7 +170,12 @@ def _get_vm_timezone_diff():
         return int(re.search(r".*, [-]?(\d+)", vm_timezone_diff).group(1))
 
     virtctl_info = get_virtctl_user_info(vm=vm)
-    windows_info = run_ssh_commands(host=vm.ssh_exec, commands=["quser"], tcp_timeout=TCP_TIMEOUT_30SEC)[0]
+    windows_info = run_ssh_commands(
+        host=vm.ssh_exec,
+        commands=["quser"],
+        tcp_timeout=TCP_TIMEOUT_30SEC,
+        wait_timeout=TIMEOUT_2MIN,
+    )[0]
     # Match timezone to VM's timezone and not use UTC
     virtctl_time = virtctl_info["loginTime"] - _get_vm_timezone_diff()
     data_mismatch = []
@@ -296,7 +302,7 @@ def get_libvirt_fs_info(vm, admin_client):
 
 def get_linux_fs_info(ssh_exec):
     cmd = shlex.split("df -TB1 | grep /dev/vd")
-    out = run_ssh_commands(host=ssh_exec, commands=cmd)[0]
+    out = run_ssh_commands(host=ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN)[0]
     disks = out.strip().split()
     return {
         "name": disks[0].split("/dev/")[1],
@@ -311,13 +317,30 @@ def get_linux_fs_info(ssh_exec):
 
 def get_windows_fs_info(ssh_exec):
     disk_name_cmd = shlex.split("fsutil volume list")
-    disk_name = run_ssh_commands(host=ssh_exec, commands=disk_name_cmd, tcp_timeout=TCP_TIMEOUT_30SEC)[0]
+    disk_name = run_ssh_commands(
+        host=ssh_exec,
+        commands=disk_name_cmd,
+        tcp_timeout=TCP_TIMEOUT_30SEC,
+        wait_timeout=TIMEOUT_2MIN,
+    )[0]
     disk_space_cmd = shlex.split("fsutil volume diskfree C:")
     disk_space = (
-        run_ssh_commands(host=ssh_exec, commands=disk_space_cmd, tcp_timeout=TCP_TIMEOUT_30SEC)[0].strip().split("\r\n")
+        run_ssh_commands(
+            host=ssh_exec,
+            commands=disk_space_cmd,
+            tcp_timeout=TCP_TIMEOUT_30SEC,
+            wait_timeout=TIMEOUT_2MIN,
+        )[0]
+        .strip()
+        .split("\r\n")
     )
     fs_type_cmd = shlex.split("fsutil fsinfo volumeinfo C:")
-    fs_type = run_ssh_commands(host=ssh_exec, commands=fs_type_cmd, tcp_timeout=TCP_TIMEOUT_30SEC)[0]
+    fs_type = run_ssh_commands(
+        host=ssh_exec,
+        commands=fs_type_cmd,
+        tcp_timeout=TCP_TIMEOUT_30SEC,
+        wait_timeout=TIMEOUT_2MIN,
+    )[0]
 
     windows_info = f"{disk_name} {windows_disk_space_parser(disk_space)} {fs_type}"
     windows_fs_info = re.search(
@@ -375,12 +398,28 @@ def get_linux_user_info(vm):
     if any(os_version in vm.name for os_version in ["rhel-7", "rhel-8", "centos-8"]):
         # Older versions use lastlog and who to get the login time
         cmd = shlex.split("lastlog | grep tty; who | awk \"'{print$3}'\"")
-        output = run_ssh_commands(host=ssh_exec, commands=cmd)[0].strip().split()
+        output = (
+            run_ssh_commands(
+                host=ssh_exec,
+                commands=cmd,
+                wait_timeout=TIMEOUT_2MIN,
+            )[0]
+            .strip()
+            .split()
+        )
         date = datetime.strptime(f"{output[7]}-{output[3]}-{output[4]} {output[5]}", "%Y-%b-%d %H:%M:%S")
     else:
         # Newer versions use last -w --time-format iso to get the login time
         cmd = shlex.split("last -w --time-format iso | grep 'tty.*still logged in'")
-        output = run_ssh_commands(host=ssh_exec, commands=cmd)[0].strip().split()
+        output = (
+            run_ssh_commands(
+                host=ssh_exec,
+                commands=cmd,
+                wait_timeout=TIMEOUT_2MIN,
+            )[0]
+            .strip()
+            .split()
+        )
         date = datetime.fromisoformat(output[2])
 
     timestamp = date.replace(tzinfo=timezone(timedelta(seconds=int(ssh_exec.os.timezone.offset) * 36))).timestamp()

tests/virt/cluster/common_templates/windows/test_windows_custom_options.py

@@ -17,6 +17,7 @@
 from utilities.constants.networking import LINUX_BRIDGE
 from utilities.constants.timeouts import (
     TCP_TIMEOUT_30SEC,
+    TIMEOUT_2MIN,
     TIMEOUT_12MIN,
 )
 from utilities.constants.virt import VIRTIO
@@ -115,6 +116,7 @@ def initialize_and_format_windows_drive(vm, disk_number, partition_number, drive
         ],
         get_pty=True,
         tcp_timeout=TCP_TIMEOUT_30SEC,
+        wait_timeout=TIMEOUT_2MIN,
     )
 
 

tests/virt/cluster/common_templates/windows/test_windows_tablet_device.py

@@ -13,7 +13,7 @@
 
 from tests.os_params import WINDOWS_10, WINDOWS_LATEST, WINDOWS_LATEST_LABELS
 from tests.virt.cluster.common_templates.utils import check_vm_xml_tablet_device, set_vm_tablet_device_dict
-from utilities.constants.timeouts import TCP_TIMEOUT_30SEC
+from utilities.constants.timeouts import TCP_TIMEOUT_30SEC, TIMEOUT_2MIN
 from utilities.constants.virt import VIRTIO
 
 pytestmark = [
@@ -33,6 +33,7 @@ def check_windows_vm_tablet_device(vm, driver_state):
         host=vm.ssh_exec,
         commands=shlex.split("%systemroot%\\\\system32\\\\driverquery /fo list /v"),
         tcp_timeout=TCP_TIMEOUT_30SEC,
+        wait_timeout=TIMEOUT_2MIN,
     )[0]
 
     assert re.search(

tests/virt/cluster/longevity_tests/utils.py

@@ -26,6 +26,7 @@
 )
 from utilities.constants.timeouts import (
     TCP_TIMEOUT_30SEC,
+    TIMEOUT_2MIN,
     TIMEOUT_5MIN,
     TIMEOUT_30MIN,
     TIMEOUT_40MIN,
@@ -116,6 +117,7 @@ def _set_interface_mtu(vm):
         run_ssh_commands(
             host=vm.ssh_exec,
             commands=shlex.split(f'netsh interface ipv4 set subinterface "{interface_name}" mtu=1400 store=persistent'),
+            wait_timeout=TIMEOUT_2MIN,
         )
 
     def _prepare_win_upgrade(vm):
@@ -134,7 +136,11 @@ def _prepare_win_upgrade(vm):
                 rf'-DestinationPath {ADMIN_DOWNLOADS_FOLDER_PATH}\PSExec"'
             ),
         ]
-        run_ssh_commands(host=vm.ssh_exec, commands=win_upgrade_prepare_cmds)
+        run_ssh_commands(
+            host=vm.ssh_exec,
+            commands=win_upgrade_prepare_cmds,
+            wait_timeout=TIMEOUT_2MIN,
+        )
 
     def _start_win_upgrade(vm):
         LOGGER.info(f"VM {vm.name}: Starting upgrade process")
@@ -220,7 +226,11 @@ def verify_windows_upgraded_recently_multi_vms(vm_list):
 
     failed_vms_list = []
     for vm in vm_list:
-        if not run_ssh_commands(host=vm.ssh_exec, commands=get_upgrade_history_cmd)[0]:
+        if not run_ssh_commands(
+            host=vm.ssh_exec,
+            commands=get_upgrade_history_cmd,
+            wait_timeout=TIMEOUT_2MIN,
+        )[0]:
             failed_vms_list.append(vm.name)
 
     assert not failed_vms_list, f"Some VMs failed to upgrade! Falied VMs: {failed_vms_list}"

tests/virt/cluster/sysprep/test_sysprep.py

@@ -22,6 +22,7 @@
 from utilities.constants.images import OS_FLAVOR_WINDOWS
 from utilities.constants.timeouts import (
     TCP_TIMEOUT_30SEC,
+    TIMEOUT_2MIN,
     TIMEOUT_5MIN,
 )
 from utilities.ssp import get_windows_timezone
@@ -53,9 +54,12 @@ def verify_changes_from_autounattend(vm, timezone, hostname):
 
     # hostname
     LOGGER.info(f"Verifying hostname change from answer file in vm {vm.name}")
-    actual_hostname = run_ssh_commands(host=vm.ssh_exec, commands=["hostname"], tcp_timeout=TCP_TIMEOUT_30SEC)[
-        0
-    ].strip()
+    actual_hostname = run_ssh_commands(
+        host=vm.ssh_exec,
+        commands=["hostname"],
+        tcp_timeout=TCP_TIMEOUT_30SEC,
+        wait_timeout=TIMEOUT_2MIN,
+    )[0].strip()
     assert actual_hostname == hostname, f"Incorrect hostname, expected {hostname}, found {actual_hostname}"
 
 
@@ -167,6 +171,7 @@ def sealed_vm(sysprep_vm):
             posix=False,
         ),
         tcp_timeout=TCP_TIMEOUT_30SEC,
+        wait_timeout=TIMEOUT_2MIN,
     )
 
 

tests/virt/cluster/vm_cloning/test_vm_cloning.py

@@ -20,6 +20,7 @@
 )
 from utilities.constants import Images
 from utilities.constants.instance_types import RHEL_WITH_INSTANCETYPE_AND_PREFERENCE
+from utilities.constants.timeouts import TIMEOUT_2MIN
 from utilities.storage import (
     add_dv_to_vm,
     check_disk_count_in_vm,
@@ -121,6 +122,7 @@ def files_created_on_pvc_disks(vm_with_dv_for_cloning):
             # update selinux: allow snapshot for second disk
             shlex.split("sudo setsebool -P virt_qemu_ga_read_nonsecurity_files 1"),
         ],
+        wait_timeout=TIMEOUT_2MIN,
     )
 
 

tests/virt/cluster/vm_cloning/utils.py

@@ -13,6 +13,7 @@
 )
 from utilities.constants.timeouts import (
     TIMEOUT_1SEC,
+    TIMEOUT_2MIN,
     TIMEOUT_10SEC,
 )
 
@@ -37,6 +38,7 @@ def check_if_files_present_after_cloning(vm):
             shlex.split(f"sudo mount {SECOND_DISK_PATH} /mnt"),
             shlex.split(f"sudo cat {SECOND_DISK_TEST_FILE_STR}"),
         ],
+        wait_timeout=TIMEOUT_2MIN,
     )
 
 

tests/virt/cluster/vm_lifecycle/test_vm_data_persistency.py

@@ -99,7 +99,11 @@ def restarted_persistence_vm(request, persistence_vm):
 
 
 def get_linux_timezone(ssh_exec):
-    return run_ssh_commands(host=ssh_exec, commands=shlex.split("timedatectl show | grep -i timezone"))[0]
+    return run_ssh_commands(
+        host=ssh_exec,
+        commands=shlex.split("timedatectl show | grep -i timezone"),
+        wait_timeout=TIMEOUT_2MIN,
+    )[0]
 
 
 def get_timezone(vm, os):

tests/virt/node/general/test_oom.py

@@ -18,6 +18,7 @@
 from utilities.constants import Images
 from utilities.constants.timeouts import (
     TCP_TIMEOUT_30SEC,
+    TIMEOUT_2MIN,
     TIMEOUT_15MIN,
 )
 from utilities.constants.virt import STRESS_CPU_MEM_IO_COMMAND
@@ -75,6 +76,7 @@ def _transfer_loop(stop_event):
         host=vm.ssh_exec,
         commands=shlex.split(f"{'wsl' if 'windows' in vm.name else ''} dd if=/dev/zero of={file_name} bs=100M count=1"),
         tcp_timeout=TCP_TIMEOUT_30SEC,
+        wait_timeout=TIMEOUT_2MIN,
     )
 
     stop_event = Event()

tests/virt/node/general/test_vm_with_sidecar.py

@@ -7,6 +7,7 @@
 import pytest
 from pyhelper_utils.shell import run_ssh_commands
 
+from utilities.constants.timeouts import TIMEOUT_2MIN
 from utilities.virt import VirtualMachineForTests, fedora_vm_body, running_vm
 
 
@@ -64,4 +65,5 @@ def test_vm_with_sidecar_hook(enabled_featuregate_scope_function, sidecar_vm):
     run_ssh_commands(
         host=sidecar_vm.ssh_exec,
         commands=shlex.split("sudo dmidecode -s baseboard-manufacturer | grep 'Radical Edward'\n"),
+        wait_timeout=TIMEOUT_2MIN,
     )

tests/virt/node/general/test_vmi_reset.py

@@ -4,6 +4,7 @@
 import pytest
 from pyhelper_utils.shell import run_ssh_commands
 
+from utilities.constants.timeouts import TIMEOUT_2MIN
 from utilities.virt import wait_for_running_vm
 
 LOGGER = logging.getLogger(__name__)
@@ -13,6 +14,7 @@ def get_vm_boot_count(vm):
     reboot_count = run_ssh_commands(
         host=vm.ssh_exec,
         commands=[shlex.split("journalctl --list-boots | wc -l")],
+        wait_timeout=TIMEOUT_2MIN,
     )[0].strip()
 
     return int(reboot_count)

tests/virt/node/general/test_windows_vtpm_bitlocker.py

@@ -40,6 +40,7 @@ def verify_tpm_in_os(vm):
             r"wmic /namespace:\\root\cimv2\security\microsofttpm path Win32_Tpm get IsEnabled_InitialValue",
             posix=False,
         ),
+        wait_timeout=TIMEOUT_2MIN,
     )[0]
     assert "TRUE" in vtpm_enabled, "TPM is not present/enabled in OS!"
 
@@ -67,11 +68,20 @@ def _wait_encryption_finish(vm):
         run_ssh_commands(
             host=vm.ssh_exec,
             commands=shlex.split('powershell -c "install-windowsfeature bitlocker"'),
+            wait_timeout=TIMEOUT_2MIN,
         )
         restart_vm_wait_for_running_vm(vm=vm)
 
-    run_ssh_commands(host=vm.ssh_exec, commands=shlex.split('powershell -c "initialize-tpm"'))
-    run_ssh_commands(host=vm.ssh_exec, commands=shlex.split("manage-bde -on c: -s"))
+    run_ssh_commands(
+        host=vm.ssh_exec,
+        commands=shlex.split('powershell -c "initialize-tpm"'),
+        wait_timeout=TIMEOUT_2MIN,
+    )
+    run_ssh_commands(
+        host=vm.ssh_exec,
+        commands=shlex.split("manage-bde -on c: -s"),
+        wait_timeout=TIMEOUT_2MIN,
+    )
     _wait_encryption_finish(vm=vm)
 
 

tests/virt/node/general/test_wsl2.py

@@ -20,7 +20,7 @@
 from tests.virt.constants import WINDOWS_10_WSL, WINDOWS_11_WSL
 from utilities.constants import Images
 from utilities.constants.images import OS_FLAVOR_WINDOWS
-from utilities.constants.timeouts import TCP_TIMEOUT_30SEC
+from utilities.constants.timeouts import TCP_TIMEOUT_30SEC, TIMEOUT_2MIN
 from utilities.virt import (
     VirtualMachineForTests,
     migrate_vm_and_verify,
@@ -48,6 +48,7 @@ def get_windows_vm_resource_usage(vm):
         host=vm.ssh_exec,
         commands=shlex.split("python C:\\\\tools\\\\cpu_mem_usage.py"),
         tcp_timeout=TCP_TIMEOUT_30SEC,
+        wait_timeout=TIMEOUT_2MIN,
     )[0]
     LOGGER.info(f"Windows VM CPU and Memory usage: {usage}")
     out = re.search(r".*CPU usage: (?P<cpu>.*),.*\(RAM\):(?P<ram>.*)", usage)

tests/virt/node/gpu/utils.py

@@ -62,6 +62,7 @@ def install_nvidia_drivers_on_windows_vm(vm, supported_gpu_device):
             )
         ],
         tcp_timeout=TCP_TIMEOUT_30SEC,
+        wait_timeout=TIMEOUT_2MIN,
     )
     # Wait for Running VM, as only vGPU VM Reboots after installing NVIDIA GRID Drivers.
     if fetch_gpu_device_name_from_vm_instance(vm=vm) == vgpu_device_name: