diff --git a/tests/virt/cluster/common_templates/rhel/test_rhel_tablet_device.py b/tests/virt/cluster/common_templates/rhel/test_rhel_tablet_device.py index 2f8e66e941..cd4dbf0804 100644 --- a/tests/virt/cluster/common_templates/rhel/test_rhel_tablet_device.py +++ b/tests/virt/cluster/common_templates/rhel/test_rhel_tablet_device.py @@ -18,6 +18,7 @@ from tests.os_params import FEDORA_LATEST, RHEL_LATEST, RHEL_LATEST_LABELS from tests.virt.cluster.common_templates.utils import check_vm_xml_tablet_device, set_vm_tablet_device_dict +from utilities.constants.timeouts import TIMEOUT_2MIN from utilities.constants.virt import VIRTIO from utilities.virt import VirtualMachineForTestsFromTemplate, migrate_vm_and_verify @@ -32,6 +33,7 @@ def check_vm_system_tablet_device(vm, expected_device): output = run_ssh_commands( host=vm.ssh_exec, commands=shlex.split(r"grep -rs '^QEMU *.* Tablet' /sys/devices ||true"), + wait_timeout=TIMEOUT_2MIN, )[0] assert re.search(rf"/sys/devices/pci(.*)QEMU {expected_device} Tablet", output), ( diff --git a/tests/virt/cluster/common_templates/utils.py b/tests/virt/cluster/common_templates/utils.py index 5762a0ecc5..22e2b867f9 100644 --- a/tests/virt/cluster/common_templates/utils.py +++ b/tests/virt/cluster/common_templates/utils.py @@ -24,6 +24,7 @@ from utilities.constants.timeouts import ( TCP_TIMEOUT_30SEC, TIMEOUT_1MIN, + TIMEOUT_2MIN, TIMEOUT_15SEC, TIMEOUT_90SEC, ) @@ -51,7 +52,7 @@ def vm_os_version(vm): grep_cmd = os_name.title() if "fedora" in vm.os_flavor else os.replace("-", ".") command = shlex.split(f"cat /etc/{os_name}-release | grep {grep_cmd}") - run_ssh_commands(host=vm.ssh_exec, commands=command) + run_ssh_commands(host=vm.ssh_exec, commands=command, wait_timeout=TIMEOUT_2MIN) # Guest agent data comparison functions. @@ -169,7 +170,12 @@ def _get_vm_timezone_diff(): return int(re.search(r".*, [-]?(\d+)", vm_timezone_diff).group(1)) virtctl_info = get_virtctl_user_info(vm=vm) - windows_info = run_ssh_commands(host=vm.ssh_exec, commands=["quser"], tcp_timeout=TCP_TIMEOUT_30SEC)[0] + windows_info = run_ssh_commands( + host=vm.ssh_exec, + commands=["quser"], + tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, + )[0] # Match timezone to VM's timezone and not use UTC virtctl_time = virtctl_info["loginTime"] - _get_vm_timezone_diff() data_mismatch = [] @@ -296,7 +302,7 @@ def get_libvirt_fs_info(vm, admin_client): def get_linux_fs_info(ssh_exec): cmd = shlex.split("df -TB1 | grep /dev/vd") - out = run_ssh_commands(host=ssh_exec, commands=cmd)[0] + out = run_ssh_commands(host=ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN)[0] disks = out.strip().split() return { "name": disks[0].split("/dev/")[1], @@ -311,13 +317,30 @@ def get_linux_fs_info(ssh_exec): def get_windows_fs_info(ssh_exec): disk_name_cmd = shlex.split("fsutil volume list") - disk_name = run_ssh_commands(host=ssh_exec, commands=disk_name_cmd, tcp_timeout=TCP_TIMEOUT_30SEC)[0] + disk_name = run_ssh_commands( + host=ssh_exec, + commands=disk_name_cmd, + tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, + )[0] disk_space_cmd = shlex.split("fsutil volume diskfree C:") disk_space = ( - run_ssh_commands(host=ssh_exec, commands=disk_space_cmd, tcp_timeout=TCP_TIMEOUT_30SEC)[0].strip().split("\r\n") + run_ssh_commands( + host=ssh_exec, + commands=disk_space_cmd, + tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, + )[0] + .strip() + .split("\r\n") ) fs_type_cmd = shlex.split("fsutil fsinfo volumeinfo C:") - fs_type = run_ssh_commands(host=ssh_exec, commands=fs_type_cmd, tcp_timeout=TCP_TIMEOUT_30SEC)[0] + fs_type = run_ssh_commands( + host=ssh_exec, + commands=fs_type_cmd, + tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, + )[0] windows_info = f"{disk_name} {windows_disk_space_parser(disk_space)} {fs_type}" windows_fs_info = re.search( @@ -375,12 +398,28 @@ def get_linux_user_info(vm): if any(os_version in vm.name for os_version in ["rhel-7", "rhel-8", "centos-8"]): # Older versions use lastlog and who to get the login time cmd = shlex.split("lastlog | grep tty; who | awk \"'{print$3}'\"") - output = run_ssh_commands(host=ssh_exec, commands=cmd)[0].strip().split() + output = ( + run_ssh_commands( + host=ssh_exec, + commands=cmd, + wait_timeout=TIMEOUT_2MIN, + )[0] + .strip() + .split() + ) date = datetime.strptime(f"{output[7]}-{output[3]}-{output[4]} {output[5]}", "%Y-%b-%d %H:%M:%S") else: # Newer versions use last -w --time-format iso to get the login time cmd = shlex.split("last -w --time-format iso | grep 'tty.*still logged in'") - output = run_ssh_commands(host=ssh_exec, commands=cmd)[0].strip().split() + output = ( + run_ssh_commands( + host=ssh_exec, + commands=cmd, + wait_timeout=TIMEOUT_2MIN, + )[0] + .strip() + .split() + ) date = datetime.fromisoformat(output[2]) timestamp = date.replace(tzinfo=timezone(timedelta(seconds=int(ssh_exec.os.timezone.offset) * 36))).timestamp() diff --git a/tests/virt/cluster/common_templates/windows/test_windows_custom_options.py b/tests/virt/cluster/common_templates/windows/test_windows_custom_options.py index 0421fe8fea..a91a8264f8 100644 --- a/tests/virt/cluster/common_templates/windows/test_windows_custom_options.py +++ b/tests/virt/cluster/common_templates/windows/test_windows_custom_options.py @@ -17,6 +17,7 @@ from utilities.constants.networking import LINUX_BRIDGE from utilities.constants.timeouts import ( TCP_TIMEOUT_30SEC, + TIMEOUT_2MIN, TIMEOUT_12MIN, ) from utilities.constants.virt import VIRTIO @@ -115,6 +116,7 @@ def initialize_and_format_windows_drive(vm, disk_number, partition_number, drive ], get_pty=True, tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, ) diff --git a/tests/virt/cluster/common_templates/windows/test_windows_tablet_device.py b/tests/virt/cluster/common_templates/windows/test_windows_tablet_device.py index 76d4d8d25b..38642e2a7b 100644 --- a/tests/virt/cluster/common_templates/windows/test_windows_tablet_device.py +++ b/tests/virt/cluster/common_templates/windows/test_windows_tablet_device.py @@ -13,7 +13,7 @@ from tests.os_params import WINDOWS_10, WINDOWS_LATEST, WINDOWS_LATEST_LABELS from tests.virt.cluster.common_templates.utils import check_vm_xml_tablet_device, set_vm_tablet_device_dict -from utilities.constants.timeouts import TCP_TIMEOUT_30SEC +from utilities.constants.timeouts import TCP_TIMEOUT_30SEC, TIMEOUT_2MIN from utilities.constants.virt import VIRTIO pytestmark = [ @@ -33,6 +33,7 @@ def check_windows_vm_tablet_device(vm, driver_state): host=vm.ssh_exec, commands=shlex.split("%systemroot%\\\\system32\\\\driverquery /fo list /v"), tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, )[0] assert re.search( diff --git a/tests/virt/cluster/longevity_tests/utils.py b/tests/virt/cluster/longevity_tests/utils.py index 7786b532c8..9d1a525e15 100644 --- a/tests/virt/cluster/longevity_tests/utils.py +++ b/tests/virt/cluster/longevity_tests/utils.py @@ -26,6 +26,7 @@ ) from utilities.constants.timeouts import ( TCP_TIMEOUT_30SEC, + TIMEOUT_2MIN, TIMEOUT_5MIN, TIMEOUT_30MIN, TIMEOUT_40MIN, @@ -116,6 +117,7 @@ def _set_interface_mtu(vm): run_ssh_commands( host=vm.ssh_exec, commands=shlex.split(f'netsh interface ipv4 set subinterface "{interface_name}" mtu=1400 store=persistent'), + wait_timeout=TIMEOUT_2MIN, ) def _prepare_win_upgrade(vm): @@ -134,7 +136,11 @@ def _prepare_win_upgrade(vm): rf'-DestinationPath {ADMIN_DOWNLOADS_FOLDER_PATH}\PSExec"' ), ] - run_ssh_commands(host=vm.ssh_exec, commands=win_upgrade_prepare_cmds) + run_ssh_commands( + host=vm.ssh_exec, + commands=win_upgrade_prepare_cmds, + wait_timeout=TIMEOUT_2MIN, + ) def _start_win_upgrade(vm): LOGGER.info(f"VM {vm.name}: Starting upgrade process") @@ -220,7 +226,11 @@ def verify_windows_upgraded_recently_multi_vms(vm_list): failed_vms_list = [] for vm in vm_list: - if not run_ssh_commands(host=vm.ssh_exec, commands=get_upgrade_history_cmd)[0]: + if not run_ssh_commands( + host=vm.ssh_exec, + commands=get_upgrade_history_cmd, + wait_timeout=TIMEOUT_2MIN, + )[0]: failed_vms_list.append(vm.name) assert not failed_vms_list, f"Some VMs failed to upgrade! Falied VMs: {failed_vms_list}" diff --git a/tests/virt/cluster/sysprep/test_sysprep.py b/tests/virt/cluster/sysprep/test_sysprep.py index 0a02a43a97..cb6e081c0b 100644 --- a/tests/virt/cluster/sysprep/test_sysprep.py +++ b/tests/virt/cluster/sysprep/test_sysprep.py @@ -22,6 +22,7 @@ from utilities.constants.images import OS_FLAVOR_WINDOWS from utilities.constants.timeouts import ( TCP_TIMEOUT_30SEC, + TIMEOUT_2MIN, TIMEOUT_5MIN, ) from utilities.ssp import get_windows_timezone @@ -53,9 +54,12 @@ def verify_changes_from_autounattend(vm, timezone, hostname): # hostname LOGGER.info(f"Verifying hostname change from answer file in vm {vm.name}") - actual_hostname = run_ssh_commands(host=vm.ssh_exec, commands=["hostname"], tcp_timeout=TCP_TIMEOUT_30SEC)[ - 0 - ].strip() + actual_hostname = run_ssh_commands( + host=vm.ssh_exec, + commands=["hostname"], + tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, + )[0].strip() assert actual_hostname == hostname, f"Incorrect hostname, expected {hostname}, found {actual_hostname}" @@ -167,6 +171,7 @@ def sealed_vm(sysprep_vm): posix=False, ), tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, ) diff --git a/tests/virt/cluster/vm_cloning/test_vm_cloning.py b/tests/virt/cluster/vm_cloning/test_vm_cloning.py index f8a8d510f3..de81c73379 100644 --- a/tests/virt/cluster/vm_cloning/test_vm_cloning.py +++ b/tests/virt/cluster/vm_cloning/test_vm_cloning.py @@ -20,6 +20,7 @@ ) from utilities.constants import Images from utilities.constants.instance_types import RHEL_WITH_INSTANCETYPE_AND_PREFERENCE +from utilities.constants.timeouts import TIMEOUT_2MIN from utilities.storage import ( add_dv_to_vm, check_disk_count_in_vm, @@ -121,6 +122,7 @@ def files_created_on_pvc_disks(vm_with_dv_for_cloning): # update selinux: allow snapshot for second disk shlex.split("sudo setsebool -P virt_qemu_ga_read_nonsecurity_files 1"), ], + wait_timeout=TIMEOUT_2MIN, ) diff --git a/tests/virt/cluster/vm_cloning/utils.py b/tests/virt/cluster/vm_cloning/utils.py index 0c8d16e0e1..71b10ac0bf 100644 --- a/tests/virt/cluster/vm_cloning/utils.py +++ b/tests/virt/cluster/vm_cloning/utils.py @@ -13,6 +13,7 @@ ) from utilities.constants.timeouts import ( TIMEOUT_1SEC, + TIMEOUT_2MIN, TIMEOUT_10SEC, ) @@ -37,6 +38,7 @@ def check_if_files_present_after_cloning(vm): shlex.split(f"sudo mount {SECOND_DISK_PATH} /mnt"), shlex.split(f"sudo cat {SECOND_DISK_TEST_FILE_STR}"), ], + wait_timeout=TIMEOUT_2MIN, ) diff --git a/tests/virt/cluster/vm_lifecycle/test_vm_data_persistency.py b/tests/virt/cluster/vm_lifecycle/test_vm_data_persistency.py index ccd00a7528..f8fe5ab584 100644 --- a/tests/virt/cluster/vm_lifecycle/test_vm_data_persistency.py +++ b/tests/virt/cluster/vm_lifecycle/test_vm_data_persistency.py @@ -99,7 +99,11 @@ def restarted_persistence_vm(request, persistence_vm): def get_linux_timezone(ssh_exec): - return run_ssh_commands(host=ssh_exec, commands=shlex.split("timedatectl show | grep -i timezone"))[0] + return run_ssh_commands( + host=ssh_exec, + commands=shlex.split("timedatectl show | grep -i timezone"), + wait_timeout=TIMEOUT_2MIN, + )[0] def get_timezone(vm, os): diff --git a/tests/virt/node/general/test_oom.py b/tests/virt/node/general/test_oom.py index 56e543c320..67e2af60f0 100644 --- a/tests/virt/node/general/test_oom.py +++ b/tests/virt/node/general/test_oom.py @@ -18,6 +18,7 @@ from utilities.constants import Images from utilities.constants.timeouts import ( TCP_TIMEOUT_30SEC, + TIMEOUT_2MIN, TIMEOUT_15MIN, ) from utilities.constants.virt import STRESS_CPU_MEM_IO_COMMAND @@ -75,6 +76,7 @@ def _transfer_loop(stop_event): host=vm.ssh_exec, commands=shlex.split(f"{'wsl' if 'windows' in vm.name else ''} dd if=/dev/zero of={file_name} bs=100M count=1"), tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, ) stop_event = Event() diff --git a/tests/virt/node/general/test_vm_with_sidecar.py b/tests/virt/node/general/test_vm_with_sidecar.py index 03bc9a3105..899e8ca2a8 100644 --- a/tests/virt/node/general/test_vm_with_sidecar.py +++ b/tests/virt/node/general/test_vm_with_sidecar.py @@ -7,6 +7,7 @@ import pytest from pyhelper_utils.shell import run_ssh_commands +from utilities.constants.timeouts import TIMEOUT_2MIN from utilities.virt import VirtualMachineForTests, fedora_vm_body, running_vm @@ -64,4 +65,5 @@ def test_vm_with_sidecar_hook(enabled_featuregate_scope_function, sidecar_vm): run_ssh_commands( host=sidecar_vm.ssh_exec, commands=shlex.split("sudo dmidecode -s baseboard-manufacturer | grep 'Radical Edward'\n"), + wait_timeout=TIMEOUT_2MIN, ) diff --git a/tests/virt/node/general/test_vmi_reset.py b/tests/virt/node/general/test_vmi_reset.py index a8a36564e8..5812a9b5c2 100644 --- a/tests/virt/node/general/test_vmi_reset.py +++ b/tests/virt/node/general/test_vmi_reset.py @@ -4,6 +4,7 @@ import pytest from pyhelper_utils.shell import run_ssh_commands +from utilities.constants.timeouts import TIMEOUT_2MIN from utilities.virt import wait_for_running_vm LOGGER = logging.getLogger(__name__) @@ -13,6 +14,7 @@ def get_vm_boot_count(vm): reboot_count = run_ssh_commands( host=vm.ssh_exec, commands=[shlex.split("journalctl --list-boots | wc -l")], + wait_timeout=TIMEOUT_2MIN, )[0].strip() return int(reboot_count) diff --git a/tests/virt/node/general/test_windows_vtpm_bitlocker.py b/tests/virt/node/general/test_windows_vtpm_bitlocker.py index 84ced181ee..2e6663e6dc 100644 --- a/tests/virt/node/general/test_windows_vtpm_bitlocker.py +++ b/tests/virt/node/general/test_windows_vtpm_bitlocker.py @@ -40,6 +40,7 @@ def verify_tpm_in_os(vm): r"wmic /namespace:\\root\cimv2\security\microsofttpm path Win32_Tpm get IsEnabled_InitialValue", posix=False, ), + wait_timeout=TIMEOUT_2MIN, )[0] assert "TRUE" in vtpm_enabled, "TPM is not present/enabled in OS!" @@ -67,11 +68,20 @@ def _wait_encryption_finish(vm): run_ssh_commands( host=vm.ssh_exec, commands=shlex.split('powershell -c "install-windowsfeature bitlocker"'), + wait_timeout=TIMEOUT_2MIN, ) restart_vm_wait_for_running_vm(vm=vm) - run_ssh_commands(host=vm.ssh_exec, commands=shlex.split('powershell -c "initialize-tpm"')) - run_ssh_commands(host=vm.ssh_exec, commands=shlex.split("manage-bde -on c: -s")) + run_ssh_commands( + host=vm.ssh_exec, + commands=shlex.split('powershell -c "initialize-tpm"'), + wait_timeout=TIMEOUT_2MIN, + ) + run_ssh_commands( + host=vm.ssh_exec, + commands=shlex.split("manage-bde -on c: -s"), + wait_timeout=TIMEOUT_2MIN, + ) _wait_encryption_finish(vm=vm) diff --git a/tests/virt/node/general/test_wsl2.py b/tests/virt/node/general/test_wsl2.py index 3188daf339..f634d32a3b 100644 --- a/tests/virt/node/general/test_wsl2.py +++ b/tests/virt/node/general/test_wsl2.py @@ -20,7 +20,7 @@ from tests.virt.constants import WINDOWS_10_WSL, WINDOWS_11_WSL from utilities.constants import Images from utilities.constants.images import OS_FLAVOR_WINDOWS -from utilities.constants.timeouts import TCP_TIMEOUT_30SEC +from utilities.constants.timeouts import TCP_TIMEOUT_30SEC, TIMEOUT_2MIN from utilities.virt import ( VirtualMachineForTests, migrate_vm_and_verify, @@ -48,6 +48,7 @@ def get_windows_vm_resource_usage(vm): host=vm.ssh_exec, commands=shlex.split("python C:\\\\tools\\\\cpu_mem_usage.py"), tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, )[0] LOGGER.info(f"Windows VM CPU and Memory usage: {usage}") out = re.search(r".*CPU usage: (?P.*),.*\(RAM\):(?P.*)", usage) diff --git a/tests/virt/node/gpu/utils.py b/tests/virt/node/gpu/utils.py index 0a1459ba92..47ae7b6cfa 100644 --- a/tests/virt/node/gpu/utils.py +++ b/tests/virt/node/gpu/utils.py @@ -62,6 +62,7 @@ def install_nvidia_drivers_on_windows_vm(vm, supported_gpu_device): ) ], tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, ) # Wait for Running VM, as only vGPU VM Reboots after installing NVIDIA GRID Drivers. if fetch_gpu_device_name_from_vm_instance(vm=vm) == vgpu_device_name: diff --git a/tests/virt/node/migration_and_maintenance/test_vm_disk_load_with_migration.py b/tests/virt/node/migration_and_maintenance/test_vm_disk_load_with_migration.py index 48ba2721b7..75565f14e9 100644 --- a/tests/virt/node/migration_and_maintenance/test_vm_disk_load_with_migration.py +++ b/tests/virt/node/migration_and_maintenance/test_vm_disk_load_with_migration.py @@ -10,7 +10,7 @@ from timeout_sampler import TimeoutExpiredError, TimeoutSampler from tests.os_params import FEDORA_LATEST, FEDORA_LATEST_LABELS -from utilities.constants.timeouts import TIMEOUT_1MIN +from utilities.constants.timeouts import TIMEOUT_1MIN, TIMEOUT_2MIN from utilities.virt import migrate_vm_and_verify, running_vm, vm_instance_from_template if TYPE_CHECKING: @@ -46,7 +46,11 @@ def vm_with_fio( @pytest.fixture() def running_fio_in_vm(vm_with_fio): LOGGER.info("Installing fio and iotop tools") - run_ssh_commands(host=vm_with_fio.ssh_exec, commands=shlex.split("sudo dnf install -y iotop fio")) + run_ssh_commands( + host=vm_with_fio.ssh_exec, + commands=shlex.split("sudo dnf install -y iotop fio"), + wait_timeout=TIMEOUT_2MIN, + ) # Random write/read - create a 1G file, and perform 4KB reads and writes using a 75%/25% LOGGER.info("Running fio in VM") @@ -55,7 +59,11 @@ def running_fio_in_vm(vm_with_fio): "--gtod_reduce=1 --name=test --filename=/home/fedora/random_read_write.fio --bs=4k --iodepth=64 " "--size=1G --readwrite=randrw --rwmixread=75 --numjobs=8 >& /dev/null &" ) - run_ssh_commands(host=vm_with_fio.ssh_exec, commands=fio_cmd) + run_ssh_commands( + host=vm_with_fio.ssh_exec, + commands=fio_cmd, + wait_timeout=TIMEOUT_2MIN, + ) get_disk_usage(ssh_exec=vm_with_fio.ssh_exec) diff --git a/tests/virt/node/rng/test_rng.py b/tests/virt/node/rng/test_rng.py index d9afa2b555..11432b89fb 100644 --- a/tests/virt/node/rng/test_rng.py +++ b/tests/virt/node/rng/test_rng.py @@ -7,6 +7,7 @@ import pytest from pyhelper_utils.shell import run_ssh_commands +from utilities.constants.timeouts import TIMEOUT_2MIN from utilities.virt import VirtualMachineForTests, fedora_vm_body, running_vm @@ -39,6 +40,7 @@ def test_vm_with_rng(rng_vm): rng_output = run_ssh_commands( host=rng_vm.ssh_exec, commands=rng_commnds, + wait_timeout=TIMEOUT_2MIN, ) assert set(rng_output[:2]) == {"1\n"}, f"Expected:1, actual: {rng_output[:2]}" assert rng_output[-1].strip() == "virtio_rng.0", f"rng_current: {rng_output[0]}" diff --git a/tests/virt/utils.py b/tests/virt/utils.py index 621ade3779..2073d21d7f 100644 --- a/tests/virt/utils.py +++ b/tests/virt/utils.py @@ -289,6 +289,7 @@ def get_num_gpu_devices_in_rhel_vm(vm): "-c", '/sbin/lspci -nnk | grep -E "controller.+NVIDIA" | wc -l', ], + wait_timeout=TIMEOUT_2MIN, )[0].strip() ) @@ -298,6 +299,7 @@ def get_gpu_device_name_from_windows_vm(vm): host=vm.ssh_exec, commands=[shlex.split("wmic path win32_VideoController get name")], tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, )[0]