diff --git a/source/isaaclab/changelog.d/jichuanh-fix-renderer-active-gpu-physical-index.rst b/source/isaaclab/changelog.d/jichuanh-fix-renderer-active-gpu-physical-index.rst new file mode 100644 index 000000000000..bd7fd354a090 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-fix-renderer-active-gpu-physical-index.rst @@ -0,0 +1,10 @@ +Fixed +^^^^^ + +* Fixed rendering failing to start when ``CUDA_VISIBLE_DEVICES`` selects GPUs that do not begin at + zero, such as ``CUDA_VISIBLE_DEVICES=1,2``. Such runs aborted with ``CUDA error 700`` after + ``omni.gpu_foundation_factory`` reported "Failed to create any GPU devices". The renderer device + is now selected through ``/renderer/multiGpu/activeCudaGpus``, which takes a CUDA device index, + instead of ``/renderer/activeGpu``, which indexes the graphics device list that + ``CUDA_VISIBLE_DEVICES`` does not filter. Runs whose visible devices already begin at zero are + unaffected. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 83705d67b226..501800b02210 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -699,7 +699,6 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: _SIM_APP_CFG_TYPES: dict[str, list[type]] = { "headless": [bool], "hide_ui": [bool, type(None)], - "active_gpu": [int, type(None)], "physics_gpu": [int], "multi_gpu": [bool], "sync_loads": [bool], @@ -1123,10 +1122,10 @@ def _resolve_device_settings(self, launcher_args: dict): # pass command line variable to kit sys.argv.append(f"--/plugins/carb.tasking.plugin/threadCount={num_threads_per_process}") - # set rendering device. We do not need to set physics_gpu because it will automatically pick the same one - # as the active_gpu device. Setting physics_gpu explicitly may result in a different device to be used. + # ``/physics/cudaDevice`` is resolved by CUDA, so the masked index is correct there. + # ``activeGpu`` is deliberately left unset; the renderer device is selected in + # :meth:`_resolve_kit_args` instead. launcher_args["physics_gpu"] = self.device_id - launcher_args["active_gpu"] = self.device_id # Defer importing torch until after SimulationApp starts. Importing # torch can import NumPy/OpenBLAS, whose at-fork handlers can crash @@ -1267,6 +1266,13 @@ def _resolve_kit_args(self, launcher_args: dict): if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args): self._kit_args.append(argument) + # Select the renderer by CUDA index; the trailing comma keeps the setting string-typed. + if launcher_args.get("multi_gpu") is False: + argument = f"--/renderer/multiGpu/activeCudaGpus={self.device_id}," + setting = argument.partition("=")[0] + if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args): + self._kit_args.append(argument) + sys.argv += self._kit_args def _create_app(self): diff --git a/source/isaaclab/test/app/test_app_launcher_argv.py b/source/isaaclab/test/app/test_app_launcher_argv.py index a65068210761..2695ae402244 100644 --- a/source/isaaclab/test/app/test_app_launcher_argv.py +++ b/source/isaaclab/test/app/test_app_launcher_argv.py @@ -3,11 +3,13 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Tests for filtering command-line arguments before Kit startup.""" +"""Tests for the command-line arguments passed to Kit at startup.""" import sys -from isaaclab.app.app_launcher import _sanitize_sys_argv_for_kit +import pytest + +from isaaclab.app.app_launcher import AppLauncher, _sanitize_sys_argv_for_kit def test_sanitize_sys_argv_removes_trailing_pytest_verbosity(monkeypatch): @@ -36,3 +38,44 @@ def test_sanitize_sys_argv_removes_pytest_marker_pair(monkeypatch): result = _sanitize_sys_argv_for_kit(["test_script.py", "-m", "not isaacsim_ci", "--keep"]) assert result == ["test_script.py", "--keep"] + + +def _resolve_devices_and_kit_args(launcher_args: dict, monkeypatch) -> tuple[dict, list[str]]: + """Resolve device settings and Kit arguments without constructing an ``AppLauncher``. + + ``_resolve_kit_args`` extends ``sys.argv``, so the caller's argv is isolated. + """ + monkeypatch.setattr(sys, "argv", ["script.py"]) + launcher = AppLauncher.__new__(AppLauncher) + launcher.device_id = 0 + launcher._deferred_cuda_device_id = None + launcher._xr = False + AppLauncher._resolve_device_settings(launcher, launcher_args) + AppLauncher._resolve_kit_args(launcher, launcher_args) + return launcher_args, launcher._kit_args + + +@pytest.mark.parametrize( + ("launcher_args", "expected_renderer_args"), + [ + pytest.param( + {"device": "cuda:1", "multi_gpu": False}, + ["--/renderer/multiGpu/activeCudaGpus=1,"], + id="single-gpu", + ), + pytest.param( + {"device": "cuda:1", "multi_gpu": False, "kit_args": "--/renderer/multiGpu/activeCudaGpus=3,"}, + ["--/renderer/multiGpu/activeCudaGpus=3,"], + id="explicit-kit-arg", + ), + pytest.param({"device": "cuda:1"}, [], id="multi-gpu"), + ], +) +def test_devices_selected_by_cuda_index(launcher_args, expected_renderer_args, monkeypatch): + """Select physics and single-GPU rendering devices by CUDA index.""" + args, kit_args = _resolve_devices_and_kit_args(launcher_args, monkeypatch) + + renderer_args = [arg for arg in kit_args if arg.startswith("--/renderer/multiGpu/activeCudaGpus=")] + assert renderer_args == expected_renderer_args + assert args["physics_gpu"] == 1 + assert "active_gpu" not in args