Select the renderer device by CUDA index

source/isaaclab/changelog.d/jichuanh-fix-renderer-active-gpu-physical-index.rst

@@ -0,0 +1,10 @@
+Fixed
+^^^^^
+
+* Fixed rendering failing to start when ``CUDA_VISIBLE_DEVICES`` selects GPUs that do not begin at
+  zero, such as ``CUDA_VISIBLE_DEVICES=1,2``. Such runs aborted with ``CUDA error 700`` after
+  ``omni.gpu_foundation_factory`` reported "Failed to create any GPU devices". The renderer device
+  is now selected through ``/renderer/multiGpu/activeCudaGpus``, which takes a CUDA device index,
+  instead of ``/renderer/activeGpu``, which indexes the graphics device list that
+  ``CUDA_VISIBLE_DEVICES`` does not filter. Runs whose visible devices already begin at zero are
+  unaffected.

source/isaaclab/isaaclab/app/app_launcher.py

@@ -721,6 +721,7 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None:
         "fast_shutdown": [bool],
         "limit_cpu_threads": [int],
         "experience": [str],
+        "extra_args": [list, type(None)],
     }
     """A dictionary containing the type of arguments passed to SimulationApp.
 
@@ -1123,10 +1124,18 @@ def _resolve_device_settings(self, launcher_args: dict):
             # pass command line variable to kit
             sys.argv.append(f"--/plugins/carb.tasking.plugin/threadCount={num_threads_per_process}")
 
-        # set rendering device. We do not need to set physics_gpu because it will automatically pick the same one
-        # as the active_gpu device. Setting physics_gpu explicitly may result in a different device to be used.
+        # Set the rendering device. ``/physics/cudaDevice`` is resolved by CUDA, so the masked index is
+        # correct there. ``/renderer/activeGpu`` instead indexes the graphics device list, which
+        # ``CUDA_VISIBLE_DEVICES`` does not filter, so the same index selects the wrong GPU whenever the
+        # visible devices do not start at zero. ``/renderer/multiGpu/activeCudaGpus`` takes CUDA indices
+        # and the renderer translates them itself, so select the device through that instead and leave
+        # ``activeGpu`` unset -- the translation is only applied when no explicit graphics index is given.
         launcher_args["physics_gpu"] = self.device_id
-        launcher_args["active_gpu"] = self.device_id
+        extra_args = list(launcher_args.get("extra_args") or [])
+        # Trailing comma: the setting is parsed as a comma-separated string, and a bare integer is
+        # silently ignored.
+        extra_args.append(f"--/renderer/multiGpu/activeCudaGpus={self.device_id},")
+        launcher_args["extra_args"] = extra_args
 
         # Defer importing torch until after SimulationApp starts.  Importing
         # torch can import NumPy/OpenBLAS, whose at-fork handlers can crash

source/isaaclab/test/app/test_app_launcher_argv.py

@@ -3,11 +3,13 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-"""Tests for filtering command-line arguments before Kit startup."""
+"""Tests for the command-line arguments passed to Kit at startup."""
 
 import sys
 
-from isaaclab.app.app_launcher import _sanitize_sys_argv_for_kit
+import pytest
+
+from isaaclab.app.app_launcher import AppLauncher, _sanitize_sys_argv_for_kit
 
 
 def test_sanitize_sys_argv_removes_trailing_pytest_verbosity(monkeypatch):
@@ -36,3 +38,52 @@ def test_sanitize_sys_argv_removes_pytest_marker_pair(monkeypatch):
     result = _sanitize_sys_argv_for_kit(["test_script.py", "-m", "not isaacsim_ci", "--keep"])
 
     assert result == ["test_script.py", "--keep"]
+
+
+def _resolve_device(launcher_args: dict) -> dict:
+    """Run device resolution without constructing an ``AppLauncher``."""
+    launcher = AppLauncher.__new__(AppLauncher)
+    launcher.device_id = 0
+    launcher._deferred_cuda_device_id = None
+    launcher._xr = False
+    AppLauncher._resolve_device_settings(launcher, launcher_args)
+    return launcher_args
+
+
+def test_renderer_device_selected_by_cuda_index():
+    """Select the renderer device through the CUDA-indexed setting."""
+    args = _resolve_device({"device": "cuda:1"})
+
+    assert "--/renderer/multiGpu/activeCudaGpus=1," in args["extra_args"]
+
+
+def test_active_gpu_is_left_unset():
+    """Leave ``activeGpu`` unset: the renderer only applies the CUDA translation without it."""
+    args = _resolve_device({"device": "cuda:1"})
+
+    assert args.get("active_gpu") is None
+
+
+def test_physics_keeps_the_cuda_index():
+    """Keep the CUDA index for physics, which CUDA resolves itself."""
+    args = _resolve_device({"device": "cuda:1"})
+
+    assert args["physics_gpu"] == 1
+
+
+@pytest.mark.parametrize("device", ["cuda:0", "cuda:3"])
+def test_cuda_index_setting_is_comma_terminated(device):
+    """Terminate the value with a comma: a bare integer is silently ignored by the renderer."""
+    args = _resolve_device({"device": device})
+
+    cuda_gpu_args = [arg for arg in args["extra_args"] if "activeCudaGpus" in arg]
+    assert len(cuda_gpu_args) == 1
+    assert cuda_gpu_args[0].endswith(",")
+
+
+def test_user_extra_args_are_preserved():
+    """Append to caller-provided ``extra_args`` rather than replacing them."""
+    args = _resolve_device({"device": "cuda:0", "extra_args": ["--/app/fastShutdown=False"]})
+
+    assert "--/app/fastShutdown=False" in args["extra_args"]
+    assert any("activeCudaGpus" in arg for arg in args["extra_args"])