Select the renderer device by CUDA index

source/isaaclab/changelog.d/jichuanh-fix-renderer-active-gpu-physical-index.rst

@@ -0,0 +1,10 @@
+Fixed
+^^^^^
+
+* Fixed rendering failing to start when ``CUDA_VISIBLE_DEVICES`` selects GPUs that do not begin at
+  zero, such as ``CUDA_VISIBLE_DEVICES=1,2``. Such runs aborted with ``CUDA error 700`` after
+  ``omni.gpu_foundation_factory`` reported "Failed to create any GPU devices". The renderer device
+  is now selected through ``/renderer/multiGpu/activeCudaGpus``, which takes a CUDA device index,
+  instead of ``/renderer/activeGpu``, which indexes the graphics device list that
+  ``CUDA_VISIBLE_DEVICES`` does not filter. Runs whose visible devices already begin at zero are
+  unaffected.

source/isaaclab/isaaclab/app/app_launcher.py

@@ -1123,10 +1123,10 @@ def _resolve_device_settings(self, launcher_args: dict):
             # pass command line variable to kit
             sys.argv.append(f"--/plugins/carb.tasking.plugin/threadCount={num_threads_per_process}")
 
-        # set rendering device. We do not need to set physics_gpu because it will automatically pick the same one
-        # as the active_gpu device. Setting physics_gpu explicitly may result in a different device to be used.
+        # ``/physics/cudaDevice`` is resolved by CUDA, so the masked index is correct there.
+        # ``activeGpu`` is deliberately left unset; the renderer device is selected in
+        # :meth:`_resolve_kit_args` instead.
         launcher_args["physics_gpu"] = self.device_id
-        launcher_args["active_gpu"] = self.device_id
 
         # Defer importing torch until after SimulationApp starts.  Importing
         # torch can import NumPy/OpenBLAS, whose at-fork handlers can crash
@@ -1267,6 +1267,22 @@ def _resolve_kit_args(self, launcher_args: dict):
         if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args):
             self._kit_args.append(argument)
 
+        # Select the renderer device by CUDA index, but only where this process is pinned to one
+        # GPU. ``/renderer/activeGpu`` indexes the graphics device list, which
+        # ``CUDA_VISIBLE_DEVICES`` does not filter, so the same index selects the wrong GPU whenever
+        # the visible devices do not start at zero. ``activeCudaGpus`` takes CUDA indices and the
+        # renderer translates them itself, but only while no explicit graphics index is given, so
+        # ``activeGpu`` is left unset. The trailing comma is required: ``=0`` is stored as an int
+        # and the setting is read with ``getString``, which then returns empty.
+        #
+        # Skipped when Kit renders across several GPUs in one process, since this setting also fills
+        # the renderer's active-device list and a one-element list caps the device count at one.
+        if launcher_args.get("multi_gpu") is False:
+            argument = f"--/renderer/multiGpu/activeCudaGpus={self.device_id},"
+            setting = argument.partition("=")[0]
+            if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args):
+                self._kit_args.append(argument)
+
         sys.argv += self._kit_args
 
     def _create_app(self):

source/isaaclab/test/app/test_app_launcher_argv.py

@@ -3,11 +3,11 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-"""Tests for filtering command-line arguments before Kit startup."""
+"""Tests for the command-line arguments passed to Kit at startup."""
 
 import sys
 
-from isaaclab.app.app_launcher import _sanitize_sys_argv_for_kit
+from isaaclab.app.app_launcher import AppLauncher, _sanitize_sys_argv_for_kit
 
 
 def test_sanitize_sys_argv_removes_trailing_pytest_verbosity(monkeypatch):
@@ -36,3 +36,57 @@ def test_sanitize_sys_argv_removes_pytest_marker_pair(monkeypatch):
     result = _sanitize_sys_argv_for_kit(["test_script.py", "-m", "not isaacsim_ci", "--keep"])
 
     assert result == ["test_script.py", "--keep"]
+
+
+def _resolve_devices_and_kit_args(launcher_args: dict, monkeypatch) -> tuple[dict, list[str]]:
+    """Resolve device settings and Kit arguments without constructing an ``AppLauncher``.
+
+    ``_resolve_kit_args`` extends ``sys.argv``, so the caller's argv is isolated.
+    """
+    monkeypatch.setattr(sys, "argv", ["script.py"])
+    launcher = AppLauncher.__new__(AppLauncher)
+    launcher.device_id = 0
+    launcher._deferred_cuda_device_id = None
+    launcher._xr = False
+    AppLauncher._resolve_device_settings(launcher, launcher_args)
+    AppLauncher._resolve_kit_args(launcher, launcher_args)
+    return launcher_args, launcher._kit_args
+
+
+def test_both_devices_selected_by_cuda_index(monkeypatch):
+    """Select both devices by CUDA index, the renderer through the setting that translates it.
+
+    The trailing comma is part of the contract: without it the value is stored as an int and the
+    renderer, which reads the setting as a string, sees nothing.
+    """
+    args, kit_args = _resolve_devices_and_kit_args({"device": "cuda:1", "multi_gpu": False}, monkeypatch)
+
+    assert "--/renderer/multiGpu/activeCudaGpus=1," in kit_args
+    assert args["physics_gpu"] == 1
+
+
+def test_active_gpu_is_left_unset(monkeypatch):
+    """Leave ``activeGpu`` unset: the renderer only applies the CUDA translation without it."""
+    args, _ = _resolve_devices_and_kit_args({"device": "cuda:1", "multi_gpu": False}, monkeypatch)
+
+    assert args.get("active_gpu") is None
+
+
+def test_user_supplied_device_setting_is_not_overridden(monkeypatch):
+    """Leave a caller-specified renderer device alone rather than adding a second setting."""
+    args, kit_args = _resolve_devices_and_kit_args(
+        {"device": "cuda:1", "multi_gpu": False, "kit_args": "--/renderer/multiGpu/activeCudaGpus=3,"}, monkeypatch
+    )
+
+    assert [arg for arg in kit_args if "activeCudaGpus" in arg] == ["--/renderer/multiGpu/activeCudaGpus=3,"]
+
+
+def test_renderer_device_is_not_pinned_for_multi_gpu_rendering(monkeypatch):
+    """Leave the device unset when Kit renders across several GPUs in one process.
+
+    The setting fills the renderer's active-device list, and a one-element list would cap the
+    device count at one.
+    """
+    _, kit_args = _resolve_devices_and_kit_args({"device": "cuda:1"}, monkeypatch)
+
+    assert not any("activeCudaGpus" in arg for arg in kit_args)