Select the renderer device by CUDA index

source/isaaclab/changelog.d/jichuanh-fix-renderer-active-gpu-physical-index.rst

@@ -0,0 +1,10 @@
+Fixed
+^^^^^
+
+* Fixed rendering failing to start when ``CUDA_VISIBLE_DEVICES`` selects GPUs that do not begin at
+  zero, such as ``CUDA_VISIBLE_DEVICES=1,2``. Such runs aborted with ``CUDA error 700`` after
+  ``omni.gpu_foundation_factory`` reported "Failed to create any GPU devices". The renderer device
+  is now selected through ``/renderer/multiGpu/activeCudaGpus``, which takes a CUDA device index,
+  instead of ``/renderer/activeGpu``, which indexes the graphics device list that
+  ``CUDA_VISIBLE_DEVICES`` does not filter. Runs whose visible devices already begin at zero are
+  unaffected.

source/isaaclab/isaaclab/app/app_launcher.py

@@ -699,7 +699,6 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None:
     _SIM_APP_CFG_TYPES: dict[str, list[type]] = {
         "headless": [bool],
         "hide_ui": [bool, type(None)],
-        "active_gpu": [int, type(None)],
         "physics_gpu": [int],
         "multi_gpu": [bool],
         "sync_loads": [bool],
@@ -1123,10 +1122,10 @@ def _resolve_device_settings(self, launcher_args: dict):
             # pass command line variable to kit
             sys.argv.append(f"--/plugins/carb.tasking.plugin/threadCount={num_threads_per_process}")
 
-        # set rendering device. We do not need to set physics_gpu because it will automatically pick the same one
-        # as the active_gpu device. Setting physics_gpu explicitly may result in a different device to be used.
+        # ``/physics/cudaDevice`` is resolved by CUDA, so the masked index is correct there.
+        # ``activeGpu`` is deliberately left unset; the renderer device is selected in
+        # :meth:`_resolve_kit_args` instead.
         launcher_args["physics_gpu"] = self.device_id
-        launcher_args["active_gpu"] = self.device_id
 
         # Defer importing torch until after SimulationApp starts.  Importing
         # torch can import NumPy/OpenBLAS, whose at-fork handlers can crash
@@ -1267,6 +1266,13 @@ def _resolve_kit_args(self, launcher_args: dict):
         if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args):
             self._kit_args.append(argument)
 
+        # Select the renderer by CUDA index; the trailing comma keeps the setting string-typed.
+        if launcher_args.get("multi_gpu") is False:
+            argument = f"--/renderer/multiGpu/activeCudaGpus={self.device_id},"
+            setting = argument.partition("=")[0]
+            if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args):
+                self._kit_args.append(argument)
+
         sys.argv += self._kit_args
 
     def _create_app(self):

source/isaaclab/test/app/test_app_launcher_argv.py

@@ -3,11 +3,13 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-"""Tests for filtering command-line arguments before Kit startup."""
+"""Tests for the command-line arguments passed to Kit at startup."""
 
 import sys
 
-from isaaclab.app.app_launcher import _sanitize_sys_argv_for_kit
+import pytest
+
+from isaaclab.app.app_launcher import AppLauncher, _sanitize_sys_argv_for_kit
 
 
 def test_sanitize_sys_argv_removes_trailing_pytest_verbosity(monkeypatch):
@@ -36,3 +38,44 @@ def test_sanitize_sys_argv_removes_pytest_marker_pair(monkeypatch):
     result = _sanitize_sys_argv_for_kit(["test_script.py", "-m", "not isaacsim_ci", "--keep"])
 
     assert result == ["test_script.py", "--keep"]
+
+
+def _resolve_devices_and_kit_args(launcher_args: dict, monkeypatch) -> tuple[dict, list[str]]:
+    """Resolve device settings and Kit arguments without constructing an ``AppLauncher``.
+
+    ``_resolve_kit_args`` extends ``sys.argv``, so the caller's argv is isolated.
+    """
+    monkeypatch.setattr(sys, "argv", ["script.py"])
+    launcher = AppLauncher.__new__(AppLauncher)
+    launcher.device_id = 0
+    launcher._deferred_cuda_device_id = None
+    launcher._xr = False
+    AppLauncher._resolve_device_settings(launcher, launcher_args)
+    AppLauncher._resolve_kit_args(launcher, launcher_args)
+    return launcher_args, launcher._kit_args
+
+
+@pytest.mark.parametrize(
+    ("launcher_args", "expected_renderer_args"),
+    [
+        pytest.param(
+            {"device": "cuda:1", "multi_gpu": False},
+            ["--/renderer/multiGpu/activeCudaGpus=1,"],
+            id="single-gpu",
+        ),
+        pytest.param(
+            {"device": "cuda:1", "multi_gpu": False, "kit_args": "--/renderer/multiGpu/activeCudaGpus=3,"},
+            ["--/renderer/multiGpu/activeCudaGpus=3,"],
+            id="explicit-kit-arg",
+        ),
+        pytest.param({"device": "cuda:1"}, [], id="multi-gpu"),
+    ],
+)
+def test_devices_selected_by_cuda_index(launcher_args, expected_renderer_args, monkeypatch):
+    """Select physics and single-GPU rendering devices by CUDA index."""
+    args, kit_args = _resolve_devices_and_kit_args(launcher_args, monkeypatch)
+
+    renderer_args = [arg for arg in kit_args if arg.startswith("--/renderer/multiGpu/activeCudaGpus=")]
+    assert renderer_args == expected_renderer_args
+    assert args["physics_gpu"] == 1
+    assert "active_gpu" not in args