diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a9adb2359569..c22a99151de7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -797,6 +797,7 @@ jobs: test_rendering_lift_kuka_homo.py, test_rendering_franka_cloth.py, test_rendering_franka_soft.py, + test_rendering_franka_cable.py, test_rendering_registered_tasks.py, test_rendering_shadow_hand.py test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} @@ -843,6 +844,7 @@ jobs: test_rendering_lift_kuka_homo_kitless.py, test_rendering_franka_cloth_kitless.py, test_rendering_franka_soft_kitless.py, + test_rendering_franka_cable_kitless.py, test_rendering_shadow_hand_kitless.py test-k-expr: legacy test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} @@ -887,6 +889,7 @@ jobs: test_rendering_lift_kuka_homo_kitless.py, test_rendering_franka_cloth_kitless.py, test_rendering_franka_soft_kitless.py, + test_rendering_franka_cable_kitless.py, test_rendering_shadow_hand_kitless.py test-k-expr: ovstage test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} diff --git a/source/isaaclab_newton/changelog.d/huidongc-cable-rendering.minor.rst b/source/isaaclab_newton/changelog.d/huidongc-cable-rendering.minor.rst new file mode 100644 index 000000000000..562f0159939f --- /dev/null +++ b/source/isaaclab_newton/changelog.d/huidongc-cable-rendering.minor.rst @@ -0,0 +1,5 @@ +Added +^^^^^ + +* Added Newton cable discovery and Fabric curve sync so rendered cable curves follow + simulated segment endpoints. diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index 17eface9b9ea..fdd951d32c23 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -1630,7 +1630,60 @@ def _initialize_fabric_body_prims(stage, fabric_hierarchy, usdrt, body_bindings: @classmethod def _initialize_fabric_cable_prims(cls, stage, fabric_hierarchy, usdrt) -> None: """Initialize Fabric curve tags and packed Newton segment mappings.""" - usd_stage = get_current_stage() + cable_shapes = cls.collect_cable_segment_shape_ids() + + shape_ids: list[int] = [] + for prim_path, segment_shape_ids in cable_shapes.items(): + segment_count = len(segment_shape_ids) + prim = stage.GetPrimAtPath(prim_path) + prim.GetAttribute("points").Set(usdrt.Vt.Vec3fArray(segment_count + 1)) + usdrt.Rt.Xformable(prim).SetWorldXformFromUsd() + offset = len(shape_ids) + prim.CreateAttribute(cls._newton_cable_offset_attr, usdrt.Sdf.ValueTypeNames.UInt, custom=True).Set(offset) + prim.CreateAttribute(cls._newton_cable_count_attr, usdrt.Sdf.ValueTypeNames.UInt, custom=True).Set( + segment_count + ) + shape_ids.extend(segment_shape_ids) + + if not shape_ids: + NewtonManager._cable_shape_ids = None + NewtonManager._cable_sync_cpu_buffers = None + return + NewtonManager._cable_shape_ids = wp.array(shape_ids, dtype=wp.int32, device=PhysicsManager._device) + # TODO: CPU mirror only needed because RTX Hydra ignores GPU Fabric BasisCurves points. + # Drop these buffers and sync on device once NVBug 6502662 is fixed. + NewtonManager._cable_sync_cpu_buffers = ( + NewtonManager._cable_shape_ids.to("cpu"), + cls._model.shape_body.to("cpu"), + wp.empty_like(cls._state_0.body_q, device="cpu"), + cls._model.shape_transform.to("cpu"), + cls._model.shape_scale.to("cpu"), + ) + fabric_hierarchy.update_world_xforms() + + @classmethod + def collect_cable_segment_shape_ids(cls) -> dict[str, list[int]]: + """Map each renderable cable prim path to its ordered Newton segment shape ids. + + Concrete destination paths and segment order come from Newton ``shape_label`` values + ``{curve}_edge_capsule_{N}``. Each returned id is the index of that capsule in Newton's + shape arrays after finalization (``shape_body``, ``shape_transform``, ``shape_scale``, …), + not the ``_edge_capsule_N`` suffix. Example: + ``{"/World/envs/env_0/Cable/geometry/mesh": [42, 43, 44]}`` means segments ``0..2`` came + from labels ``.../mesh_edge_capsule_0``, ``_1``, ``_2``, and Newton assigned those shapes + indices ``42..44`` after earlier scene shapes. + + When the labeled prim exists on the host USD stage, topology is validated against that + ``BasisCurves`` prim. Kit-less replicated destinations often exist only in the render + backend; for those paths topology is validated against a source prototype via the active + clone plan. + + Returns: + Concrete cable prim paths mapped to ordered Newton segment shape ids. + """ + if cls._model is None: + return {} + cable_shapes: dict[str, dict[int, int]] = {} for shape_id, label in enumerate(cls._model.shape_label): if label is None: @@ -1644,15 +1697,42 @@ def _initialize_fabric_cable_prims(cls, stage, fabric_hierarchy, usdrt) -> None: raise RuntimeError(f"Cable visualization requires one Newton shape labeled {label}.") segments[segment] = shape_id - shape_ids: list[int] = [] + stage = get_current_stage() + + sim = SimulationContext.instance() + clone_plan = sim.get_clone_plan() if sim is not None else None + + # Filter label groups into renderable ordered shape-id lists. Validate topology on the host + # destination when present; otherwise use the clone-plan source prototype. Skip unsupported + # topology (non-linear / periodic / bad counts); require segment labels to match the curve. + ordered: dict[str, list[int]] = {} for prim_path, segments in cable_shapes.items(): - usd_prim = usd_stage.GetPrimAtPath(prim_path) - if not usd_prim.IsValid() or not usd_prim.IsA(UsdGeom.BasisCurves): - continue - if not has_deformable_curve_api(usd_prim): + prim = stage.GetPrimAtPath(prim_path) + validation_prim = prim + + # If destination is missing on host USD: validate topology against the clone source prototype. + if not prim.IsValid() and clone_plan is not None: + from isaaclab.cloner.query import path_to_source # noqa: PLC0415 + + resolved = path_to_source(clone_plan, prim_path) + if resolved is not None: + source_path, _, asset_suffix = resolved + validation_prim = stage.GetPrimAtPath(source_path + asset_suffix) + + if not ( + validation_prim.IsValid() + and validation_prim.IsA(UsdGeom.BasisCurves) + and has_deformable_curve_api(validation_prim) + ): + logger.debug( + "Skipping cable '%s': validation prim '%s' is missing, not BasisCurves, or lacks" + " DeformableCurveAPI.", + prim_path, + validation_prim.GetPath().pathString if validation_prim.IsValid() else "", + ) continue - curve = UsdGeom.BasisCurves(usd_prim) + curve = UsdGeom.BasisCurves(validation_prim) counts = curve.GetCurveVertexCountsAttr().Get() if ( len(counts) != 1 @@ -1660,37 +1740,23 @@ def _initialize_fabric_cable_prims(cls, stage, fabric_hierarchy, usdrt) -> None: or curve.GetTypeAttr().Get() != UsdGeom.Tokens.linear or curve.GetWrapAttr().Get() == UsdGeom.Tokens.periodic ): + logger.debug( + "Skipping cable '%s': unsupported BasisCurves topology (vertex_counts=%s, type=%s, wrap=%s).", + prim_path, + counts, + curve.GetTypeAttr().Get(), + curve.GetWrapAttr().Get(), + ) continue segment_count = int(counts[0]) - 1 if set(segments) != set(range(segment_count)): - raise RuntimeError(f"Cable visualization requires {segment_count} ordered segment shapes.") - segment_shape_ids = [segments[segment] for segment in range(segment_count)] - prim = stage.GetPrimAtPath(prim_path) - prim.GetAttribute("points").Set(usdrt.Vt.Vec3fArray(segment_count + 1)) - usdrt.Rt.Xformable(prim).SetWorldXformFromUsd() - offset = len(shape_ids) - prim.CreateAttribute(cls._newton_cable_offset_attr, usdrt.Sdf.ValueTypeNames.UInt, custom=True).Set(offset) - prim.CreateAttribute(cls._newton_cable_count_attr, usdrt.Sdf.ValueTypeNames.UInt, custom=True).Set( - segment_count - ) - shape_ids.extend(segment_shape_ids) + raise RuntimeError( + f"Cable visualization for '{prim_path}' requires {segment_count} ordered segment shapes." + ) + ordered[prim_path] = [segments[segment] for segment in range(segment_count)] - if not shape_ids: - NewtonManager._cable_shape_ids = None - NewtonManager._cable_sync_cpu_buffers = None - return - NewtonManager._cable_shape_ids = wp.array(shape_ids, dtype=wp.int32, device=PhysicsManager._device) - # TODO: CPU mirror only needed because RTX Hydra ignores GPU Fabric BasisCurves points. - # Drop these buffers and sync on device once NVBug 6502662 is fixed. - NewtonManager._cable_sync_cpu_buffers = ( - NewtonManager._cable_shape_ids.to("cpu"), - cls._model.shape_body.to("cpu"), - wp.empty_like(cls._state_0.body_q, device="cpu"), - cls._model.shape_transform.to("cpu"), - cls._model.shape_scale.to("cpu"), - ) - fabric_hierarchy.update_world_xforms() + return ordered @staticmethod def _initialize_fabric_particle_prims(stage, fabric_hierarchy, usdrt, prim_paths: Iterable[str]) -> None: diff --git a/source/isaaclab_ov/changelog.d/huidongc-cable-rendering.minor.rst b/source/isaaclab_ov/changelog.d/huidongc-cable-rendering.minor.rst new file mode 100644 index 000000000000..630124e48d3f --- /dev/null +++ b/source/isaaclab_ov/changelog.d/huidongc-cable-rendering.minor.rst @@ -0,0 +1,4 @@ +Added +^^^^^ + +* Added OVRTX cable curve point updates driven by Newton segment shapes. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index c760467042ba..31d679e4677b 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -89,6 +89,7 @@ ) from .ovrtx_renderer_cfg import OVRTXRendererCfg from .ovrtx_renderer_kernels import ( + compute_cable_points_world_kernel, create_camera_transforms_kernel, extract_all_tiles_kernel, generate_random_colors_from_ids_kernel, @@ -361,6 +362,13 @@ def __init__(self, cfg: OVRTXRendererCfg): self._deformable_particle_counts: list[int] = [] self._particle_visual_offsets: list[int] = [] self._particle_visual_counts: list[int] = [] + # Shared Newton cable curve state used by both legacy and ovstage write paths. + self._cable_segment_counts: list[int] = [] + self._cable_max_points: int = 0 + self._cable_shape_ids: wp.array | None = None + self._cable_offsets: wp.array | None = None + self._cable_counts: wp.array | None = None + self._cable_points: wp.array | None = None self._initialized_scene = False self._exported_usd_string: str | None = None self._camera_rel_path: str | None = None @@ -501,6 +509,10 @@ def _init_fields_legacy(self) -> None: self._object_xform_binding = None self._deformable_points_binding = None self._particle_points_binding = None + self._particle_workaround_applied = False + self._cable_points_binding = None + # Stable Warp views into ``_cable_points`` for ASYNC GPU writes. + self._cable_point_slices: list[wp.array] = [] def _initialize_from_spec_legacy(self, spec: CameraRenderSpec): """Initialize the OVRTX renderer with internal environment cloning. @@ -584,6 +596,7 @@ def _initialize_from_spec_legacy(self, spec: CameraRenderSpec): self._setup_xform_bindings() self._setup_deformable_bindings(num_envs) self._setup_particle_bindings() + self._setup_cable_bindings() def _clone_sources_in_ovrtx(self): """Clone sources in OVRTX using the scene :class:`~isaaclab.cloner.ClonePlan`.""" @@ -793,6 +806,52 @@ def _setup_deformable_bindings_legacy(self, num_envs: int): if self._deformable_points_binding is None: raise RuntimeError("Failed to create OVRTX deformable body bindings") + def _setup_cable_bindings_legacy(self) -> None: + """Setup OVRTX ``points`` bindings for Newton cables (UsdGeom.BasisCurves). + + Cables are rigid segment bodies, not particles, so their curve points are derived from + ``body_q`` each frame rather than sliced out of ``particle_q``. + """ + discovered = self._discover_cable_segment_bindings() + if discovered is None: + return + + cable_prim_paths, flat_shape_ids, offsets, counts = discovered + prim_count = len(cable_prim_paths) + # Points are written in world space, so neutralise the inherited env/asset transform the + # same way the deformable path does; otherwise the transform is applied twice. + self._renderer.write_attribute( + prim_paths=cable_prim_paths, + attribute_name="omni:resetXformStack", + tensor=np.full(prim_count, True, dtype=np.bool_), + prim_mode=PrimMode.MUST_EXIST, + ) + self._renderer.write_attribute( + prim_paths=cable_prim_paths, + attribute_name="omni:xform", + tensor=np.tile(np.eye(4, dtype=np.float64), (prim_count, 1, 1)), + semantic=Semantic.XFORM_MAT4x4, + prim_mode=PrimMode.MUST_EXIST, + ) + + self._cable_points_binding = self._renderer.bind_array_attribute( + prim_paths=cable_prim_paths, + attribute_name="points", + dtype=np.float32, + shape=(3,), + prim_mode=PrimMode.MUST_EXIST, + flags=BindingFlag.OPTIMIZE, + ) + if self._cable_points_binding is None: + raise RuntimeError("Failed to create OVRTX cable point bindings") + + # Device-resident buffers select OVRTX's GPU-interop update path via DLPack device. + self._allocate_cable_device_buffers(flat_shape_ids, offsets, counts) + self._cable_point_slices = [ + self._cable_points[offset + curve : offset + curve + segment_count + 1] + for curve, (offset, segment_count) in enumerate(zip(offsets, counts, strict=True)) + ] + def _setup_particle_bindings_legacy(self) -> None: """Setup OVRTX bindings for Newton particle clouds.""" try: @@ -912,27 +971,9 @@ def _update_transforms_legacy(self) -> None: def _update_geometries_legacy(self) -> None: """Sync geometries to OVRTX.""" - if self._deformable_points_binding is None and self._particle_points_binding is None: - return - - # If self._deformable_points_binding is not None, then Newton's the current physics backend - from isaaclab_newton.physics import NewtonManager - - newton_state = NewtonManager.get_state() - if newton_state is None: - raise RuntimeError("Newton state should not be None") - - # particle_q is the world-space particle positions for all deformable bodies and - # particle clouds. A non-None geometry binding means entries were registered, so Newton - # must expose particle state; a missing particle_q here is an inconsistent state. - particle_q = getattr(newton_state, "particle_q", None) - if particle_q is None: - raise RuntimeError("Newton state has no particle_q but geometry bindings exist") - if self._deformable_points_binding is not None: self._write_particle_q_slices( self._deformable_points_binding, - particle_q, self._deformable_particle_offsets, self._deformable_particle_counts, ) @@ -940,15 +981,29 @@ def _update_geometries_legacy(self) -> None: if self._particle_points_binding is not None: self._write_particle_q_slices( self._particle_points_binding, - particle_q, self._particle_visual_offsets, self._particle_visual_counts, ) + if self._cable_points_binding is not None: + self._write_cable_points_legacy() + + def _write_cable_points_legacy(self) -> None: + """Recompute world-space cable curve points from Newton bodies and write them to OVRTX.""" + self._compute_cable_points_world() + + # Slices alias ``_cable_points``. Pass Warp's CUDA stream so OVRTX waits on-GPU instead of + # forcing a host sync. ``DataAccess.ASYNC`` + device tensors select GPU interop; ``SYNC`` or + # a host array silently takes the CPU path. + self._cable_points_binding.write( + cast(Any, self._cable_point_slices), + data_access=DataAccess.ASYNC, + cuda_stream=wp.get_stream(self._device).cuda_stream, + ) + def _write_particle_q_slices( self, binding: Any, - particle_q: wp.array, particle_offsets: list[int], particle_counts: list[int], ) -> None: @@ -956,10 +1011,19 @@ def _write_particle_q_slices( Args: binding: OVRTX array-attribute binding for the ``points`` attribute. - particle_q: Flat world-space particle positions [m], shape ``[total_particles]``. - particle_offsets: Start index of each prim's slice into :paramref:`particle_q`. + particle_offsets: Start index of each prim's slice into Newton's ``particle_q``. particle_counts: Number of particles in each prim's slice. """ + from isaaclab_newton.physics import NewtonManager + + state = NewtonManager.get_state() + if state is None: + raise RuntimeError("Newton state should not be None") + + particle_q = getattr(state, "particle_q", None) + if particle_q is None: + raise RuntimeError("Newton state has no particle_q but particle geometry bindings exist") + particle_slices = [ particle_q[particle_offset : particle_offset + particle_count] for particle_offset, particle_count in zip(particle_offsets, particle_counts, strict=True) @@ -1439,11 +1503,21 @@ def _safe_unbind(binding, name: str) -> None: self._deformable_points_binding = None _safe_unbind(self._particle_points_binding, "particle points") self._particle_points_binding = None + _safe_unbind(self._cable_points_binding, "cable points") + self._cable_points_binding = None self._deformable_particle_offsets = [] self._deformable_particle_counts = [] self._particle_visual_offsets = [] self._particle_visual_counts = [] + self._particle_workaround_applied = False + self._cable_point_slices = [] + self._cable_segment_counts = [] + self._cable_max_points = 0 + self._cable_points = None + self._cable_shape_ids = None + self._cable_offsets = None + self._cable_counts = None if self._renderer: try: @@ -1491,6 +1565,83 @@ def _setup_particle_bindings(self) -> None: else: self._setup_particle_bindings_legacy() + def _setup_cable_bindings(self) -> None: + if self._use_ovstage: + self._setup_cable_bindings_ovstage() + else: + self._setup_cable_bindings_legacy() + + @staticmethod + def _discover_cable_segment_bindings() -> tuple[list[str], list[int], list[int], list[int]] | None: + """Collect cable prim paths and packed Newton segment shape ids, or ``None`` to skip. + + Returns: + ``None`` when Newton is unavailable or no renderable cables exist. Otherwise a tuple + ``(cable_prim_paths, flat_shape_ids, offsets, counts)`` where: + + * ``cable_prim_paths``: concrete ``BasisCurves`` prim paths to bind, one per curve. + * ``flat_shape_ids``: Newton shape ids for all curves, packed contiguously in segment + order (indices into ``model.shape_body`` / ``shape_transform`` / ``shape_scale``). + * ``offsets``: start index into ``flat_shape_ids`` for each curve. + * ``counts``: number of segment shapes (capsules) for each curve. + """ + try: + from isaaclab_newton.physics import NewtonManager + except ImportError: + logger.debug("NewtonManager not available, skipping cable point bindings") + return None + + cable_segment_shape_ids = NewtonManager.collect_cable_segment_shape_ids() + if not cable_segment_shape_ids: + logger.debug("No renderable Newton cables found, skipping cable point bindings") + return None + + cable_prim_paths: list[str] = [] + flat_shape_ids: list[int] = [] + offsets: list[int] = [] + counts: list[int] = [] + for prim_path, segment_shape_ids in cable_segment_shape_ids.items(): + cable_prim_paths.append(prim_path) + offsets.append(len(flat_shape_ids)) + counts.append(len(segment_shape_ids)) + flat_shape_ids.extend(segment_shape_ids) + return cable_prim_paths, flat_shape_ids, offsets, counts + + def _allocate_cable_device_buffers(self, flat_shape_ids: list[int], offsets: list[int], counts: list[int]) -> None: + """Allocate shared Warp arrays used by the cable point kernel.""" + device = self._device + self._cable_shape_ids = wp.array(flat_shape_ids, dtype=wp.int32, device=device) + self._cable_offsets = wp.array(offsets, dtype=wp.int32, device=device) + self._cable_counts = wp.array(counts, dtype=wp.int32, device=device) + self._cable_segment_counts = counts + self._cable_max_points = max(counts) + 1 if counts else 0 + self._cable_points = wp.zeros(sum(count + 1 for count in counts), dtype=wp.vec3f, device=device) + + def _compute_cable_points_world(self) -> None: + """Launch the cable endpoint kernel into ``_cable_points``.""" + from isaaclab_newton.physics import NewtonManager + + model = NewtonManager.get_model() + state = NewtonManager.get_state() + if state is None: + raise RuntimeError("Newton state should not be None") + + wp.launch( + compute_cable_points_world_kernel, + dim=(len(self._cable_segment_counts), self._cable_max_points), + inputs=[ + self._cable_shape_ids, + self._cable_offsets, + self._cable_counts, + model.shape_body, + state.body_q, + model.shape_transform, + model.shape_scale, + self._cable_points, + ], + device=self._device, + ) + def update_transforms(self) -> None: """Sync transforms to OVRTX.""" if self._use_ovstage: @@ -1571,6 +1722,8 @@ def _init_fields_ovstage(self) -> None: self._deformable_paths_list = None self._particle_points_query = None self._particle_paths_list = None + self._cable_points_query = None + self._cable_paths_list = None def _initialize_from_spec_ovstage(self, spec: CameraRenderSpec) -> None: """Initialize the OVRTX renderer with internal environment cloning (ovstage path). @@ -1673,6 +1826,7 @@ def _initialize_from_spec_ovstage(self, spec: CameraRenderSpec) -> None: self._setup_xform_bindings_ovstage() self._setup_deformable_bindings_ovstage(num_envs) self._setup_particle_bindings_ovstage() + self._setup_cable_bindings_ovstage() # Commit all init-time writes then attach. attach_ovstage happens last so the renderer # immediately sees the fully-configured scene on its first step. @@ -1905,6 +2059,45 @@ def _setup_deformable_bindings_ovstage(self, num_envs: int) -> None: if self._deformable_points_query is None: raise RuntimeError("Failed to create OVRTX deformable body bindings") + def _setup_cable_bindings_ovstage(self) -> None: + """Setup ovstage ``points`` bindings for Newton cables (``UsdGeom.BasisCurves``). + + Mirrors :meth:`_setup_cable_bindings_legacy`, except that the per-frame write goes through a + host copy: ovstage 0.1.0's ``make_dltensor`` accepts the lanes=3 dtype override only on + numpy arrays, so a warp ``vec3f`` slice is rejected against the ``points`` column. The + endpoint kernel still runs on device; only the handover is host-side. + """ + discovered = self._discover_cable_segment_bindings() + if discovered is None: + return + + cable_prim_paths, flat_shape_ids, offsets, counts = discovered + prim_count = len(cable_prim_paths) + self._cable_paths_list = self._stage_paths.create_path_list_from_strings(cable_prim_paths) + self._cable_points_query = self._stage.query_from_path_list(self._cable_paths_list) + + # The kernel emits world space, so reset the xform stack and pin an identity omni:xform to + # stop the env-root and asset-root ancestor transforms being applied on top. + self._stage.write_attribute( + self._cable_points_query, + "omni:resetXformStack", + ordinal=self._current_ordinal, + tensors=np.full(prim_count, True, dtype=np.bool_), + is_array=False, + ).wait() + + identity_xforms = np.tile(np.eye(4, dtype=np.float64), (prim_count, 1, 1)) + self._stage.write_attribute( + self._cable_points_query, + "omni:xform", + ordinal=self._current_ordinal, + tensors=_xform_tensor_from_numpy(identity_xforms), + is_array=False, + semantic=ovstage.AttributeSemantic.MATRIX, + ).wait() + + self._allocate_cable_device_buffers(flat_shape_ids, offsets, counts) + def _setup_particle_bindings_ovstage(self) -> None: """Setup OVRTX bindings for Newton particle clouds (ovstage path).""" try: @@ -1999,47 +2192,48 @@ def _update_transforms_ovstage(self) -> None: ).wait() def _update_geometries_ovstage(self) -> None: - if self._deformable_points_query is None and self._particle_points_query is None: - return - - # If either geometry query is not None, then Newton's the current physics backend - from isaaclab_newton.physics import NewtonManager - - newton_state = NewtonManager.get_state() - if newton_state is None: - raise RuntimeError("Newton state should not be None") - - # particle_q is the world-space particle positions for all deformable bodies and particle - # clouds. A non-None geometry query means entries were registered, so Newton must - # expose particle state; a missing particle_q here is an inconsistent state. - particle_q = getattr(newton_state, "particle_q", None) - if particle_q is None: - raise RuntimeError("Newton state has no particle_q but geometry bindings exist") + if self._deformable_points_query is not None or self._particle_points_query is not None: + # If either geometry query is not None, then Newton's the current physics backend + from isaaclab_newton.physics import NewtonManager - # ovstage write_attribute needs one DLPack tensor per prim, not one flat ``particle_q`` - # plus offsets. Synchronize then copy to CPU numpy once (shared by both queries below): - # the ``points`` column is ``point3f[]`` (lanes=3), and ovstage's make_dltensor only - # accepts the lanes=3 dtype override on numpy arrays, not DLPack producers. A warp - # ``vec3f`` slice exports as ``(N, 3)`` lanes=1, which is rejected as a type mismatch - # against the lanes=3 column. - wp.synchronize_device(self._device) - particle_np = particle_q.numpy() + newton_state = NewtonManager.get_state() + if newton_state is None: + raise RuntimeError("Newton state should not be None") + + # particle_q is the world-space particle positions for all deformable bodies and particle + # clouds. A non-None geometry query means entries were registered, so Newton must + # expose particle state; a missing particle_q here is an inconsistent state. + particle_q = getattr(newton_state, "particle_q", None) + if particle_q is None: + raise RuntimeError("Newton state has no particle_q but particle geometry queries exist") + + # ovstage write_attribute needs one DLPack tensor per prim, not one flat ``particle_q`` + # plus offsets. Synchronize then copy to CPU numpy once (shared by both queries below): + # the ``points`` column is ``point3f[]`` (lanes=3), and ovstage's make_dltensor only + # accepts the lanes=3 dtype override on numpy arrays, not DLPack producers. A warp + # ``vec3f`` slice exports as ``(N, 3)`` lanes=1, which is rejected as a type mismatch + # against the lanes=3 column. + wp.synchronize_device(self._device) + particle_np = particle_q.numpy() + + if self._deformable_points_query is not None: + self._write_particle_q_slices_ovstage( + self._deformable_points_query, + particle_np, + self._deformable_particle_offsets, + self._deformable_particle_counts, + ) - if self._deformable_points_query is not None: - self._write_particle_q_slices_ovstage( - self._deformable_points_query, - particle_np, - self._deformable_particle_offsets, - self._deformable_particle_counts, - ) + if self._particle_points_query is not None: + self._write_particle_q_slices_ovstage( + self._particle_points_query, + particle_np, + self._particle_visual_offsets, + self._particle_visual_counts, + ) - if self._particle_points_query is not None: - self._write_particle_q_slices_ovstage( - self._particle_points_query, - particle_np, - self._particle_visual_offsets, - self._particle_visual_counts, - ) + if self._cable_points_query is not None: + self._write_cable_points_ovstage() def _write_particle_q_slices_ovstage( self, @@ -2052,9 +2246,8 @@ def _write_particle_q_slices_ovstage( Args: query: ovstage query selecting the prims whose ``points`` attribute is written. - particle_np: Host copy of the flat world-space particle positions [m], shape - ``[total_particles, 3]``. - particle_offsets: Start index of each prim's slice into :paramref:`particle_np`. + particle_np: Host copy of Newton particle positions [m], shape ``[total_particles, 3]``. + particle_offsets: Start index of each prim's slice into Newton's ``particle_q``. particle_counts: Number of particles in each prim's slice. """ particle_slices = [ @@ -2071,6 +2264,32 @@ def _write_particle_q_slices_ovstage( semantic=ovstage.AttributeSemantic.POINT, ).wait() + def _write_cable_points_ovstage(self) -> None: + """Recompute world-space cable curve points on device and write them through ovstage.""" + self._compute_cable_points_world() + + # ovstage write_attribute needs one DLPack tensor per prim, not one flat ``particle_q`` + # plus offsets. Synchronize then copy to CPU numpy once (shared by both queries below): + # the ``points`` column is ``point3f[]`` (lanes=3), and ovstage's make_dltensor only + # accepts the lanes=3 dtype override on numpy arrays, not DLPack producers. A warp + # ``vec3f`` slice exports as ``(N, 3)`` lanes=1, which is rejected as a type mismatch + # against the lanes=3 column. + points_np = self._cable_points.numpy() + cable_slices = [] + point_offset = 0 + for segment_count in self._cable_segment_counts: + cable_slices.append(_points_tensor_from_numpy(points_np[point_offset : point_offset + segment_count + 1])) + point_offset += segment_count + 1 + + self._stage.write_attribute( + self._cable_points_query, + "points", + ordinal=self._current_ordinal, + tensors=cable_slices, + is_array=True, + semantic=ovstage.AttributeSemantic.POINT, + ).wait() + def _update_camera_ovstage( self, render_data: OVRTXRenderData, @@ -2174,6 +2393,11 @@ def _safe_destroy_path_list(path_list, name: str) -> None: _safe_destroy_path_list(self._particle_paths_list, "particle paths") self._particle_paths_list = None + _safe_release_query(self._cable_points_query, "cable points") + self._cable_points_query = None + _safe_destroy_path_list(self._cable_paths_list, "cable paths") + self._cable_paths_list = None + self._object_newton_indices = None self._object_scales = None self._object_scales_by_path = {} @@ -2181,6 +2405,12 @@ def _safe_destroy_path_list(path_list, name: str) -> None: self._deformable_particle_counts = [] self._particle_visual_offsets = [] self._particle_visual_counts = [] + self._cable_segment_counts = [] + self._cable_max_points = 0 + self._cable_points = None + self._cable_shape_ids = None + self._cable_offsets = None + self._cable_counts = None # Detach before closing ExitStack: the renderer holds a live reference into the stage, # so detaching first avoids a use-after-free when ExitStack destroys Stage and PathDictionary. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_kernels.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_kernels.py index 50de56aef1fa..9d83de2e4757 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_kernels.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_kernels.py @@ -200,3 +200,88 @@ def sync_newton_transforms_kernel( ) ) ) + + +@wp.func +def _cable_capsule_endpoint_world( + shape_id: int, + z_sign: float, + shape_body: wp.array(dtype=wp.int32), + body_q: wp.array(dtype=wp.transformf), + shape_transform: wp.array(dtype=wp.transformf), + shape_scale: wp.array(dtype=wp.vec3f), +) -> wp.vec3f: + """World-space tip of a Newton cable capsule along local ±Z. + + Args: + shape_id: Newton shape id of the capsule segment. + z_sign: ``+1`` / ``-1`` selects the local +Z / -Z capsule tip. + shape_body: Newton shape-to-body index map. + body_q: Body poses in world frame [m, quaternion]. + shape_transform: Local shape transforms relative to body [m, quaternion]. + shape_scale: Shape scales; capsule half-length is ``shape_scale[shape_id][1]`` [m]. + + Returns: + Capsule tip position in world frame [m]. + """ + shape_q = wp.transform_multiply(body_q[shape_body[shape_id]], shape_transform[shape_id]) + return wp.transform_point(shape_q, wp.vec3f(0.0, 0.0, z_sign * shape_scale[shape_id][1])) + + +@wp.kernel(enable_backward=False) +def compute_cable_points_world_kernel( + shape_ids: wp.array(dtype=wp.int32), # type: ignore + offsets: wp.array(dtype=wp.int32), # type: ignore + counts: wp.array(dtype=wp.int32), # type: ignore + shape_body: wp.array(dtype=wp.int32), # type: ignore + body_q: wp.array(dtype=wp.transformf), # type: ignore + shape_transform: wp.array(dtype=wp.transformf), # type: ignore + shape_scale: wp.array(dtype=wp.vec3f), # type: ignore + points_out: wp.array(dtype=wp.vec3f), # type: ignore +): + """Write world-space cable curve points from Newton segment bodies. + + Same endpoint construction as NewtonManager Fabric cable sync, but emits world space because + the OVRTX cable prims pin an identity omni:xform with the transform stack reset. Endpoints + come from the first and last capsule; interior points are the midpoint of the two adjacent + capsule ends. + + Launch with ``dim=(num_curves, max_segment_count + 1)``. Each thread owns one + ``(curve, point)``; threads with ``point > counts[curve]`` return immediately. + + Args: + shape_ids: Flattened Newton segment shape ids packed by curve. + offsets: Start index into ``shape_ids`` for each curve. + counts: Segment count per curve. + shape_body: Newton shape-to-body index map. + body_q: Body poses in world frame [m, quaternion]. + shape_transform: Local shape transforms relative to body [m, quaternion]. + shape_scale: Shape scales; capsule half-length is ``shape_scale[shape][1]`` [m]. + points_out: Flattened world-space curve points [m]. + """ + curve, point = wp.tid() + segment_count = counts[curve] + if point > segment_count: + return + + offset = offsets[curve] + # ``offset`` is the prefix sum of segment counts. Every preceding curve contributes one + # additional endpoint, so its point-buffer start is ``offset + curve``. + point_base = offset + curve + if point == 0: + endpoint_w = _cable_capsule_endpoint_world( + shape_ids[offset], -1.0, shape_body, body_q, shape_transform, shape_scale + ) + elif point == segment_count: + endpoint_w = _cable_capsule_endpoint_world( + shape_ids[offset + segment_count - 1], 1.0, shape_body, body_q, shape_transform, shape_scale + ) + else: + left_w = _cable_capsule_endpoint_world( + shape_ids[offset + point - 1], 1.0, shape_body, body_q, shape_transform, shape_scale + ) + right_w = _cable_capsule_endpoint_world( + shape_ids[offset + point], -1.0, shape_body, body_q, shape_transform, shape_scale + ) + endpoint_w = (left_w + right_w) * 0.5 + points_out[point_base + point] = endpoint_w diff --git a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py index 6e3a8e2ad1cc..75ee17a08a8c 100644 --- a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py +++ b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py @@ -95,6 +95,7 @@ def _make_renderer_without_backend(device: str = "cpu") -> tuple[OVRTXRenderer, renderer.cfg = OVRTXRendererCfg() renderer._device = device renderer._camera_rel_path = "Camera" + renderer._clone_plan = None renderer._renderer = _FakeOVRTXBackend() renderer._deformable_points_binding = None renderer._deformable_particle_offsets = [] @@ -102,6 +103,12 @@ def _make_renderer_without_backend(device: str = "cpu") -> tuple[OVRTXRenderer, renderer._particle_points_binding = None renderer._particle_visual_offsets = [] renderer._particle_visual_counts = [] + renderer._particle_workaround_applied = False + # Cable bindings are set in __init__, which this fixture bypasses via __new__. Without them + # _update_geometries_legacy raises AttributeError on its cable check before reaching anything + # this module is testing. + renderer._cable_points_binding = None + renderer._cable_segment_counts = [] renderer._use_ovstage = False return renderer, renderer._renderer @@ -459,3 +466,83 @@ class _FakeStream: assert renderer._particle_points_binding.write_kwargs is not None assert renderer._particle_points_binding.write_kwargs["data_access"] is DataAccess.ASYNC assert renderer._particle_points_binding.write_kwargs["cuda_stream"] == 42 + assert len(backend.writes) == 0 + + +def _install_cable_shapes(shapes: dict[str, list[int]], monkeypatch: pytest.MonkeyPatch) -> None: + """Install a fake :meth:`NewtonManager.collect_cable_segment_shape_ids` result.""" + monkeypatch.setattr(NewtonManager, "collect_cable_segment_shape_ids", classmethod(lambda cls: dict(shapes))) + + +def test_setup_cable_bindings_binds_curve_points(monkeypatch: pytest.MonkeyPatch): + """Renderable cables create a ``points`` array binding over their curve prims.""" + renderer, backend = _make_renderer_without_backend() + _install_cable_shapes({"/World/envs/env_0/Cable/geometry/mesh": [4, 5, 6]}, monkeypatch) + + renderer._setup_cable_bindings() + + assert len(backend.calls) == 1 + assert backend.calls[0]["prim_paths"] == ["/World/envs/env_0/Cable/geometry/mesh"] + assert backend.calls[0]["attribute_name"] == "points" + assert backend.calls[0]["dtype"] is np.float32 + assert backend.calls[0]["shape"] == (3,) + assert backend.calls[0]["flags"] is BindingFlag.OPTIMIZE + assert renderer._cable_points_binding is backend.bindings["points"] + + # World-space points are written directly, so the inherited env transform must be neutralised + # or it is applied twice -- the same contract the deformable path relies on. + assert [write["attribute_name"] for write in backend.writes] == ["omni:resetXformStack", "omni:xform"] + + +def test_setup_cable_bindings_noop_without_cables(monkeypatch: pytest.MonkeyPatch): + """A scene with no renderable cables binds nothing rather than failing.""" + renderer, backend = _make_renderer_without_backend() + _install_cable_shapes({}, monkeypatch) + + renderer._setup_cable_bindings() + + assert renderer._cable_points_binding is None + assert backend.calls == [] + + +def test_update_geometries_writes_one_slice_per_cable(monkeypatch: pytest.MonkeyPatch): + """Cable updates use disjoint point slices and GPU interop for unequal-length curves.""" + renderer, _ = _make_renderer_without_backend() + _install_cable_shapes( + { + "/World/envs/env_0/Cable/geometry/mesh": [0, 1, 2], + "/World/envs/env_1/Cable/geometry/mesh": [3, 4, 5, 6, 7], + "/World/envs/env_2/Cable/geometry/mesh": [8, 9], + }, + monkeypatch, + ) + renderer._setup_cable_bindings() + + model = SimpleNamespace(shape_body=None, shape_transform=None, shape_scale=None) + monkeypatch.setattr(NewtonManager, "get_model", classmethod(lambda cls: model)) + monkeypatch.setattr(NewtonManager, "get_state", classmethod(lambda cls: SimpleNamespace(body_q=None))) + # The kernel needs a live Newton model; this test covers the slicing around it, not the maths in it. + launch_kwargs: dict = {} + + def _capture_launch(*args, **kwargs): + launch_kwargs.update(kwargs) + if args: + launch_kwargs["kernel"] = args[0] + + monkeypatch.setattr(ovrtx_renderer_module.wp, "launch", _capture_launch) + monkeypatch.setattr(ovrtx_renderer_module.wp, "get_stream", lambda device: SimpleNamespace(cuda_stream=1234)) + + renderer.update_geometries() + + assert launch_kwargs["dim"] == (3, 6) + written = renderer._cable_points_binding.written + assert written is not None + assert [len(slice_) for slice_ in written] == [4, 6, 3] + assert written[0].ptr == renderer._cable_points[0:4].ptr + assert written[1].ptr == renderer._cable_points[4:10].ptr + assert written[2].ptr == renderer._cable_points[10:13].ptr + # Zero-copy: OVRTX is handed the Warp stream so it waits on the kernel instead of forcing a host + # round-trip. Switching to SYNC would silently reintroduce a per-frame device copy, and is the + # only guard against that -- the downgrade does not raise, it just renders from a stale copy. + assert renderer._cable_points_binding.write_kwargs["data_access"] is DataAccess.ASYNC + assert renderer._cable_points_binding.write_kwargs["cuda_stream"] == 1234 diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 1cb9de1386f0..5935acfd9ca9 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -535,10 +535,13 @@ def reset_stage(self) -> None: renderer._object_xform_binding = _RecordingBinding(events, "object") renderer._deformable_points_binding = _RecordingBinding(events, "deformable") renderer._particle_points_binding = _RecordingBinding(events, "particle") + renderer._cable_points_binding = _RecordingBinding(events, "cable") renderer._deformable_particle_offsets = [0] renderer._deformable_particle_counts = [1] renderer._particle_visual_offsets = [0] renderer._particle_visual_counts = [1] + renderer._particle_workaround_applied = True + renderer._cable_segment_counts = [1] renderer._renderer = Backend() renderer._render_product_paths = ["/Render/RenderProduct_camera"] renderer._output_id_color_buffers = {"semantic_segmentation": object()} @@ -582,6 +585,8 @@ def close(self) -> None: renderer._deformable_paths_list = "deformable" renderer._particle_points_query = "particle" renderer._particle_paths_list = "particle" + renderer._cable_points_query = "cable" + renderer._cable_paths_list = "cable" renderer._object_newton_indices = object() renderer._deformable_particle_offsets = [0] renderer._deformable_particle_counts = [1] @@ -608,12 +613,15 @@ def test_ovrtx_close_releases_legacy_renderer_state(): "unbind:object", "unbind:deformable", "unbind:particle", + "unbind:cable", "reset_stage", ] assert renderer._camera_xform_binding is None assert renderer._object_xform_binding is None assert renderer._deformable_points_binding is None assert renderer._particle_points_binding is None + assert renderer._cable_points_binding is None + assert renderer._particle_workaround_applied is False assert renderer._renderer is None assert renderer._render_product_paths == [] assert renderer._output_id_color_buffers == {} @@ -642,11 +650,15 @@ def test_ovrtx_close_releases_ovstage_renderer_state(): "destroy_path_list:deformable", "release_query:particle", "destroy_path_list:particle", + "release_query:cable", + "destroy_path_list:cable", "detach_ovstage", "exit_stack_close", ] assert renderer._camera_xform_query is None assert renderer._particle_paths_list is None + assert renderer._cable_points_query is None + assert renderer._cable_paths_list is None assert renderer._object_newton_indices is None assert renderer._renderer is None assert renderer._ovstage_exit_stack is None diff --git a/source/isaaclab_tasks/changelog.d/huidongc-franka-cable-rendering-tests.skip b/source/isaaclab_tasks/changelog.d/huidongc-franka-cable-rendering-tests.skip new file mode 100644 index 000000000000..d796a4bf2fac --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/huidongc-franka-cable-rendering-tests.skip @@ -0,0 +1 @@ +Internal: Franka cable rendering tests and golden images only; no public API release note. diff --git a/source/isaaclab_tasks/test/core/test_rendering_franka_cable.py b/source/isaaclab_tasks/test/core/test_rendering_franka_cable.py new file mode 100644 index 000000000000..98af43bceda9 --- /dev/null +++ b/source/isaaclab_tasks/test/core/test_rendering_franka_cable.py @@ -0,0 +1,37 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Rendering correctness tests for the Franka cable camera setup.""" + +# Launch Isaac Sim Simulator first for kit-based combinations. +from isaaclab.app import AppLauncher + +app_launcher = AppLauncher(headless=True, enable_cameras=True) +simulation_app = app_launcher.app + +from pathlib import Path # noqa: E402 + +import pytest # noqa: E402 +from rendering_test_utils import ( # noqa: E402 + PHYSICS_RENDERER_AOV_COMBINATIONS, + make_attach_comparison_properties_fixture, + make_determinism_fixture, + make_generate_html_report_fixture, + rendering_test_franka_cable, +) + +pytestmark = pytest.mark.isaacsim_ci + +_COMPARISON_SCORES: list[dict] = [] + +_determinism_fixture = make_determinism_fixture() +_generate_html_report_fixture = make_generate_html_report_fixture(_COMPARISON_SCORES, Path(__file__).stem + ".html") +_attach_comparison_properties_fixture = make_attach_comparison_properties_fixture(_COMPARISON_SCORES) + + +@pytest.mark.parametrize("physics_backend,renderer,data_type", PHYSICS_RENDERER_AOV_COMBINATIONS) +def test_rendering_franka_cable(physics_backend, renderer, data_type): + """Test Franka cable rendering correctness across AOVs.""" + rendering_test_franka_cable(physics_backend, renderer, data_type, _COMPARISON_SCORES) diff --git a/source/isaaclab_tasks/test/core/test_rendering_franka_cable_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_franka_cable_kitless.py new file mode 100644 index 000000000000..ec1a5bb4855c --- /dev/null +++ b/source/isaaclab_tasks/test/core/test_rendering_franka_cable_kitless.py @@ -0,0 +1,36 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Kit-less rendering correctness tests for the Franka cable camera setup.""" + +from pathlib import Path + +import pytest +from rendering_test_utils import ( + make_attach_comparison_properties_fixture, + make_determinism_fixture, + make_generate_html_report_fixture, + make_kitless_rendering_params_franka, + make_require_ovlibs_install_fixture, + rendering_test_franka_cable, +) + +pytestmark = pytest.mark.isaacsim_ci + +_RENDERING_PARAMS = make_kitless_rendering_params_franka() +_COMPARISON_SCORES: list[dict] = [] + +_determinism_fixture = make_determinism_fixture() +_generate_html_report_fixture = make_generate_html_report_fixture(_COMPARISON_SCORES, Path(__file__).stem + ".html") +_attach_comparison_properties_fixture = make_attach_comparison_properties_fixture(_COMPARISON_SCORES) +_require_ovlibs_install_fixture = make_require_ovlibs_install_fixture() + + +@pytest.mark.parametrize( + "ovstage_variant,physics_backend,renderer,data_type", _RENDERING_PARAMS, indirect=["ovstage_variant"] +) +def test_rendering_franka_cable_kitless(ovstage_variant, physics_backend, renderer, data_type): + """Camera output must match golden images for the Franka cable test setup.""" + rendering_test_franka_cable(physics_backend, renderer, data_type, _COMPARISON_SCORES) diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-albedo.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-albedo.png new file mode 100644 index 000000000000..6f85f1b2ea1d --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-albedo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35fe8d6a7a579122ad87dc12f30d894f7a7129ece33e5e50449d22bfc234f1f3 +size 2675 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-depth.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-depth.png new file mode 100644 index 000000000000..e576ee8e4f30 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-depth.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7868e6438c1e333426961317d32139c07662ff5be46cc142f707272a001f1ede +size 4738 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-distance_to_camera.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-distance_to_camera.png new file mode 100644 index 000000000000..8bcb7e1c103d --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-distance_to_camera.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e2dc71345dba6f77a8b608f5372a735923edf53b88808dd47be2d035b887338 +size 8136 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-distance_to_image_plane.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-distance_to_image_plane.png new file mode 100644 index 000000000000..e576ee8e4f30 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-distance_to_image_plane.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7868e6438c1e333426961317d32139c07662ff5be46cc142f707272a001f1ede +size 4738 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-instance_id_segmentation_fast.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-instance_id_segmentation_fast.png new file mode 100644 index 000000000000..95137ff6fd87 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-instance_id_segmentation_fast.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c198236679c8c0fda9d5fbc5c35f682815954aaac724ad59667cd2611cec22a +size 3836 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-instance_segmentation.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-instance_segmentation.png new file mode 100644 index 000000000000..9d1059cb80a2 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-instance_segmentation.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58125579407d1a9150ca742ae343ad9c1eda5ff203d4244018f790b0ad9a3cfe +size 1804 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-normals.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-normals.png new file mode 100644 index 000000000000..7b24fda4cce6 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-normals.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c2ad4bbb8c3412cae620cc76e78c5fe7ef9dbb2e1af90ac5dfba1a2865a12e9 +size 10517 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-rgb.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-rgb.png new file mode 100644 index 000000000000..12a62ca30e65 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-rgb.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea375376c6e24043f7dc5f245a2685e946c826e226de36e93cfd7f163af67795 +size 46884 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-rgba.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-rgba.png new file mode 100644 index 000000000000..fd515a6e5dcb --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-rgba.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca6e30f77bc1c2135a17ab5de1394bc05d72b3cac9bd36c790e17f06833563f9 +size 52854 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-semantic_segmentation.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-semantic_segmentation.png new file mode 100644 index 000000000000..9d1059cb80a2 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-semantic_segmentation.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58125579407d1a9150ca742ae343ad9c1eda5ff203d4244018f790b0ad9a3cfe +size 1804 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-simple_shading_constant_diffuse.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-simple_shading_constant_diffuse.png new file mode 100644 index 000000000000..6b80119880ef --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-simple_shading_constant_diffuse.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f22ec1fb28545438b1033a8a5fa8779afdd3cb91508eda8c677a4d736da330b +size 7945 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-simple_shading_diffuse_mdl.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-simple_shading_diffuse_mdl.png new file mode 100644 index 000000000000..187be643e452 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-simple_shading_diffuse_mdl.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c1a7584c06da6825e11c732d1a5ac180cce3f35cbaff7884bb126331146d486 +size 7564 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-simple_shading_full_mdl.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-simple_shading_full_mdl.png new file mode 100644 index 000000000000..150a9454ed3e --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-isaacsim_rtx_renderer-simple_shading_full_mdl.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97c8942796839183e7bd638f08d687983e6f05e927ef5cd04e90bdb38f800fba +size 7925 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-depth.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-depth.png new file mode 100644 index 000000000000..3140f40e2ba5 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-depth.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9fc6a67dfb8d2e003a62244f4cd65ef2cfb2b449065802e532fbc5414f3b582 +size 5444 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-distance_to_camera.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-distance_to_camera.png new file mode 100644 index 000000000000..d4c311f7050d --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-distance_to_camera.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:462751a4310c0d07583d7cab85ea7db767d4d90a65f47fc9ae40d44dadbbc53b +size 6743 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-distance_to_image_plane.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-distance_to_image_plane.png new file mode 100644 index 000000000000..3140f40e2ba5 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-distance_to_image_plane.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9fc6a67dfb8d2e003a62244f4cd65ef2cfb2b449065802e532fbc5414f3b582 +size 5444 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-instance_segmentation.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-instance_segmentation.png new file mode 100644 index 000000000000..6ac10efa946a --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-instance_segmentation.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c4c0cd4605d44838e089262bcba37b729ff5bb3232ed9ef4f028a3eef45ad01 +size 1901 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-normals.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-normals.png new file mode 100644 index 000000000000..c6d3e113132a --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-normals.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d70f99cd99232ec0d732223e2b5c80d34d74ca5b839ac65763ec080ab90c99ae +size 12401 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-rgb.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-rgb.png new file mode 100644 index 000000000000..e5ead6596a00 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-rgb.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6c47d1cbafdfb440f2a1729ddf97a7077a98b6efd957c2f61b5f786f6777bbe +size 10457 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-rgba.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-rgba.png new file mode 100644 index 000000000000..ce00ec52a0f0 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-rgba.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:727672ed7c002780100fb767eec6301725e0c37cc9f498221b24819faac43715 +size 11584 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-semantic_segmentation.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-semantic_segmentation.png new file mode 100644 index 000000000000..6ac10efa946a --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-newton_renderer-semantic_segmentation.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c4c0cd4605d44838e089262bcba37b729ff5bb3232ed9ef4f028a3eef45ad01 +size 1901 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-albedo.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-albedo.png new file mode 100644 index 000000000000..9169d1478bb5 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-albedo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8b423f5f0d56d0a1e44caf74e2d98856ca13347a77964ab77112a89f207b71c +size 5871 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-depth.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-depth.png new file mode 100644 index 000000000000..1460ec847059 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-depth.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a7f0aa31cbf95b2260b1f69ebb4c218f986de9cf5b9306d05badbf26babd4c5 +size 5370 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-distance_to_camera.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-distance_to_camera.png new file mode 100644 index 000000000000..a2f2f1b44ccb --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-distance_to_camera.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f027ab43e600abd29f518025c0785fa854a1fd1b6b00813970d1342b784eede3 +size 6864 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-distance_to_image_plane.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-distance_to_image_plane.png new file mode 100644 index 000000000000..1460ec847059 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-distance_to_image_plane.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a7f0aa31cbf95b2260b1f69ebb4c218f986de9cf5b9306d05badbf26babd4c5 +size 5370 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-normals.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-normals.png new file mode 100644 index 000000000000..b3fa33858a29 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-normals.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e720f04eede3800878d4e342d3c0546dff2b3e3c8b4a37e31e0e3900a6342f4 +size 12101 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-rgb.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-rgb.png new file mode 100644 index 000000000000..e32afceaa760 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-rgb.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:084bd2f58a4924298b5545011bd0ce6384529a7f98fdcf51cf65ab27c94c4724 +size 55822 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-rgba.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-rgba.png new file mode 100644 index 000000000000..c7d79b7626cc --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-rgba.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1967261167585b5f8de772ff67e72b0163304a4f9d96323ab10e65728ff1a57 +size 63297 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-semantic_segmentation.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-semantic_segmentation.png new file mode 100644 index 000000000000..080452bd906f --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-semantic_segmentation.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5271e2b9e8d8a495806faa6fa607f32f769a46b53d9d79ea0c85746566fefb94 +size 1735 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-simple_shading_constant_diffuse.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-simple_shading_constant_diffuse.png new file mode 100644 index 000000000000..226107cbfd60 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-simple_shading_constant_diffuse.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b93899cc31d30c1fee22861cbb118af0c1b676e9c44a6ac5afa1d91f8a97d8a +size 8731 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-simple_shading_diffuse_mdl.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-simple_shading_diffuse_mdl.png new file mode 100644 index 000000000000..2a72dd1aa3d0 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-simple_shading_diffuse_mdl.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4075ed26ff0ab62be8cc0f4f252a8aaa895df2131cbded7ff5bc10c39e11e951 +size 7991 diff --git a/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-simple_shading_full_mdl.png b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-simple_shading_full_mdl.png new file mode 100644 index 000000000000..97ad907ca31b --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/franka_cable/newton-ovrtx_renderer-simple_shading_full_mdl.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7a3dfde5255aa9af6cfe5197c567b71953d8f6c3b0c16267ae0a7880daf473c +size 19682 diff --git a/source/isaaclab_tasks/test/rendering_test_utils.py b/source/isaaclab_tasks/test/rendering_test_utils.py index de05347d85a2..9d9e83bd26be 100644 --- a/source/isaaclab_tasks/test/rendering_test_utils.py +++ b/source/isaaclab_tasks/test/rendering_test_utils.py @@ -51,6 +51,7 @@ # Aliasing artifacts of shadow on the table. "franka_cloth": 8.0, "franka_soft": 8.0, + "franka_cable": 8.0, # Shadow-hand renderings (incl. ``Isaac-Reorient-Cube-Shadow-Camera-Direct``) show up to # ~3.28 % per-pixel diff from anti-aliasing noise along the many finger/cube edges. 5.0 gives # headroom above that without masking real regressions, which the SSIM gate still catches. @@ -673,6 +674,66 @@ def _save_comparison_image(img: Image.Image, filename: str) -> str: return path +def _rendering_gif_step_count() -> int | None: + """Return the GIF capture step count when ``ISAAC_LAB_SAVE_RENDERING_GIF`` enables recording. + + Unset, empty, or ``0`` disables recording. A positive integer is used as the step count; any + other non-empty value falls back to 60 steps. + """ + raw = os.environ.get("ISAAC_LAB_SAVE_RENDERING_GIF") + if raw is None: + return None + stripped = raw.strip() + if stripped == "" or stripped == "0": + return None + try: + steps = int(stripped) + except ValueError: + return 60 + return steps if steps > 0 else None + + +def _camera_outputs_to_pil_image(camera_outputs: dict[str, ProxyArray]) -> Image.Image: + """Convert camera AOVs to an RGB PIL image using the same display path as golden validation.""" + assert len(camera_outputs) > 0, "No camera outputs available for GIF capture." + data_type, output = next(iter(camera_outputs.items())) + tensor = output if isinstance(output, torch.Tensor) else output.torch + condition = torch.logical_or(torch.isinf(tensor), torch.isnan(tensor)) + corrected = torch.where(condition, torch.zeros_like(tensor), tensor) + normalized = normalize_camera_output_for_display(corrected, data_type) + grid = make_camera_output_grid(normalized) + ndarr = grid.mul(255).add_(0.5).clamp_(0, 255).permute(1, 2, 0).to("cpu", torch.uint8).numpy() + return Image.fromarray(ndarr).convert("RGB") + + +def save_rendering_gif( + frames: list[Image.Image], + test_name: str, + physics_backend: str, + renderer: str, + data_type: str, +) -> str: + """Write captured camera frames as a GIF in the current working directory.""" + if not frames: + raise ValueError("Cannot write a rendering GIF with no captured frames.") + + safe_test_name = test_name.replace("/", "_") + out_path = os.path.join( + os.getcwd(), + f"{safe_test_name}-{physics_backend}-{renderer}-{data_type}.gif", + ) + frames[0].save( + out_path, + format="GIF", + save_all=True, + append_images=frames[1:], + duration=50, + loop=0, + ) + logger.info("[ISAAC_LAB_SAVE_RENDERING_GIF] wrote %s (%d frames)", out_path, len(frames)) + return out_path + + def _format_bcompare_command(actual_path: str, golden_path: str) -> str: """Build a shell command that opens actual and golden images in Beyond Compare.""" return f"bcompare \\\n {actual_path} \\\n {golden_path}" @@ -1580,8 +1641,6 @@ def rendering_test_lift_kuka( setup_homogeneous_envs: bool, comparison_scores: list[dict], ) -> None: - _skip_if_newton_motion_vectors(physics_backend, data_type) - from isaaclab.envs import ManagerBasedRLEnv from isaaclab.sensors import CameraCfg from isaaclab.utils.configclass import configclass @@ -1697,8 +1756,8 @@ class _KukaAllegroLiftCameraTestEnvCfg(KukaAllegroLiftCameraEnvCfg): env = None -def _configure_franka_camera_test_env_cfg(env_cfg: Any, data_type: str) -> None: - """Apply deterministic golden rendering test overrides to a resolved Franka camera config.""" +def _apply_franka_camera_golden_scene_overrides(env_cfg: Any, data_type: str) -> None: + """Shrink the scene and force image-only observations for Franka golden AOV tests.""" from isaaclab.envs import mdp as env_mdp from isaaclab.managers import ObservationGroupCfg as ObsGroup from isaaclab.managers import ObservationTermCfg as ObsTerm @@ -1726,6 +1785,11 @@ def __post_init__(self) -> None: env_cfg.scene.env_spacing = 3.0 env_cfg.scene.base_camera.data_types = [data_type] env_cfg.observations = TestFrankaCameraObservationsCfg() + + +def _configure_franka_camera_test_env_cfg(env_cfg: Any, data_type: str) -> None: + """Apply deterministic golden rendering test overrides to a resolved Franka camera config.""" + _apply_franka_camera_golden_scene_overrides(env_cfg, data_type) env_cfg.commands.deformable_pose.debug_vis = False env_cfg.events.reset_deformable.params["position_range"] = { "x": (0.0, 0.0), @@ -1873,3 +1937,88 @@ def rendering_test_franka_soft( # This invokes camera sensor and renderer cleanup explicitly before pytest teardown, otherwise OV # native code could probably complain about leaks and trigger segmentation fault. env = None + + +def rendering_test_franka_cable( + physics_backend: str, + renderer: str, + data_type: str, + comparison_scores: list[dict], +) -> None: + """Golden-image AOV coverage for the Franka cable camera env. + + Newton cables under CouplerProxy have no PhysX preset; unsupported backends skip via + ``_skip_if_physics_preset_unsupported``. Settle with zero actions so the cable drapes + deterministically before capture. OVRTX may still cull animated BasisCurves after large + motion; these goldens intentionally exercise the cable binding surface. + + When ``ISAAC_LAB_SAVE_RENDERING_GIF`` is set, skip golden validation and instead step the + env while capturing camera frames, then write a GIF to the current working directory. + """ + if renderer == "ovrtx_renderer" and data_type == "instance_segmentation": + pytest.skip("instance_segmentation crashes with the OVRTX renderer on franka_cable (NVBUG#6463802).") + + _skip_if_newton_motion_vectors(physics_backend, data_type) + + from isaaclab.envs import ManagerBasedRLEnv + + from isaaclab_tasks.core.lift.config.franka_soft.franka_cable_env_cfg import FrankaCableCameraEnvCfg + + env_cfg = FrankaCableCameraEnvCfg() + + physics_preset_name = _physics_preset_name_deformable(physics_backend) + _skip_if_physics_preset_unsupported(env_cfg, physics_preset_name) + + env_cfg = _apply_overrides_to_env_cfg(env_cfg, [f"presets={physics_preset_name},{renderer}"]) + env_cfg.events.reset_cable.params["position_range"] = { + "x": (0.0, 0.0), + "y": (0.0, 0.0), + "z": (0.0, 0.0), + } + + # Training ramps gravity from ~0 → -9.81; without this, reset installs g≈0 and the cable floats. + # Same as FrankaSoftEnvCfg.play_mode(): keep variable_gravity's fixed -9.81. + if env_cfg.curriculum is not None: + env_cfg.curriculum.gravity = None + + _apply_franka_camera_golden_scene_overrides(env_cfg, data_type) + + _maybe_enable_physx_determinism_for_motion(env_cfg, physics_backend, data_type) + + test_name = "franka_cable" + env = None + gif_steps = _rendering_gif_step_count() + + try: + env = ManagerBasedRLEnv(env_cfg) + + maybe_save_stage(test_name, physics_backend, renderer, data_type) + + zero_actions = torch.zeros(env.num_envs, env.action_manager.total_action_dim, device=env.device) + + if gif_steps is not None: + frames: list[Image.Image] = [] + for _ in range(gif_steps): + env.step(zero_actions) + frames.append(_camera_outputs_to_pil_image(env.scene.sensors["base_camera"].data.output)) + save_rendering_gif(frames, test_name, physics_backend, renderer, data_type) + return + + # Let the cable settle under gravity so golden frames are not first-frame spawn poses. + env.step(zero_actions) + + validate_camera_outputs( + test_name, + physics_backend, + renderer, + env.scene.sensors["base_camera"].data.output, + max_different_pixels_percentage=MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME[test_name], + comparison_scores=comparison_scores, + ) + finally: + if env is not None: + env.close() + + # This invokes camera sensor and renderer cleanup explicitly before pytest teardown, otherwise OV + # native code could probably complain about leaks and trigger segmentation fault. + env = None